diff --git a/aossceilo/CONFIG.py b/aossceilo/CONFIG.py
index 17fbc9e928450863e7ea002e87393fd91ba27596..14edf5813e12b45333a0eaed5f62a66c7de524d5 100644
--- a/aossceilo/CONFIG.py
+++ b/aossceilo/CONFIG.py
@@ -1,118 +1,118 @@
-"""This module holds all directory/path/URL information for metobs.ceilo
-packages. This file should be the only place that holds this static information.
-
-This file should not do any file manipulation at all, only return strings or pass
-other static information such as what plot number goes with what data.
-
-This module will check for environment variables to create constants of PATHS
-and URL bases. It also has functions for getting file locations for any file
-for a specified day.
-"""
-import os
-import re
-from datetime import datetime, timedelta
-#from metobs.util import RODict, CONFIG as c
-
-CEILO_INCOMING_DIR = os.environ.get( 'CEILO_INCOMING_DIR', '/beach/incoming/Instrument_Data/METOBS/RIG/Ceilo/raw')
-CEILO_PRAW_DIR = os.environ.get( 'CEILO_PRAW_DIR', '/beach/raw/aoss/ceilo')
-CEILO_CACHE_DIR = os.environ.get( 'CEILO_CACHE_DIR', '/beach/cache/aoss/ceilo')
-CEILO_LATEST_DIR = os.environ.get( 'CEILO_LATEST_DIR', '/beach/cache/aoss/ceilo')
-CEILO_DIR_FORMAT = os.environ.get( 'CEILO_DIR_FORMAT', '%Y/%m')
-CEILO_ASCII_LOC = os.environ.get( 'CEILO_ASCII_LOC', '/beach/cache/aoss/ceilo')
-CEILO_NC_LOC = os.environ.get( 'CEILO_NC_LOC', '/beach/cache/aoss/ceilo')
-CEILO_IMG_LOC = os.environ.get( 'CEILO_IMG_LOC', 'http://metobs.ssec.wisc.edu/pub/cache/aoss/ceilo')
-inst = 'ceilo'
-RE_DIGITS = re.compile(r'\d+')
-
-
-def get_incoming_dir():
- "Return incoming directory for specified date"
- return os.path.join( CEILO_INCOMING_DIR )
-
-def get_praw_dir(when = None):
- "Return raw directory for specified date and data_type"
- when = when or datetime.now()
- return os.path.join( CEILO_PRAW_DIR, when.strftime( CEILO_DIR_FORMAT ) )
-
-def get_sraw_dir(when = None):
- "Return raw directory for specified date and data_type"
- raise NotImplementedError("This function is not used anymore, there should only be one primary storage location")
- #when = when or datetime.now()
- #return os.path.join( CEILO_SRAW_DIR, when.strftime( CEILO_DIR_FORMAT ) )
-
-def get_cache_dir(data_type, when = None):
- "Return cache directory for specified date and data_type"
- when = when or datetime.now()
- return os.path.join( CEILO_CACHE_DIR, data_type, when.strftime( CEILO_DIR_FORMAT ) )
-
-def get_latest_dir():
- "Return latest directory"
- return os.path.join( CEILO_LATEST_DIR )
-
-def get_ascii_filename(when = None, site="rig", description=""):
- "Return the standard filename of the ascii file for the specified date"
- when = when or datetime.now()
- return c.get_filename(site, inst, when, ext='ascii', description=description)
-
-def get_ascii_url(when = None, site="rig", description=""):
- "Return the standard url of the ascii file for the specified date"
- when = when or datetime.now()
- return os.path.join(CEILO_ASCII_LOC, 'ascii', when.strftime(CEILO_DIR_FORMAT),
- get_ascii_filename(when, site=site, description=description))
-
-def get_nc_filename(when = None, site="rig", description=""):
- "Return the standard filename of the netCDF file for the specified date"
- when = when or datetime.now()
- return c.get_filename(site, inst, when, ext='nc', description=description)
-
-def get_nc_url(when = None, site="rig", description=""):
- "Return the standard url of the netCDF file for the specified date"
- when = when or datetime.now()
- return os.path.join(CEILO_NC_LOC, 'nc', when.strftime(CEILO_DIR_FORMAT),
- get_nc_filename(when, site=site, description=description))
-
-def get_img_filename(begin, end, ptype=1, tag='', site="rig", description=""):
- "Return the standard filename of the image that goes from begin to end"
- pname = _handle_plot_type(ptype)
- return c.get_filename(site, inst, begin, end=end, ext="png", plotname=pname, description=description, tag=tag)
-
-def get_quicklook_filename(begin, end, ptype=1, site='rig', description=''):
- return get_img_filename(begin, end, ptype, tag='', site=site, description=description)
-
-def get_thumbnail_filename(begin, end, ptype=1, site='rig', description=''):
- return get_img_filename(begin, end, ptype, tag='tn', site=site, description=description)
-
-def get_img_url(begin, end, ptype=1, tag='', site="rig", description=""):
- "Return the standard url of the image that goes from begin to end"
- return os.path.join(CEILO_IMG_LOC, 'img', begin.strftime(CEILO_DIR_FORMAT),
- get_img_filename(begin,end,ptype,tag=tag, site=site, description=description))
-
-def get_quicklook_url(begin, end, ptype=1, site='rig', description=''):
- return get_img_url(begin, end, ptype, tag='', site=site, description=description)
-
-def get_thumbnail_url(begin, end, ptype=1, site='rig', description=''):
- return get_img_url(begin, end, ptype, tag='tn', site=site, description=description)
-
-def rename_incoming(incoming_file, site='rig', description=''):
- file_date = datetime(*tuple( [ int(x) for x in RE_DIGITS.findall(incoming_file) ] ))
- present_date = datetime.now()
- praw = get_praw_dir(when = file_date)
- cache = get_cache_dir('ascii', when = file_date)
- rn = get_ascii_filename(when = file_date, site=site, description=description)
- remove = file_date.date() < (present_date - timedelta(days=30)).date()
- return praw,cache,rn,remove
-
-def get_type_name():
- return RODict({1:'Backscatter',
- 2:'Cloud Base Height',
- 3:'Vertical Visibility'})
-
-def _handle_plot_type(plottype=1):
- if plottype == 1:
- return ''
- elif plottype == 2:
- return ''
- elif plottype == 3:
- return ''
- else:
- raise ValueError("Plot type must be between 1-3")
+"""This module holds all directory/path/URL information for metobs.ceilo
+packages. This file should be the only place that holds this static information.
+
+This file should not do any file manipulation at all, only return strings or pass
+other static information such as what plot number goes with what data.
+
+This module will check for environment variables to create constants of PATHS
+and URL bases. It also has functions for getting file locations for any file
+for a specified day.
+"""
+import os
+import re
+from datetime import datetime, timedelta
+from metobscommon.util import RODict, CONFIG as c
+
+CEILO_INCOMING_DIR = os.environ.get( 'CEILO_INCOMING_DIR', '/beach/incoming/Instrument_Data/METOBS/RIG/Ceilo/raw')
+CEILO_PRAW_DIR = os.environ.get( 'CEILO_PRAW_DIR', '/beach/raw/aoss/ceilo')
+CEILO_CACHE_DIR = os.environ.get( 'CEILO_CACHE_DIR', '/beach/cache/aoss/ceilo')
+CEILO_LATEST_DIR = os.environ.get( 'CEILO_LATEST_DIR', '/beach/cache/aoss/ceilo')
+CEILO_DIR_FORMAT = os.environ.get( 'CEILO_DIR_FORMAT', '%Y/%m')
+CEILO_ASCII_LOC = os.environ.get( 'CEILO_ASCII_LOC', '/beach/cache/aoss/ceilo')
+CEILO_NC_LOC = os.environ.get( 'CEILO_NC_LOC', '/beach/cache/aoss/ceilo')
+CEILO_IMG_LOC = os.environ.get( 'CEILO_IMG_LOC', 'http://metobs.ssec.wisc.edu/pub/cache/aoss/ceilo')
+inst = 'ceilo'
+RE_DIGITS = re.compile(r'\d+')
+
+
+def get_incoming_dir():
+ "Return incoming directory for specified date"
+ return os.path.join( CEILO_INCOMING_DIR )
+
+def get_praw_dir(when = None):
+ "Return raw directory for specified date and data_type"
+ when = when or datetime.now()
+ return os.path.join( CEILO_PRAW_DIR, when.strftime( CEILO_DIR_FORMAT ) )
+
+def get_sraw_dir(when = None):
+ "Return raw directory for specified date and data_type"
+ raise NotImplementedError("This function is not used anymore, there should only be one primary storage location")
+ #when = when or datetime.now()
+ #return os.path.join( CEILO_SRAW_DIR, when.strftime( CEILO_DIR_FORMAT ) )
+
+def get_cache_dir(data_type, when = None):
+ "Return cache directory for specified date and data_type"
+ when = when or datetime.now()
+ return os.path.join( CEILO_CACHE_DIR, data_type, when.strftime( CEILO_DIR_FORMAT ) )
+
+def get_latest_dir():
+ "Return latest directory"
+ return os.path.join( CEILO_LATEST_DIR )
+
+def get_ascii_filename(when = None, site="rig", description=""):
+ "Return the standard filename of the ascii file for the specified date"
+ when = when or datetime.now()
+ return c.get_filename(site, inst, when, ext='ascii', description=description)
+
+def get_ascii_url(when = None, site="rig", description=""):
+ "Return the standard url of the ascii file for the specified date"
+ when = when or datetime.now()
+ return os.path.join(CEILO_ASCII_LOC, 'ascii', when.strftime(CEILO_DIR_FORMAT),
+ get_ascii_filename(when, site=site, description=description))
+
+def get_nc_filename(when = None, site="rig", description=""):
+ "Return the standard filename of the netCDF file for the specified date"
+ when = when or datetime.now()
+ return c.get_filename(site, inst, when, ext='nc', description=description)
+
+def get_nc_url(when = None, site="rig", description=""):
+ "Return the standard url of the netCDF file for the specified date"
+ when = when or datetime.now()
+ return os.path.join(CEILO_NC_LOC, 'nc', when.strftime(CEILO_DIR_FORMAT),
+ get_nc_filename(when, site=site, description=description))
+
+def get_img_filename(begin, end, ptype=1, tag='', site="rig", description=""):
+ "Return the standard filename of the image that goes from begin to end"
+ pname = _handle_plot_type(ptype)
+ return c.get_filename(site, inst, begin, end=end, ext="png", plotname=pname, description=description, tag=tag)
+
+def get_quicklook_filename(begin, end, ptype=1, site='rig', description=''):
+ return get_img_filename(begin, end, ptype, tag='', site=site, description=description)
+
+def get_thumbnail_filename(begin, end, ptype=1, site='rig', description=''):
+ return get_img_filename(begin, end, ptype, tag='tn', site=site, description=description)
+
+def get_img_url(begin, end, ptype=1, tag='', site="rig", description=""):
+ "Return the standard url of the image that goes from begin to end"
+ return os.path.join(CEILO_IMG_LOC, 'img', begin.strftime(CEILO_DIR_FORMAT),
+ get_img_filename(begin,end,ptype,tag=tag, site=site, description=description))
+
+def get_quicklook_url(begin, end, ptype=1, site='rig', description=''):
+ return get_img_url(begin, end, ptype, tag='', site=site, description=description)
+
+def get_thumbnail_url(begin, end, ptype=1, site='rig', description=''):
+ return get_img_url(begin, end, ptype, tag='tn', site=site, description=description)
+
+def rename_incoming(incoming_file, site='rig', description=''):
+ file_date = datetime(*tuple( [ int(x) for x in RE_DIGITS.findall(incoming_file) ] ))
+ present_date = datetime.now()
+ praw = get_praw_dir(when = file_date)
+ cache = get_cache_dir('ascii', when = file_date)
+ rn = get_ascii_filename(when = file_date, site=site, description=description)
+ remove = file_date.date() < (present_date - timedelta(days=30)).date()
+ return praw,cache,rn,remove
+
+def get_type_name():
+ return RODict({1:'Backscatter',
+ 2:'Cloud Base Height',
+ 3:'Vertical Visibility'})
+
+def _handle_plot_type(plottype=1):
+ if plottype == 1:
+ return ''
+ elif plottype == 2:
+ return ''
+ elif plottype == 3:
+ return ''
+ else:
+ raise ValueError("Plot type must be between 1-3")
diff --git a/aossceilo/ingest.py b/aossceilo/ingest.py
index 61a3e659980b1d67fba5aec1c22344d3f25df339..a1c15e9104ad5af5513bd2ef651e4c88f9169ce3 100755
--- a/aossceilo/ingest.py
+++ b/aossceilo/ingest.py
@@ -39,6 +39,8 @@ import serial
# https://stackoverflow.com/a/13638084/433202
TRACE_LEVEL_NUM = 9
logging.addLevelName(TRACE_LEVEL_NUM, "TRACE")
+
+
def debugv(self, message, *args, **kws):
# Yes, logger takes its '*args' as 'args'.
if self.isEnabledFor(TRACE_LEVEL_NUM):
@@ -47,16 +49,19 @@ logging.Logger.debugv = debugv
LOG = logging.getLogger(__name__)
+
def epoch_secs(dt):
"""Datetime to seconds from epoch.
"""
return time.mktime(dt.utctimetuple())
+
def is_header(line):
"""Is the line a valid message 2 header.
"""
return re.match(r'^\x01CT[A-Z0-9][0-9]{2}[2].\x02\r\n$', line)
+
def process_lines(in_lines, ref_dt):
"""Process lines from the serial port. Epoch timestamps are injected
before the message header. All lines are stripped of white space before
@@ -72,6 +77,7 @@ def process_lines(in_lines, ref_dt):
out_lines.append(line)
return num_hdrs, out_lines
+
def init_ceilo(portdev):
"""Initialize ceilometer by sending default configuration values to
instrument. When this completes the instrument should be in autosend mode
@@ -107,31 +113,32 @@ def init_ceilo(portdev):
stopbits=1,
timeout=7.5)
+
def read_cfg(cfgfile):
- from ConfigParser import SafeConfigParser
- parser = SafeConfigParser()
+ from configparser import ConfigParser
+ parser = ConfigParser()
parser.read(cfgfile)
return dict(parser.items('ct25k'))
-def main():
+def main():
from argparse import ArgumentParser
parser = ArgumentParser()
- levels = {'trace':9, 'debug':logging.DEBUG, 'info':logging.INFO,
- 'warn':logging.WARN, 'error':logging.ERROR}
- parser.add_argument('-v', dest="loglvl", choices=levels.keys(),
- default='info')
+ levels = {'trace': 9, 'debug': logging.DEBUG, 'info': logging.INFO,
+ 'warn': logging.WARN, 'error': logging.ERROR}
+ parser.add_argument('-v', dest="loglvl", choices=levels.keys(),
+ default='info')
parser.add_argument('-o', dest='outdir', default='.')
parser.add_argument('-f', dest='fmt', default='rig_ceilo-%Y-%m-%d.ascii',
- help="output filename (supports date formatting)")
+ help="output filename (supports date formatting)")
parser.add_argument('-p', dest='port', help="serial device")
- parser.add_argument('-c', dest='cfgfile',
- help="INI style config. If provided all other options are ignored")
+ parser.add_argument('-c', dest='cfgfile',
+ help="INI style config. If provided all other options are ignored")
- args = parser.parse_args()
+ args = parser.parse_args()
if args.cfgfile:
from logging.config import fileConfig
@@ -167,7 +174,7 @@ def main():
datalog.fptr = None
def handle_signal(*args, **kwargs):
- LOG.warn("received TERM or INT")
+ LOG.warning("received TERM or INT")
signal.signal(signal.SIGTERM, handle_signal)
signal.signal(signal.SIGINT, handle_signal)
@@ -177,7 +184,6 @@ def main():
LOG.info("starting ingest")
while True:
-
fptr = datalog()
LOG.log(9, "got log %s", fptr.name)
@@ -200,7 +206,7 @@ def main():
try:
port.close()
- except:
+ except (serial.SerialException, IOError, OSError):
pass
if __name__ == '__main__':