Copy ceilometer code from old subversion repos

AossCeilo/CONFIG.py

+"""This module holds all directory/path/URL information for metobs.ceilo
+packages.  This file should be the only place that holds this static information.
+
+This file should not do any file manipulation at all, only return strings or pass
+other static information such as what plot number goes with what data.
+
+This module will check for environment variables to create constants of PATHS
+and URL bases.  It also has functions for getting file locations for any file
+for a specified day.
+"""
+import os
+import re
+from datetime import datetime, timedelta
+from metobs.util import RODict, CONFIG as c
+
+CEILO_INCOMING_DIR = os.environ.get( 'CEILO_INCOMING_DIR', '/beach/incoming/Instrument_Data/METOBS/RIG/Ceilo/raw')
+CEILO_PRAW_DIR =     os.environ.get( 'CEILO_PRAW_DIR', '/beach/raw/aoss/ceilo')
+CEILO_CACHE_DIR =    os.environ.get( 'CEILO_CACHE_DIR', '/beach/cache/aoss/ceilo')
+CEILO_LATEST_DIR =   os.environ.get( 'CEILO_LATEST_DIR', '/beach/cache/aoss/ceilo')
+CEILO_DIR_FORMAT =   os.environ.get( 'CEILO_DIR_FORMAT', '%Y/%m')
+CEILO_ASCII_LOC =    os.environ.get( 'CEILO_ASCII_LOC', '/beach/cache/aoss/ceilo')
+CEILO_NC_LOC =       os.environ.get( 'CEILO_NC_LOC', '/beach/cache/aoss/ceilo')
+CEILO_IMG_LOC =      os.environ.get( 'CEILO_IMG_LOC', 'http://metobs.ssec.wisc.edu/pub/cache/aoss/ceilo')
+inst = 'ceilo'
+RE_DIGITS = re.compile(r'\d+')
+
+
+def get_incoming_dir():
+    "Return incoming directory for specified date"
+    return os.path.join( CEILO_INCOMING_DIR )
+
+def get_praw_dir(when = None):
+    "Return raw directory for specified date and data_type"
+    when = when or datetime.now()
+    return os.path.join( CEILO_PRAW_DIR, when.strftime( CEILO_DIR_FORMAT ) )
+
+def get_sraw_dir(when = None):
+    "Return raw directory for specified date and data_type"
+    raise NotImplementedError("This function is not used anymore, there should only be one primary storage location")
+    #when = when or datetime.now()
+    #return os.path.join( CEILO_SRAW_DIR, when.strftime( CEILO_DIR_FORMAT ) )
+
+def get_cache_dir(data_type, when = None):
+    "Return cache directory for specified date and data_type"
+    when = when or datetime.now()
+    return os.path.join( CEILO_CACHE_DIR, data_type, when.strftime( CEILO_DIR_FORMAT ) )
+
+def get_latest_dir():
+    "Return latest directory"
+    return os.path.join( CEILO_LATEST_DIR )
+
+def get_ascii_filename(when = None, site="rig", description=""):
+    "Return the standard filename of the ascii file for the specified date"
+    when = when or datetime.now()
+    return c.get_filename(site, inst, when, ext='ascii', description=description)
+
+def get_ascii_url(when = None, site="rig", description=""):
+    "Return the standard url of the ascii file for the specified date"
+    when = when or datetime.now()
+    return os.path.join(CEILO_ASCII_LOC, 'ascii', when.strftime(CEILO_DIR_FORMAT),
+            get_ascii_filename(when, site=site, description=description))
+
+def get_nc_filename(when = None, site="rig", description=""):
+    "Return the standard filename of the netCDF file for the specified date"
+    when = when or datetime.now()
+    return c.get_filename(site, inst, when, ext='nc', description=description)
+
+def get_nc_url(when = None, site="rig", description=""):
+    "Return the standard url of the netCDF file for the specified date"
+    when = when or datetime.now()
+    return os.path.join(CEILO_NC_LOC, 'nc', when.strftime(CEILO_DIR_FORMAT),
+            get_nc_filename(when, site=site, description=description))
+
+def get_img_filename(begin, end, ptype=1, tag='', site="rig", description=""):
+    "Return the standard filename of the image that goes from begin to end"
+    pname = _handle_plot_type(ptype)
+    return c.get_filename(site, inst, begin, end=end, ext="png", plotname=pname, description=description, tag=tag)
+
+def get_quicklook_filename(begin, end, ptype=1, site='rig', description=''):
+    return get_img_filename(begin, end, ptype, tag='', site=site, description=description)
+
+def get_thumbnail_filename(begin, end, ptype=1, site='rig', description=''):
+    return get_img_filename(begin, end, ptype, tag='tn', site=site, description=description)
+
+def get_img_url(begin, end, ptype=1, tag='', site="rig", description=""):
+    "Return the standard url of the image that goes from begin to end"
+    return os.path.join(CEILO_IMG_LOC, 'img', begin.strftime(CEILO_DIR_FORMAT),
+            get_img_filename(begin,end,ptype,tag=tag, site=site, description=description))
+
+def get_quicklook_url(begin, end, ptype=1, site='rig', description=''):
+    return get_img_url(begin, end, ptype, tag='', site=site, description=description)
+
+def get_thumbnail_url(begin, end, ptype=1, site='rig', description=''):
+    return get_img_url(begin, end, ptype, tag='tn', site=site, description=description)
+
+def rename_incoming(incoming_file, site='rig', description=''):
+    file_date = datetime(*tuple( [ int(x) for x in RE_DIGITS.findall(incoming_file) ] ))
+    present_date = datetime.now()
+    praw = get_praw_dir(when = file_date)
+    cache = get_cache_dir('ascii', when = file_date)
+    rn = get_ascii_filename(when = file_date, site=site, description=description)
+    remove = file_date.date() < (present_date - timedelta(days=30)).date()
+    return praw,cache,rn,remove
+
+def get_type_name():
+    return RODict({1:'Backscatter',
+            2:'Cloud Base Height',
+            3:'Vertical Visibility'})
+
+def _handle_plot_type(plottype=1):
+    if plottype == 1:
+        return ''
+    elif plottype == 2:
+        return ''
+    elif plottype == 3:
+        return ''
+    else:
+        raise ValueError("Plot type must be between 1-3")

AossCeilo/__init__.py

+__import__('pkg_resources').declare_namespace(__name__)
+__docformat__ = 'Epytext'
+
+import os
+from metobs.ceilo import CONFIG as c
+
+instrument_name = 'ceilo'
+
+def get_ascii_name(dt, site='rig'):
+    """
+    Make a standard filename for a tower ascii file.
+    
+    @type dt: datetime
+    @param dt: datetime of the url to generate.
+    @type site: str
+    @param site: Name of an implemented instrument site
+    """
+    return c.get_ascii_filename(dt)
+
+def get_nc_name(dt, site='rig'):
+    """
+    Make a standard filename for a ceilometer netcdf file.
+    
+    @type dt: datetime
+    @param dt: datetime of the url to generate.
+    @type site: str
+    @param site: Name of an implemented instrument site
+    """
+    return c.get_nc_filename(dt)
+
+def get_thumbnail_name(begin, end=None, site='rig', plottype=1):
+    """Make a standard filename for a ceilometer image thumbnail file.
+    """
+    return c.get_image_filename(begin, end, ptype=plottype, tag='tn')
+
+def get_nc_url(dt, site='rig', host=None):
+    """
+    Get a URL to a ceilometer NetCDF file.
+    
+    @type dt: datetime
+    @param dt: datetime of the url to generate.
+    @type site: str
+    @param site: Name of an implemented instrument site
+    
+    @rtype: str
+    @return: A full URL suitable for OPeNDAP access
+    """
+    return c.get_nc_url(dt)
+    
+def get_ascii_url(dt, site="rig", host=None):
+    """Get a URL to a ceilometer ASCII file on the opendap server
+    
+    @type dt: datetime
+    @param dt: datetime of the url to generate.
+    @type site: str
+    @param site: Name of an implemented instrument site
+    
+    @rtype: str
+    @return: A full URL suitible for HTTP access
+    """
+    return c.get_ascii_url(dt)
+
+def get_thumbnail_url(begin, end=None, site="rig", host=None, plottype=1):
+    return c.get_image_url(begin, end, ptype=plottype, tag='tn')
+
+def get_image_url(begin, end=None, site="rig", host=None, plottype=1):
+    return c.get_image_url(begin, end, ptype=plottype, tag='')
+
+def get_type_name():
+    return c.get_type_name()
+
+def _handle_plottype(plottype=1):
+    return c._handle_plot_type(plottype)

AossCeilo/ingest.py

+#!/usr/bin/env python
+"""Ingestor for Viasala CT25K Ceilometer.
+
+Reads messages from serial port injecting an epoch timestamp before the header
+of each message. No validation of message data is done. 
+
+The output should match the legacy output written by the older Java software.
+
+In-general, to configure ingest:
+
+    1. Install python >= 2.7
+    2. Install metobs.ceilo
+    3. Install rc_userboot.py 
+        https://groups.ssec.wisc.edu/employee-info/for-programmers/programming-environment/rc-userboot
+    4. Create a config file, see example in SVN
+    5. Add ~/.bootrc to start ingest in screen detached session:
+        #!/usr/bin/env bash
+        screen -d -m $HOME/env/production/bin/ct25k_ingest $HOME/ceilo.cfg
+    6. Create cronjob script to push data:
+        #!/usr/bin/env bash
+        export TZ="UTC-00:01:00"
+        SRC="$HOME/data/rig_ceilo-$(date +%Y-%m-%d).ascii"
+        if [ -e $SRC ]; then
+        rsync -aux $SRC rsync://tahiti.ssec.wisc.edu/incoming/Instrument_Data/METOBS/RIG/Ceilo/raw
+        fi
+    7. Create cronjob for push
+        */5 * * * * $HOME/data/rsync_data.sh &> /dev/null
+
+"""
+import logging
+import time
+import re
+import os
+import signal
+from datetime import datetime, timedelta
+
+import serial
+
+# FIXME: bad-joo-joo
+logging._levelNames[9] = 'TRACE'
+logging._levelNames['TRACE'] = 9
+
+LOG = logging.getLogger(__name__)
+
+def epoch_secs(dt):
+    """Datetime to seconds from epoch.
+    """
+    return time.mktime(dt.utctimetuple())
+
+def is_header(line):
+    """Is the line a valid message 2 header.
+    """
+    return re.match(r'^\x01CT[A-Z0-9][0-9]{2}[2].\x02\r\n$', line) 
+
+def process_lines(in_lines, ref_dt):
+    """Process lines from the serial port. Epoch timestamps are injected
+    before the message header. All lines are stripped of white space before 
+    being returned, except for a NL before the epoch timestamp.
+    """
+    out_lines = []
+    num_hdrs = 0
+    for line in in_lines:
+        if is_header(line):
+            secs = epoch_secs(ref_dt - timedelta(seconds=15) * num_hdrs)
+            out_lines.append('%d\r\n' % secs)
+            num_hdrs += 1
+        out_lines.append(line)
+    return num_hdrs, out_lines
+
+def init_ceilo(portdev):
+    """Initialize ceilometer by sending default configuration values to 
+    instrument. When this completes the instrument should be in autosend mode
+    and generating messages.
+    """
+    port = serial.Serial(port=portdev, 
+                            baudrate=2400, 
+                            bytesize=7, 
+                            parity='E', 
+                            stopbits=1,
+                            timeout=1)
+    init_commands = ("\r\r\r",
+                        "OPEN\r\n",
+                        "SET MESSAGE MODE AUTOSEND\r\n",
+                        "SET MESSAGE PORT DATA\r\n",
+                        "SET MESSAGE TYPE MSG2\r\n",
+                        "SET OPER_MODE CONTINUOUS\r\n",
+                        "CLOSE\r\n")
+    for line in init_commands:
+        LOG.log(9, "SEND: %s", line.strip())
+        port.write(line)
+        port.flush()
+        lines = port.readlines()
+        for l in lines:
+            LOG.log(9, "RECV: %s", l.strip())
+    
+    port.close()
+    
+    return serial.Serial(port=portdev, 
+                            baudrate=2400, 
+                            bytesize=7, 
+                            parity='E', 
+                            stopbits=1, 
+                            timeout=7.5)
+
+def read_cfg(cfgfile):
+
+    from ConfigParser import SafeConfigParser
+    parser = SafeConfigParser()
+    parser.read(cfgfile)
+
+    return dict(parser.items('ct25k'))
+
+def main():
+
+    from argparse import ArgumentParser
+    parser = ArgumentParser()
+
+    levels = {'trace':9, 'debug':logging.DEBUG, 'info':logging.INFO, 
+                'warn':logging.WARN, 'error':logging.ERROR}
+    parser.add_argument('-v', dest="loglvl", choices=levels.keys(), 
+        default='info')
+    parser.add_argument('-o', dest='outdir', default='.')
+    parser.add_argument('-f', dest='fmt', default='rig_ceilo-%Y-%m-%d.ascii',
+        help="output filename (supports date formatting)")
+    parser.add_argument('-p', dest='port', help="serial device")
+    parser.add_argument('-c', dest='cfgfile', 
+        help="INI style config. If provided all other options are ignored")
+
+    args =  parser.parse_args()
+
+    if args.cfgfile:
+        from logging.config import fileConfig
+        fileConfig(args.cfgfile)
+        config = read_cfg(args.cfgfile) 
+        portdev = config.get('port')
+        filefmt = config.get('filefmt')
+        outdir = config.get('outdir')
+    else:
+        outdir = args.outdir
+        portdev = args.port
+        filefmt = args.fmt
+        loglvl = levels[args.loglvl]
+        logging.basicConfig(level=loglvl)
+
+    for name in ['portdev', 'filefmt', 'outdir']:
+        if not locals().get(name):
+            parser.print_usage()
+            parser.exit(1)
+
+    def datalog():
+        now = datetime.now()
+        if not datalog.fptr or now.date() > datalog.date:
+            datalog.date = now.date()
+            fn = datalog.date.strftime(filefmt)
+            fpth = os.path.join(outdir, fn)
+            if datalog.fptr:
+                LOG.info("closing %s", datalog.fptr.name)
+                datalog.fptr.close()
+            datalog.fptr = open(fpth, 'a') 
+            LOG.info("opened %s", datalog.fptr.name)
+        return datalog.fptr
+    datalog.fptr = None
+
+    def handle_signal(*args, **kwargs):
+        LOG.warn("received TERM or INT")
+    signal.signal(signal.SIGTERM, handle_signal)
+    signal.signal(signal.SIGINT, handle_signal)
+
+    LOG.info("initializing ceilometer...")
+    port = init_ceilo(portdev)
+
+    LOG.info("starting ingest")
+
+    while True:
+
+        fptr = datalog()
+        LOG.log(9, "got log %s", fptr.name)
+
+        try:
+            in_lines = port.readlines()
+            LOG.debug("read %s lines", len(in_lines))
+            
+            num_hdrs, out_lines = process_lines(in_lines, datetime.now())
+            LOG.debug("found %s potential messages", num_hdrs)
+            LOG.log(9, ''.join(out_lines))
+            
+            LOG.debug("writing %s lines", len(out_lines))
+            fptr.write(''.join(out_lines))
+            fptr.flush()
+        except Exception as err:
+            if err.args and err.args[0] == 4:
+                # interrupted syscall
+                break
+            raise
+
+    try:
+        port.close()
+    except:
+        pass
+
+if __name__ == '__main__':
+
+    main()

AossCeilo/message.py

+"""
+For processing Ceilometer CT25K Messages
+"""
+import logging
+from calendar import timegm
+from datetime import datetime
+#from urllib2 import urlopen
+from metobs.util.gen_open import open_ascii as urlopen
+
+from numpy import array
+
+LOG = logging.getLogger(__name__)
+
+#: Value to use for NetCDF missing_value attribute
+missing_value = -9999 
+#: Function to determine if a value is a missing_value or not
+is_missing = lambda val: val == missing_value 
+
+#: number of allowable missing values before backscatter is considered invalid
+BK_THRESH = 256 * .25
+
+START_OF_HEADING = '\x01'
+START_OF_TEXT = '\x02'
+END_OF_TEXT = '\x03'
+
+MANUAL_BLOWER_CONTROL = 2
+HIGH_BACKGROUND_RADIANCE = 4
+TILT_ANGLE_45_PLUS = 8
+MANUAL_SETTINGS_EFF = 16
+SINGLE_SEQ_ON = 32
+WORKING_FROM_BATT = 64
+POLLING_ON = 128
+UNITS_IN_METERS = 256
+INTERNAL_HEATER_ON = 512
+BLOWER_HEATER_ON = 1024
+BLOWER_ON = 2048
+
+BLOWER_SUSPECT = 32768
+RCV_OPTICAL_XTLK_POOR = 65536
+REL_HUMIDITY_HIGH = 131072
+VOLTAGE_ABNORMAL = 262144
+INT_TEMP_HIGH = 524288
+LASER_TEMP_HIGH = 1048576
+LASER_PWR_LOW = 2097152
+BATTERY_LOW = 4194304
+WINDOW_CONTAMINATED = 8388608
+
+VOLTAGE_FAILURE = 268435456
+RECEIVER_FAILURE = 536870912
+LASER_FAILURE = 1073741824
+LASER_TEMP_SHUTOFF = 2147483648
+
+WARNING_FLAGS = [BLOWER_SUSPECT,
+        RCV_OPTICAL_XTLK_POOR, REL_HUMIDITY_HIGH, VOLTAGE_ABNORMAL,
+        INT_TEMP_HIGH, LASER_TEMP_HIGH, LASER_PWR_LOW, BATTERY_LOW,
+        WINDOW_CONTAMINATED ]
+
+WARNING_MSGS = [
+        "Blower suspect", "Receiver optical cross-talk poor",
+        "Relative humidity is greater than 85%", "Voltage is low or high",
+        "Internal temperature is high or low", "Laser temperature is high",
+        "Laser power is low", "Battery is low", "Window is contaminated" ]
+
+INFO_FLAGS = [
+        MANUAL_BLOWER_CONTROL, HIGH_BACKGROUND_RADIANCE,
+        TILT_ANGLE_45_PLUS, MANUAL_SETTINGS_EFF, SINGLE_SEQ_ON,
+        WORKING_FROM_BATT, POLLING_ON, UNITS_IN_METERS, INTERNAL_HEATER_ON,
+        BLOWER_HEATER_ON, BLOWER_ON ]
+
+INFO_MSGS = [
+        "Using manual blower control", "Background radiance is high",
+        "Tilt angle is greater than 45 degrees",
+        "Manual settings are effective", "Single sequence mode is ON",
+        "Working from the battery", "Polling mode is ON",
+        "Units are in Meters", "Internal heater is ON",
+        "Blower heater is ON", "Blower is ON" ]
+
+ALARM_FLAGS = [VOLTAGE_FAILURE,
+        RECEIVER_FAILURE, LASER_FAILURE, LASER_TEMP_SHUTOFF]
+
+ALARM_MSGS = [
+        "VOLTAGE FAILURE", "RECEIVER FAILURE", "LASER FAILURE",
+        "LASER TEMPERATURE SHUT_OFF"]
+
+class MessageError(StandardError):
+    """General message error."""
+
+
+class Message2(object):
+    NUM_LINES = 20
+    def __init__(self, lines, stamp):
+        """Initialize this message, disecting the various message components 
+        according to the CT25K Message 2 format.
+
+        The first line in a message is the header and it must start with a
+        SOH and end with SOT, and the entire message must end with EOT 
+        ASCII chars to be valid. Assertion errors are raised if these 
+        conditions are not met.
+
+        The backscatter is initially filled with L{missing_value} so if a
+        line is the incorrect length all of it's values will be missing_value.
+        Likewise, if a value cannot be parsed as a HEX string it's value will
+        be missing_value.
+
+        @param lines: Exactly the 20 lines that comprise a valid message
+        @type lines: list
+        @param stamp: The message time. If the 'stamp' is a naieve datetime the 
+            tzinfo will be set to metobs.util.time.UTC, otherwise time operations
+            will proceed with provided tzinfo.
+        @type stamp: datetime
+
+        @raises MessageError: If this instance cannot be created due to an error
+            parsing.
+        """
+        assert len(lines) == self.NUM_LINES, \
+            "A Message2 must contain %s lines" % self.NUM_LINES
+        
+        self._epoch = timegm(stamp.utctimetuple())
+
+        self._lines = lines
+        self._stamp = stamp
+        
+        self._header = lines[0]
+        # strip non-printables
+        self._header = self._header.replace(START_OF_HEADING, '')
+        self._header = self._header.replace(START_OF_TEXT, '')
+
+        self._msg_num = int(self.header[5])
+        if self._msg_num != 2: 
+            raise MessageError("Invalid message number", self.header)
+        
+        self._status_string = lines[1][21:29]
+        self._status_flag = lines[1][1]
+    
+        self._detection_status = _int(lines[1][0])
+        self._first_cbh = missing_value
+        self._second_cbh = missing_value
+        self._third_cbh = missing_value
+        if self.detection_status == 3:
+            self._third_cbh = _int(lines[1][15:20])
+        if self.detection_status == 2:
+            self._second_cbh = _int(lines[1][9:14])
+        if self.detection_status == 1:
+            self._first_cbh = _int(lines[1][3:8])
+        
+        if self.detection_status == 4:
+            self._vertical_visibility = _int(lines[1][3:8])
+            self._alt_highest_signal = _int(lines[1][9:14])
+        else:
+            self._vertical_visibility = missing_value
+            self._alt_highest_signal = missing_value
+
+        meas_params = lines[2].split()
+        if len(meas_params) < 10:
+            LOG.warn("Invalid measurement parameters for message with time %s", self.epoch)
+        self._scale = _int(meas_params[0])
+        self._measurement_mode = _str(meas_params[1])
+        self._laser_pulse_energy = _float(meas_params[2])
+        self._laser_temperature = _float(meas_params[3])
+        self._receiver_sensitivity = _float(meas_params[4])
+        self._window_contamination = _float(meas_params[5])
+        self._tilt_angle = _float(meas_params[6])
+        self._background_light = _float(meas_params[7])
+        self._measurement_parameters = _str(meas_params[8])
+        self._sum_backscatter = _float(meas_params[9])
+
+        self._backscatter = parse_backscatter(lines[3:])
+
+        # backscatter can contain no more than BK_THRESH missing values
+        missing = [b for b in self._backscatter if is_missing(b)]
+        if len(missing) >= BK_THRESH:
+            raise MessageError("Backscatter errors exceeds threshold")
+
+    @property
+    def epoch(self):
+        return self._epoch
+    @property
+    def lines(self):
+        return self._lines
+    @property
+    def stamp(self):
+        return self._stamp
+    @property
+    def header(self):
+        return self._header
+    @property
+    def msg_num(self):
+        return self._msg_num
+    @property
+    def status_string(self):
+        return self._status_string
+    @property
+    def status_flag(self):
+        return self._status_flag
+    @property
+    def detection_status(self):
+        return self._detection_status
+    @property
+    def cbh(self):
+        return (self._first_cbh, self._second_cbh, self._third_cbh)
+    @property
+    def first_cbh(self):
+        return self._first_cbh
+    @property
+    def second_cbh(self):
+        return self._second_cbh
+    @property
+    def third_cbh(self):
+        return self._third_cbh
+    @property
+    def vertical_visibility(self):
+        return self._vertical_visibility
+    @property
+    def alt_highest_signal(self):