Unverified Commit 8df0e487 authored by David Hoese's avatar David Hoese
Browse files

Copy ceilometer code from old subversion repos

parent af5bc944
"""This module holds all directory/path/URL information for metobs.ceilo
packages. This file should be the only place that holds this static information.
This file should not do any file manipulation at all, only return strings or pass
other static information such as what plot number goes with what data.
This module will check for environment variables to create constants of PATHS
and URL bases. It also has functions for getting file locations for any file
for a specified day.
"""
import os
import re
from datetime import datetime, timedelta
from metobs.util import RODict, CONFIG as c
CEILO_INCOMING_DIR = os.environ.get( 'CEILO_INCOMING_DIR', '/beach/incoming/Instrument_Data/METOBS/RIG/Ceilo/raw')
CEILO_PRAW_DIR = os.environ.get( 'CEILO_PRAW_DIR', '/beach/raw/aoss/ceilo')
CEILO_CACHE_DIR = os.environ.get( 'CEILO_CACHE_DIR', '/beach/cache/aoss/ceilo')
CEILO_LATEST_DIR = os.environ.get( 'CEILO_LATEST_DIR', '/beach/cache/aoss/ceilo')
CEILO_DIR_FORMAT = os.environ.get( 'CEILO_DIR_FORMAT', '%Y/%m')
CEILO_ASCII_LOC = os.environ.get( 'CEILO_ASCII_LOC', '/beach/cache/aoss/ceilo')
CEILO_NC_LOC = os.environ.get( 'CEILO_NC_LOC', '/beach/cache/aoss/ceilo')
CEILO_IMG_LOC = os.environ.get( 'CEILO_IMG_LOC', 'http://metobs.ssec.wisc.edu/pub/cache/aoss/ceilo')
inst = 'ceilo'
RE_DIGITS = re.compile(r'\d+')
def get_incoming_dir():
"Return incoming directory for specified date"
return os.path.join( CEILO_INCOMING_DIR )
def get_praw_dir(when = None):
"Return raw directory for specified date and data_type"
when = when or datetime.now()
return os.path.join( CEILO_PRAW_DIR, when.strftime( CEILO_DIR_FORMAT ) )
def get_sraw_dir(when = None):
"Return raw directory for specified date and data_type"
raise NotImplementedError("This function is not used anymore, there should only be one primary storage location")
#when = when or datetime.now()
#return os.path.join( CEILO_SRAW_DIR, when.strftime( CEILO_DIR_FORMAT ) )
def get_cache_dir(data_type, when = None):
"Return cache directory for specified date and data_type"
when = when or datetime.now()
return os.path.join( CEILO_CACHE_DIR, data_type, when.strftime( CEILO_DIR_FORMAT ) )
def get_latest_dir():
"Return latest directory"
return os.path.join( CEILO_LATEST_DIR )
def get_ascii_filename(when = None, site="rig", description=""):
"Return the standard filename of the ascii file for the specified date"
when = when or datetime.now()
return c.get_filename(site, inst, when, ext='ascii', description=description)
def get_ascii_url(when = None, site="rig", description=""):
"Return the standard url of the ascii file for the specified date"
when = when or datetime.now()
return os.path.join(CEILO_ASCII_LOC, 'ascii', when.strftime(CEILO_DIR_FORMAT),
get_ascii_filename(when, site=site, description=description))
def get_nc_filename(when = None, site="rig", description=""):
"Return the standard filename of the netCDF file for the specified date"
when = when or datetime.now()
return c.get_filename(site, inst, when, ext='nc', description=description)
def get_nc_url(when = None, site="rig", description=""):
"Return the standard url of the netCDF file for the specified date"
when = when or datetime.now()
return os.path.join(CEILO_NC_LOC, 'nc', when.strftime(CEILO_DIR_FORMAT),
get_nc_filename(when, site=site, description=description))
def get_img_filename(begin, end, ptype=1, tag='', site="rig", description=""):
"Return the standard filename of the image that goes from begin to end"
pname = _handle_plot_type(ptype)
return c.get_filename(site, inst, begin, end=end, ext="png", plotname=pname, description=description, tag=tag)
def get_quicklook_filename(begin, end, ptype=1, site='rig', description=''):
return get_img_filename(begin, end, ptype, tag='', site=site, description=description)
def get_thumbnail_filename(begin, end, ptype=1, site='rig', description=''):
return get_img_filename(begin, end, ptype, tag='tn', site=site, description=description)
def get_img_url(begin, end, ptype=1, tag='', site="rig", description=""):
"Return the standard url of the image that goes from begin to end"
return os.path.join(CEILO_IMG_LOC, 'img', begin.strftime(CEILO_DIR_FORMAT),
get_img_filename(begin,end,ptype,tag=tag, site=site, description=description))
def get_quicklook_url(begin, end, ptype=1, site='rig', description=''):
return get_img_url(begin, end, ptype, tag='', site=site, description=description)
def get_thumbnail_url(begin, end, ptype=1, site='rig', description=''):
return get_img_url(begin, end, ptype, tag='tn', site=site, description=description)
def rename_incoming(incoming_file, site='rig', description=''):
file_date = datetime(*tuple( [ int(x) for x in RE_DIGITS.findall(incoming_file) ] ))
present_date = datetime.now()
praw = get_praw_dir(when = file_date)
cache = get_cache_dir('ascii', when = file_date)
rn = get_ascii_filename(when = file_date, site=site, description=description)
remove = file_date.date() < (present_date - timedelta(days=30)).date()
return praw,cache,rn,remove
def get_type_name():
return RODict({1:'Backscatter',
2:'Cloud Base Height',
3:'Vertical Visibility'})
def _handle_plot_type(plottype=1):
if plottype == 1:
return ''
elif plottype == 2:
return ''
elif plottype == 3:
return ''
else:
raise ValueError("Plot type must be between 1-3")
__import__('pkg_resources').declare_namespace(__name__)
__docformat__ = 'Epytext'
import os
from metobs.ceilo import CONFIG as c
instrument_name = 'ceilo'
def get_ascii_name(dt, site='rig'):
"""
Make a standard filename for a tower ascii file.
@type dt: datetime
@param dt: datetime of the url to generate.
@type site: str
@param site: Name of an implemented instrument site
"""
return c.get_ascii_filename(dt)
def get_nc_name(dt, site='rig'):
"""
Make a standard filename for a ceilometer netcdf file.
@type dt: datetime
@param dt: datetime of the url to generate.
@type site: str
@param site: Name of an implemented instrument site
"""
return c.get_nc_filename(dt)
def get_thumbnail_name(begin, end=None, site='rig', plottype=1):
"""Make a standard filename for a ceilometer image thumbnail file.
"""
return c.get_image_filename(begin, end, ptype=plottype, tag='tn')
def get_nc_url(dt, site='rig', host=None):
"""
Get a URL to a ceilometer NetCDF file.
@type dt: datetime
@param dt: datetime of the url to generate.
@type site: str
@param site: Name of an implemented instrument site
@rtype: str
@return: A full URL suitable for OPeNDAP access
"""
return c.get_nc_url(dt)
def get_ascii_url(dt, site="rig", host=None):
"""Get a URL to a ceilometer ASCII file on the opendap server
@type dt: datetime
@param dt: datetime of the url to generate.
@type site: str
@param site: Name of an implemented instrument site
@rtype: str
@return: A full URL suitible for HTTP access
"""
return c.get_ascii_url(dt)
def get_thumbnail_url(begin, end=None, site="rig", host=None, plottype=1):
return c.get_image_url(begin, end, ptype=plottype, tag='tn')
def get_image_url(begin, end=None, site="rig", host=None, plottype=1):
return c.get_image_url(begin, end, ptype=plottype, tag='')
def get_type_name():
return c.get_type_name()
def _handle_plottype(plottype=1):
return c._handle_plot_type(plottype)
This diff is collapsed.
#!/usr/bin/env python
"""Ingestor for Viasala CT25K Ceilometer.
Reads messages from serial port injecting an epoch timestamp before the header
of each message. No validation of message data is done.
The output should match the legacy output written by the older Java software.
In-general, to configure ingest:
1. Install python >= 2.7
2. Install metobs.ceilo
3. Install rc_userboot.py
https://groups.ssec.wisc.edu/employee-info/for-programmers/programming-environment/rc-userboot
4. Create a config file, see example in SVN
5. Add ~/.bootrc to start ingest in screen detached session:
#!/usr/bin/env bash
screen -d -m $HOME/env/production/bin/ct25k_ingest $HOME/ceilo.cfg
6. Create cronjob script to push data:
#!/usr/bin/env bash
export TZ="UTC-00:01:00"
SRC="$HOME/data/rig_ceilo-$(date +%Y-%m-%d).ascii"
if [ -e $SRC ]; then
rsync -aux $SRC rsync://tahiti.ssec.wisc.edu/incoming/Instrument_Data/METOBS/RIG/Ceilo/raw
fi
7. Create cronjob for push
*/5 * * * * $HOME/data/rsync_data.sh &> /dev/null
"""
import logging
import time
import re
import os
import signal
from datetime import datetime, timedelta
import serial
# FIXME: bad-joo-joo
logging._levelNames[9] = 'TRACE'
logging._levelNames['TRACE'] = 9
LOG = logging.getLogger(__name__)
def epoch_secs(dt):
"""Datetime to seconds from epoch.
"""
return time.mktime(dt.utctimetuple())
def is_header(line):
"""Is the line a valid message 2 header.
"""
return re.match(r'^\x01CT[A-Z0-9][0-9]{2}[2].\x02\r\n$', line)
def process_lines(in_lines, ref_dt):
"""Process lines from the serial port. Epoch timestamps are injected
before the message header. All lines are stripped of white space before
being returned, except for a NL before the epoch timestamp.
"""
out_lines = []
num_hdrs = 0
for line in in_lines:
if is_header(line):
secs = epoch_secs(ref_dt - timedelta(seconds=15) * num_hdrs)
out_lines.append('%d\r\n' % secs)
num_hdrs += 1
out_lines.append(line)
return num_hdrs, out_lines
def init_ceilo(portdev):
"""Initialize ceilometer by sending default configuration values to
instrument. When this completes the instrument should be in autosend mode
and generating messages.
"""
port = serial.Serial(port=portdev,
baudrate=2400,
bytesize=7,
parity='E',
stopbits=1,
timeout=1)
init_commands = ("\r\r\r",
"OPEN\r\n",
"SET MESSAGE MODE AUTOSEND\r\n",
"SET MESSAGE PORT DATA\r\n",
"SET MESSAGE TYPE MSG2\r\n",
"SET OPER_MODE CONTINUOUS\r\n",
"CLOSE\r\n")
for line in init_commands:
LOG.log(9, "SEND: %s", line.strip())
port.write(line)
port.flush()
lines = port.readlines()
for l in lines:
LOG.log(9, "RECV: %s", l.strip())
port.close()
return serial.Serial(port=portdev,
baudrate=2400,
bytesize=7,
parity='E',
stopbits=1,
timeout=7.5)
def read_cfg(cfgfile):
from ConfigParser import SafeConfigParser
parser = SafeConfigParser()
parser.read(cfgfile)
return dict(parser.items('ct25k'))
def main():
from argparse import ArgumentParser
parser = ArgumentParser()
levels = {'trace':9, 'debug':logging.DEBUG, 'info':logging.INFO,
'warn':logging.WARN, 'error':logging.ERROR}
parser.add_argument('-v', dest="loglvl", choices=levels.keys(),
default='info')
parser.add_argument('-o', dest='outdir', default='.')
parser.add_argument('-f', dest='fmt', default='rig_ceilo-%Y-%m-%d.ascii',
help="output filename (supports date formatting)")
parser.add_argument('-p', dest='port', help="serial device")
parser.add_argument('-c', dest='cfgfile',
help="INI style config. If provided all other options are ignored")
args = parser.parse_args()
if args.cfgfile:
from logging.config import fileConfig
fileConfig(args.cfgfile)
config = read_cfg(args.cfgfile)
portdev = config.get('port')
filefmt = config.get('filefmt')
outdir = config.get('outdir')
else:
outdir = args.outdir
portdev = args.port
filefmt = args.fmt
loglvl = levels[args.loglvl]
logging.basicConfig(level=loglvl)
for name in ['portdev', 'filefmt', 'outdir']:
if not locals().get(name):
parser.print_usage()
parser.exit(1)
def datalog():
now = datetime.now()
if not datalog.fptr or now.date() > datalog.date:
datalog.date = now.date()
fn = datalog.date.strftime(filefmt)
fpth = os.path.join(outdir, fn)
if datalog.fptr:
LOG.info("closing %s", datalog.fptr.name)
datalog.fptr.close()
datalog.fptr = open(fpth, 'a')
LOG.info("opened %s", datalog.fptr.name)
return datalog.fptr
datalog.fptr = None
def handle_signal(*args, **kwargs):
LOG.warn("received TERM or INT")
signal.signal(signal.SIGTERM, handle_signal)
signal.signal(signal.SIGINT, handle_signal)
LOG.info("initializing ceilometer...")
port = init_ceilo(portdev)
LOG.info("starting ingest")
while True:
fptr = datalog()
LOG.log(9, "got log %s", fptr.name)
try:
in_lines = port.readlines()
LOG.debug("read %s lines", len(in_lines))
num_hdrs, out_lines = process_lines(in_lines, datetime.now())
LOG.debug("found %s potential messages", num_hdrs)
LOG.log(9, ''.join(out_lines))
LOG.debug("writing %s lines", len(out_lines))
fptr.write(''.join(out_lines))
fptr.flush()
except Exception as err:
if err.args and err.args[0] == 4:
# interrupted syscall
break
raise
try:
port.close()
except:
pass
if __name__ == '__main__':
main()
"""
For processing Ceilometer CT25K Messages
"""
import logging
from calendar import timegm
from datetime import datetime
#from urllib2 import urlopen
from metobs.util.gen_open import open_ascii as urlopen
from numpy import array
LOG = logging.getLogger(__name__)
#: Value to use for NetCDF missing_value attribute
missing_value = -9999
#: Function to determine if a value is a missing_value or not
is_missing = lambda val: val == missing_value
#: number of allowable missing values before backscatter is considered invalid
BK_THRESH = 256 * .25
START_OF_HEADING = '\x01'
START_OF_TEXT = '\x02'
END_OF_TEXT = '\x03'
MANUAL_BLOWER_CONTROL = 2
HIGH_BACKGROUND_RADIANCE = 4
TILT_ANGLE_45_PLUS = 8
MANUAL_SETTINGS_EFF = 16
SINGLE_SEQ_ON = 32
WORKING_FROM_BATT = 64
POLLING_ON = 128
UNITS_IN_METERS = 256
INTERNAL_HEATER_ON = 512
BLOWER_HEATER_ON = 1024
BLOWER_ON = 2048
BLOWER_SUSPECT = 32768
RCV_OPTICAL_XTLK_POOR = 65536
REL_HUMIDITY_HIGH = 131072
VOLTAGE_ABNORMAL = 262144
INT_TEMP_HIGH = 524288
LASER_TEMP_HIGH = 1048576
LASER_PWR_LOW = 2097152
BATTERY_LOW = 4194304
WINDOW_CONTAMINATED = 8388608
VOLTAGE_FAILURE = 268435456
RECEIVER_FAILURE = 536870912
LASER_FAILURE = 1073741824
LASER_TEMP_SHUTOFF = 2147483648
WARNING_FLAGS = [BLOWER_SUSPECT,
RCV_OPTICAL_XTLK_POOR, REL_HUMIDITY_HIGH, VOLTAGE_ABNORMAL,
INT_TEMP_HIGH, LASER_TEMP_HIGH, LASER_PWR_LOW, BATTERY_LOW,
WINDOW_CONTAMINATED ]
WARNING_MSGS = [
"Blower suspect", "Receiver optical cross-talk poor",
"Relative humidity is greater than 85%", "Voltage is low or high",
"Internal temperature is high or low", "Laser temperature is high",
"Laser power is low", "Battery is low", "Window is contaminated" ]
INFO_FLAGS = [
MANUAL_BLOWER_CONTROL, HIGH_BACKGROUND_RADIANCE,
TILT_ANGLE_45_PLUS, MANUAL_SETTINGS_EFF, SINGLE_SEQ_ON,
WORKING_FROM_BATT, POLLING_ON, UNITS_IN_METERS, INTERNAL_HEATER_ON,
BLOWER_HEATER_ON, BLOWER_ON ]
INFO_MSGS = [
"Using manual blower control", "Background radiance is high",
"Tilt angle is greater than 45 degrees",
"Manual settings are effective", "Single sequence mode is ON",
"Working from the battery", "Polling mode is ON",
"Units are in Meters", "Internal heater is ON",
"Blower heater is ON", "Blower is ON" ]
ALARM_FLAGS = [VOLTAGE_FAILURE,
RECEIVER_FAILURE, LASER_FAILURE, LASER_TEMP_SHUTOFF]
ALARM_MSGS = [
"VOLTAGE FAILURE", "RECEIVER FAILURE", "LASER FAILURE",
"LASER TEMPERATURE SHUT_OFF"]
class MessageError(StandardError):
"""General message error."""
class Message2(object):
NUM_LINES = 20
def __init__(self, lines, stamp):
"""Initialize this message, disecting the various message components
according to the CT25K Message 2 format.
The first line in a message is the header and it must start with a
SOH and end with SOT, and the entire message must end with EOT
ASCII chars to be valid. Assertion errors are raised if these
conditions are not met.
The backscatter is initially filled with L{missing_value} so if a
line is the incorrect length all of it's values will be missing_value.
Likewise, if a value cannot be parsed as a HEX string it's value will
be missing_value.
@param lines: Exactly the 20 lines that comprise a valid message
@type lines: list
@param stamp: The message time. If the 'stamp' is a naieve datetime the
tzinfo will be set to metobs.util.time.UTC, otherwise time operations
will proceed with provided tzinfo.
@type stamp: datetime
@raises MessageError: If this instance cannot be created due to an error
parsing.
"""
assert len(lines) == self.NUM_LINES, \
"A Message2 must contain %s lines" % self.NUM_LINES
self._epoch = timegm(stamp.utctimetuple())
self._lines = lines
self._stamp = stamp
self._header = lines[0]
# strip non-printables
self._header = self._header.replace(START_OF_HEADING, '')
self._header = self._header.replace(START_OF_TEXT, '')
self._msg_num = int(self.header[5])
if self._msg_num != 2:
raise MessageError("Invalid message number", self.header)
self._status_string = lines[1][21:29]
self._status_flag = lines[1][1]
self._detection_status = _int(lines[1][0])
self._first_cbh = missing_value
self._second_cbh = missing_value
self._third_cbh = missing_value
if self.detection_status == 3:
self._third_cbh = _int(lines[1][15:20])
if self.detection_status == 2:
self._second_cbh = _int(lines[1][9:14])
if self.detection_status == 1:
self._first_cbh = _int(lines[1][3:8])
if self.detection_status == 4:
self._vertical_visibility = _int(lines[1][3:8])
self._alt_highest_signal = _int(lines[1][9:14])
else:
self._vertical_visibility = missing_value
self._alt_highest_signal = missing_value
meas_params = lines[2].split()
if len(meas_params) < 10:
LOG.warn("Invalid measurement parameters for message with time %s", self.epoch)
self._scale = _int(meas_params[0])
self._measurement_mode = _str(meas_params[1])
self._laser_pulse_energy = _float(meas_params[2])
self._laser_temperature = _float(meas_params[3])
self._receiver_sensitivity = _float(meas_params[4])
self._window_contamination = _float(meas_params[5])
self._tilt_angle = _float(meas_params[6])
self._background_light = _float(meas_params[7])
self._measurement_parameters = _str(meas_params[8])
self._sum_backscatter = _float(meas_params[9])
self._backscatter = parse_backscatter(lines[3:])
# backscatter can contain no more than BK_THRESH missing values
missing = [b for b in self._backscatter if is_missing(b)]
if len(missing) >= BK_THRESH:
raise MessageError("Backscatter errors exceeds threshold")
@property
def epoch(self):
return self._epoch
@property
def lines(self):
return self._lines
@property
def stamp(self):
return self._stamp
@property
def header(self):
return self._header
@property
def msg_num(self):
return self._msg_num
@property
def status_string(self):
return self._status_string
@property
def status_flag(self):
return self._status_flag
@property
def detection_status(self):
return self._detection_status
@property
def cbh(self):
return (self._first_cbh, self._second_cbh, self._third_cbh)
@property
def first_cbh(self):
return self._first_cbh
@property
def second_cbh(self):
return self._second_cbh
@property
def third_cbh(self):
return self._third_cbh
@property
def vertical_visibility(self):
return self._vertical_visibility
@property
def alt_highest_signal(self):
return self._alt_highest_signal
@property
def scale(self):
return self._scale
@property
def measurement_mode(self):
return self._measurement_mode
@property
def laser_pulse_energy(self):