Skip to content

GitLab

Unverified Commit c46fbdda authored by David Hoese's avatar David Hoese
Browse files

Fix python 3 compatiblity in ingest

parent 890ad462
Hide whitespace changes
Inline Side-by-side
aossceilo/CONFIG.py
View file @ c46fbdda
"""This module holds all directory/path/URL information for metobs.ceilo
packages. This file should be the only place that holds this static information.
This file should not do any file manipulation at all, only return strings or pass
other static information such as what plot number goes with what data.
This module will check for environment variables to create constants of PATHS
and URL bases. It also has functions for getting file locations for any file
for a specified day.
"""
import os
import re
from datetime import datetime, timedelta
#from metobs.util import RODict, CONFIG as c
CEILO_INCOMING_DIR = os.environ.get( 'CEILO_INCOMING_DIR', '/beach/incoming/Instrument_Data/METOBS/RIG/Ceilo/raw')
CEILO_PRAW_DIR = os.environ.get( 'CEILO_PRAW_DIR', '/beach/raw/aoss/ceilo')
CEILO_CACHE_DIR = os.environ.get( 'CEILO_CACHE_DIR', '/beach/cache/aoss/ceilo')
CEILO_LATEST_DIR = os.environ.get( 'CEILO_LATEST_DIR', '/beach/cache/aoss/ceilo')
CEILO_DIR_FORMAT = os.environ.get( 'CEILO_DIR_FORMAT', '%Y/%m')
CEILO_ASCII_LOC = os.environ.get( 'CEILO_ASCII_LOC', '/beach/cache/aoss/ceilo')
CEILO_NC_LOC = os.environ.get( 'CEILO_NC_LOC', '/beach/cache/aoss/ceilo')
CEILO_IMG_LOC = os.environ.get( 'CEILO_IMG_LOC', 'http://metobs.ssec.wisc.edu/pub/cache/aoss/ceilo')
inst = 'ceilo'
RE_DIGITS = re.compile(r'\d+')
def get_incoming_dir():
"Return incoming directory for specified date"
return os.path.join( CEILO_INCOMING_DIR )
def get_praw_dir(when = None):
"Return raw directory for specified date and data_type"
when = when or datetime.now()
return os.path.join( CEILO_PRAW_DIR, when.strftime( CEILO_DIR_FORMAT ) )
def get_sraw_dir(when = None):
"Return raw directory for specified date and data_type"
raise NotImplementedError("This function is not used anymore, there should only be one primary storage location")
#when = when or datetime.now()
#return os.path.join( CEILO_SRAW_DIR, when.strftime( CEILO_DIR_FORMAT ) )
def get_cache_dir(data_type, when = None):
"Return cache directory for specified date and data_type"
when = when or datetime.now()
return os.path.join( CEILO_CACHE_DIR, data_type, when.strftime( CEILO_DIR_FORMAT ) )
def get_latest_dir():
"Return latest directory"
return os.path.join( CEILO_LATEST_DIR )
def get_ascii_filename(when = None, site="rig", description=""):
"Return the standard filename of the ascii file for the specified date"
when = when or datetime.now()
return c.get_filename(site, inst, when, ext='ascii', description=description)
def get_ascii_url(when = None, site="rig", description=""):
"Return the standard url of the ascii file for the specified date"
when = when or datetime.now()
return os.path.join(CEILO_ASCII_LOC, 'ascii', when.strftime(CEILO_DIR_FORMAT),
get_ascii_filename(when, site=site, description=description))
def get_nc_filename(when = None, site="rig", description=""):
"Return the standard filename of the netCDF file for the specified date"
when = when or datetime.now()
return c.get_filename(site, inst, when, ext='nc', description=description)
def get_nc_url(when = None, site="rig", description=""):
"Return the standard url of the netCDF file for the specified date"
when = when or datetime.now()
return os.path.join(CEILO_NC_LOC, 'nc', when.strftime(CEILO_DIR_FORMAT),
get_nc_filename(when, site=site, description=description))
def get_img_filename(begin, end, ptype=1, tag='', site="rig", description=""):
"Return the standard filename of the image that goes from begin to end"
pname = _handle_plot_type(ptype)
return c.get_filename(site, inst, begin, end=end, ext="png", plotname=pname, description=description, tag=tag)
def get_quicklook_filename(begin, end, ptype=1, site='rig', description=''):
return get_img_filename(begin, end, ptype, tag='', site=site, description=description)
def get_thumbnail_filename(begin, end, ptype=1, site='rig', description=''):
return get_img_filename(begin, end, ptype, tag='tn', site=site, description=description)
def get_img_url(begin, end, ptype=1, tag='', site="rig", description=""):
"Return the standard url of the image that goes from begin to end"
return os.path.join(CEILO_IMG_LOC, 'img', begin.strftime(CEILO_DIR_FORMAT),
get_img_filename(begin,end,ptype,tag=tag, site=site, description=description))
def get_quicklook_url(begin, end, ptype=1, site='rig', description=''):
return get_img_url(begin, end, ptype, tag='', site=site, description=description)
def get_thumbnail_url(begin, end, ptype=1, site='rig', description=''):
return get_img_url(begin, end, ptype, tag='tn', site=site, description=description)
def rename_incoming(incoming_file, site='rig', description=''):
file_date = datetime(*tuple( [ int(x) for x in RE_DIGITS.findall(incoming_file) ] ))
present_date = datetime.now()
praw = get_praw_dir(when = file_date)
cache = get_cache_dir('ascii', when = file_date)
rn = get_ascii_filename(when = file_date, site=site, description=description)
remove = file_date.date() < (present_date - timedelta(days=30)).date()
return praw,cache,rn,remove
def get_type_name():
return RODict({1:'Backscatter',
2:'Cloud Base Height',
3:'Vertical Visibility'})
def _handle_plot_type(plottype=1):
if plottype == 1:
return ''
elif plottype == 2:
return ''
elif plottype == 3:
return ''
else:
raise ValueError("Plot type must be between 1-3")
"""This module holds all directory/path/URL information for metobs.ceilo
packages. This file should be the only place that holds this static information.
This file should not do any file manipulation at all, only return strings or pass
other static information such as what plot number goes with what data.
This module will check for environment variables to create constants of PATHS
and URL bases. It also has functions for getting file locations for any file
for a specified day.
"""
import os
import re
from datetime import datetime, timedelta
from metobscommon.util import RODict, CONFIG as c
CEILO_INCOMING_DIR = os.environ.get( 'CEILO_INCOMING_DIR', '/beach/incoming/Instrument_Data/METOBS/RIG/Ceilo/raw')
CEILO_PRAW_DIR = os.environ.get( 'CEILO_PRAW_DIR', '/beach/raw/aoss/ceilo')
CEILO_CACHE_DIR = os.environ.get( 'CEILO_CACHE_DIR', '/beach/cache/aoss/ceilo')
CEILO_LATEST_DIR = os.environ.get( 'CEILO_LATEST_DIR', '/beach/cache/aoss/ceilo')
CEILO_DIR_FORMAT = os.environ.get( 'CEILO_DIR_FORMAT', '%Y/%m')
CEILO_ASCII_LOC = os.environ.get( 'CEILO_ASCII_LOC', '/beach/cache/aoss/ceilo')
CEILO_NC_LOC = os.environ.get( 'CEILO_NC_LOC', '/beach/cache/aoss/ceilo')
CEILO_IMG_LOC = os.environ.get( 'CEILO_IMG_LOC', 'http://metobs.ssec.wisc.edu/pub/cache/aoss/ceilo')
inst = 'ceilo'
RE_DIGITS = re.compile(r'\d+')
def get_incoming_dir():
"Return incoming directory for specified date"
return os.path.join( CEILO_INCOMING_DIR )
def get_praw_dir(when = None):
"Return raw directory for specified date and data_type"
when = when or datetime.now()
return os.path.join( CEILO_PRAW_DIR, when.strftime( CEILO_DIR_FORMAT ) )
def get_sraw_dir(when = None):
"Return raw directory for specified date and data_type"
raise NotImplementedError("This function is not used anymore, there should only be one primary storage location")
#when = when or datetime.now()
#return os.path.join( CEILO_SRAW_DIR, when.strftime( CEILO_DIR_FORMAT ) )
def get_cache_dir(data_type, when = None):
"Return cache directory for specified date and data_type"
when = when or datetime.now()
return os.path.join( CEILO_CACHE_DIR, data_type, when.strftime( CEILO_DIR_FORMAT ) )
def get_latest_dir():
"Return latest directory"
return os.path.join( CEILO_LATEST_DIR )
def get_ascii_filename(when = None, site="rig", description=""):
"Return the standard filename of the ascii file for the specified date"
when = when or datetime.now()
return c.get_filename(site, inst, when, ext='ascii', description=description)
def get_ascii_url(when = None, site="rig", description=""):
"Return the standard url of the ascii file for the specified date"
when = when or datetime.now()
return os.path.join(CEILO_ASCII_LOC, 'ascii', when.strftime(CEILO_DIR_FORMAT),
get_ascii_filename(when, site=site, description=description))
def get_nc_filename(when = None, site="rig", description=""):
"Return the standard filename of the netCDF file for the specified date"
when = when or datetime.now()
return c.get_filename(site, inst, when, ext='nc', description=description)
def get_nc_url(when = None, site="rig", description=""):
"Return the standard url of the netCDF file for the specified date"
when = when or datetime.now()
return os.path.join(CEILO_NC_LOC, 'nc', when.strftime(CEILO_DIR_FORMAT),
get_nc_filename(when, site=site, description=description))
def get_img_filename(begin, end, ptype=1, tag='', site="rig", description=""):
"Return the standard filename of the image that goes from begin to end"
pname = _handle_plot_type(ptype)
return c.get_filename(site, inst, begin, end=end, ext="png", plotname=pname, description=description, tag=tag)
def get_quicklook_filename(begin, end, ptype=1, site='rig', description=''):
return get_img_filename(begin, end, ptype, tag='', site=site, description=description)
def get_thumbnail_filename(begin, end, ptype=1, site='rig', description=''):
return get_img_filename(begin, end, ptype, tag='tn', site=site, description=description)
def get_img_url(begin, end, ptype=1, tag='', site="rig", description=""):
"Return the standard url of the image that goes from begin to end"
return os.path.join(CEILO_IMG_LOC, 'img', begin.strftime(CEILO_DIR_FORMAT),
get_img_filename(begin,end,ptype,tag=tag, site=site, description=description))
def get_quicklook_url(begin, end, ptype=1, site='rig', description=''):
return get_img_url(begin, end, ptype, tag='', site=site, description=description)
def get_thumbnail_url(begin, end, ptype=1, site='rig', description=''):
return get_img_url(begin, end, ptype, tag='tn', site=site, description=description)
def rename_incoming(incoming_file, site='rig', description=''):
file_date = datetime(*tuple( [ int(x) for x in RE_DIGITS.findall(incoming_file) ] ))
present_date = datetime.now()
praw = get_praw_dir(when = file_date)
cache = get_cache_dir('ascii', when = file_date)
rn = get_ascii_filename(when = file_date, site=site, description=description)
remove = file_date.date() < (present_date - timedelta(days=30)).date()
return praw,cache,rn,remove
def get_type_name():
return RODict({1:'Backscatter',
2:'Cloud Base Height',
3:'Vertical Visibility'})
def _handle_plot_type(plottype=1):
if plottype == 1:
return ''
elif plottype == 2:
return ''
elif plottype == 3:
return ''
else:
raise ValueError("Plot type must be between 1-3")
aossceilo/ingest.py
View file @ c46fbdda
......@@ -39,6 +39,8 @@ import serial
# https://stackoverflow.com/a/13638084/433202
TRACE_LEVEL_NUM = 9
logging.addLevelName(TRACE_LEVEL_NUM, "TRACE")
def debugv(self, message, *args, **kws):
# Yes, logger takes its '*args' as 'args'.
if self.isEnabledFor(TRACE_LEVEL_NUM):
......@@ -47,16 +49,19 @@ logging.Logger.debugv = debugv
LOG = logging.getLogger(__name__)
def epoch_secs(dt):
"""Datetime to seconds from epoch.
"""
return time.mktime(dt.utctimetuple())
def is_header(line):
"""Is the line a valid message 2 header.
"""
return re.match(r'^\x01CT[A-Z0-9][0-9]{2}[2].\x02\r\n$', line)
def process_lines(in_lines, ref_dt):
"""Process lines from the serial port. Epoch timestamps are injected
before the message header. All lines are stripped of white space before
......@@ -72,6 +77,7 @@ def process_lines(in_lines, ref_dt):
out_lines.append(line)
return num_hdrs, out_lines
def init_ceilo(portdev):
"""Initialize ceilometer by sending default configuration values to
instrument. When this completes the instrument should be in autosend mode
......@@ -107,31 +113,32 @@ def init_ceilo(portdev):
stopbits=1,
timeout=7.5)
def read_cfg(cfgfile):
from ConfigParser import SafeConfigParser
parser = SafeConfigParser()
from configparser import ConfigParser
parser = ConfigParser()
parser.read(cfgfile)
return dict(parser.items('ct25k'))
def main():
def main():
from argparse import ArgumentParser
parser = ArgumentParser()
levels = {'trace':9, 'debug':logging.DEBUG, 'info':logging.INFO,
'warn':logging.WARN, 'error':logging.ERROR}
parser.add_argument('-v', dest="loglvl", choices=levels.keys(),
default='info')
levels = {'trace': 9, 'debug': logging.DEBUG, 'info': logging.INFO,
'warn': logging.WARN, 'error': logging.ERROR}
parser.add_argument('-v', dest="loglvl", choices=levels.keys(),
default='info')
parser.add_argument('-o', dest='outdir', default='.')
parser.add_argument('-f', dest='fmt', default='rig_ceilo-%Y-%m-%d.ascii',
help="output filename (supports date formatting)")
help="output filename (supports date formatting)")
parser.add_argument('-p', dest='port', help="serial device")
parser.add_argument('-c', dest='cfgfile',
help="INI style config. If provided all other options are ignored")
parser.add_argument('-c', dest='cfgfile',
help="INI style config. If provided all other options are ignored")
args = parser.parse_args()
args = parser.parse_args()
if args.cfgfile:
from logging.config import fileConfig
......@@ -167,7 +174,7 @@ def main():
datalog.fptr = None
def handle_signal(*args, **kwargs):
LOG.warn("received TERM or INT")
LOG.warning("received TERM or INT")
signal.signal(signal.SIGTERM, handle_signal)
signal.signal(signal.SIGINT, handle_signal)
......@@ -177,7 +184,6 @@ def main():
LOG.info("starting ingest")
while True:
fptr = datalog()
LOG.log(9, "got log %s", fptr.name)
......@@ -200,7 +206,7 @@ def main():
try:
port.close()
except:
except (serial.SerialException, IOError, OSError):
pass
if __name__ == '__main__':
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment