Verified Commit 56a60344 authored by David Hoese's avatar David Hoese
Browse files

Add old quicklook scripts and do further python 3 porting

parent 492a25dc
include aossceilo/ceilo.ncml
......@@ -6,4 +6,4 @@ and higher level product generation.
NOTE: Most of the code in this repository was copied from the old subversion
repository. This code was executed on python 2.6/2.7 environments, but will
be run from python 3.6 from now on. Some scripts may not work out of the box.
be run from python 3.6+ from now on. Some scripts may not work out of the box.
__import__('pkg_resources').declare_namespace(__name__)
__docformat__ = 'Epytext'
import os
from aossceilo import CONFIG as c
instrument_name = 'ceilo'
......
......@@ -4,8 +4,7 @@ For processing Ceilometer CT25K Messages
import logging
from calendar import timegm
from datetime import datetime
#from urllib2 import urlopen
from metobs.util.gen_open import open_ascii as urlopen
from metobscommon.util.gen_open import open_ascii as urlopen
from numpy import array
......@@ -82,7 +81,7 @@ ALARM_MSGS = [
"VOLTAGE FAILURE", "RECEIVER FAILURE", "LASER FAILURE",
"LASER TEMPERATURE SHUT_OFF"]
class MessageError(StandardError):
class MessageError(RuntimeError):
"""General message error."""
......@@ -268,7 +267,7 @@ class Message2(object):
@return: 0 if the flag is not set, otherwise the value of the flag.
"""
return long(self.status_string, 16) & flag
return int(self.status_string, 16) & flag
def get_status_messages(self):
"""
......@@ -278,13 +277,13 @@ class Message2(object):
"""
messages = {'alarms':[], 'warnings':[], 'informational':[]}
for idx in range(len(WARNING_FLAGS)):
if long(self.status_string, 16) & WARNING_FLAGS[idx]:
if int(self.status_string, 16) & WARNING_FLAGS[idx]:
messages['warnings'].append(WARNING_MSGS[idx])
for idx in range(len(ALARM_FLAGS)):
if long(self.status_string, 16) & ALARM_FLAGS[idx]:
if int(self.status_string, 16) & ALARM_FLAGS[idx]:
messages['alarms'].append(ALARM_MSGS[idx])
for idx in range(len(INFO_FLAGS)):
if long(self.status_string, 16) & INFO_FLAGS[idx]:
if int(self.status_string, 16) & INFO_FLAGS[idx]:
messages['informational'].append(INFO_MSGS[idx])
return messages
......@@ -362,9 +361,9 @@ def parse_sky_condition(line, feet=False):
return sky_cond
def parse_backscatter(data):
if isinstance(data, (str, unicode)):
def parse_backscatter(data):
if isinstance(data, str):
lines = data.split('\n')
else:
lines = data
......@@ -459,7 +458,7 @@ def load_messages(url, on_error=None):
stamp = datetime.fromtimestamp(int(lines[0]))
try:
messages.append(Message2([l.strip() for l in lines[1:]], stamp))
except Exception, e:
except Exception as e:
if hasattr(on_error, '__call__'):
on_error(lines, e)
# disregard possible garbage between messages
......
......@@ -12,7 +12,7 @@ from calendar import timegm
from datetime import datetime, timedelta
import pkg_resources
from numpy import array, zeros, int32, int8, uint16
from numpy import array, zeros, int32
from metobscommon.util import mytime
from metobscommon.util.gen_open import open_ascii as urlopen
......@@ -214,7 +214,7 @@ def make_ceilo_files(begin, basedir=None, end=None, loc=ssec_loc, site="rig", de
url = get_ascii_url(dt, site=site, description=description)
try:
urlopen(url)
except StandardError, e:
except Exception as e:
_log.warn('%s: %s', e, url)
continue
_log.debug('Creating %s', fpth)
......
"""Set of variables used by metobs.tower.quicklook to manipulate data and plot
format.
Any other options not seen in this CONFIG file are probably available in the
metobs.tower.quicklook script flags.
"""
__author__ = 'David Hoese, SSEC'
__docformat__ = 'Epytext'
from aossceilo import CONFIG as c
"""What visual backend matplotlib should use"""
MATPLOTLIB_BACKEND='agg'
"""Use pycdf instead of the opendap dap module"""
USE_PYCDF = True
"""Do not change the key(first) value of the dictionary values.
Will be used to get the data for each NetCDF variable.
If a variable is not present in the NetCDF file manually specify the "ptype"
with the proper script flag when running image generation.
"""
CDF_VARS = {'time_var':'time',
'backscatter_var':'backscatter',
'fcbh_var':'first_cbh',
'scbh_var':'second_cbh',
'tcbh_var':'third_cbh'}
"""Valid min and max values are used to hide bad values that were measured by
an instrument. If there is an attribute of the NetCDF variable that has
this information set the value to a string of the attributes name. Otherwise
specify the integer or float value boundaries for the data. Valid min and
max must be in the units that the data was measured in.
"""
# Masked where data equals BACK_VMIN
BACK_VMIN=0 # 'valid_min'
# BACK_VMAX='valid_max'
FCBH_VMIN=-9999
#FCBH_VMAX=7000
SCBH_VMIN=-9999
#SCBH_VMAX=7000
TCBH_VMIN=-9999
#TCBH_VMAX=7000
### Data Conversions ###
"""Time can not be converted it will be calculated using epoch time and
taking (time - time.min())/3600.0 to give a 0 - 24 time range.
"""
"""The MEAS variables represent what the data was measured in, the CONV
is the units to convert the data to.
"""
"""'m' only"""
# ALT_MEAS='m' # Meters
# ALT_CONV='m' # Meters
"""Make sure to label the plots correctly in the plot settings section below"""
"""Thumbnail size is in inches."""
TN_SIZE=(1,1)
### Plot Settings ###
### Backscatter Settings ###
"""The title to place at the top of the backscatter figure
If you would like to place the date in the format MM/DD/YYYY put the string
in single quotes 'date1'
ex. BACK_TITLE="RIG Tower Meteorogram 'date1'"
These settings will only effect the quicklook because the thumbnail has no
labels.
"""
BACK_TITLE="Ceilometer Backscatter 'date1'"
BACK_YLABEL="Altitude (km)"
#BACK_YLABEL_SIZE=8
BACK_XLABEL="Time (UTC)"
#BACK_XLABEL_SIZE=8
"""Cloud base height colors and edge colors.
"""
FCBH_MARKER = 'o'
SCBH_MARKER = 'o'
TCBH_MARKER = 'o'
FCBH_COLOR = "red"
SCBH_COLOR = "#003300" # Green
TCBH_COLOR = "purple"
FCBH_ECOLOR = "white"
SCBH_ECOLOR = "white"
TCBH_ECOLOR = "white"
FCBH_MSIZE = 3
SCBH_MSIZE = 3
TCBH_MSIZE = 3
### File Settings ###
"""GET_IMG_DIR fucntion will be used to save the images to the directory:
RETURNED_PATH/image.png
the function must take a datetime object as its only parameter
"""
def get_img_dir(dt):
"""Temp. function for returning the current image/cache directory"""
return c.get_cache_dir("img", dt)
GET_IMG_DIR = get_img_dir
def get_img_filename(begin, end, ptype=1, tag="", site="rig", description=""):
"""Intermediate function for returning a standard image filename"""
return c.get_img_filename(begin, end, ptype, tag=tag, site=site, description=description)
GET_IMG_FILENAME = get_img_filename
GET_NC_URL = c.get_nc_url
"""Create plots of the ceilometer instrument data
Functions
=========
main()
make_plot()
@author: 'David Hoese'
@organization: 'SSEC'
@contact: 'david.hoese@ssec.wisc.edu'
@version: '1.5'
"""
import sys, os, logging
from metobscommon.util import mytime
import aossceilo.quicklook.CONFIG as c
import numpy
if c.USE_PYCDF:
UCDF=True
from pycdf import CDF, CDFError
else:
UCDF=False
from dap import client as dap
# Must set backend for matplotlib before importing pylab
import matplotlib as mpl
mpl.use(c.MATPLOTLIB_BACKEND)
import matplotlib.pylab as plt
_log = logging.getLogger(__name__)
def make_plot(ncfile=None, numplots=2, quality=1, site="rig", description="", format="%m/%d/%Y",
datestring=None, dt=None, blank=True):
"""Creates a plot using ceilometer data from the NetCDF file or date
specified
If no arguments are provided plots of the current day will be created.
Specifying a datetime object for C{dt} AND a filename for C{filename} will
results in a C{ValueError}.
A blank plot will only be created if a datestring or datetime object is
specified otherwise there is no way of finding the date of the attempted
plot.
Logging
=======
Log messages will be logged to I{stdout}.
Log messages occur for the following:
- Specifying a date or filename for which the file does not exist
- Specifying a date or filename for a non-NetCDF file
- Specifying a date or filename for a NetCDF file that does not have
the correct variables
- Specifying a date or filename for a NetCDF file that has corrupt
data
- Any problem while plotting the data
Plotting
========
This function will create 2 large quicklook images and 2 small
(1 in. x 1 in. default) thumbnail images by default with a black
background.
The number of plots can be changed by specifying a value for
C{numplots}. For example, if C{numplots=2}, 2 plots would be created
per day per type. So 2 larges quicklook images, 00:00:00 to 11:59:59
and 12:00:00 to 23:59:59 and the same time frames for the 2 thumbnail
images.
First the large quicklook images are created, in order from the
beginning of the day to the end, number of plots depends on the value of
C{numplots} as described above. After all large quicklook images are
created the thumbnail images are created in the same manner as the large
quicklook images.
Image Labels:
1. The Y-axis:
- Units: Kilometers
- Range: 0 km to 7 km
2. The X-axis:
- Units: Hours (UTC Timezone)
- Range: I{Varies according to C{numplots}}
3. The Colorbar:
- Units: I{Arbitrary}
- Range: 0 to 10
4. Title:
- Value: "Ceilometer Backscatter MM/DD/YYYY"
- I{Note: Thumbnails have no title by default and are different
than the title of the large quicklook images."}
- Set thumbnails to have a title by setting C{title} to
C{True}.
I{Note: Currently this code can only generate one day of data at a time}
@param filename: The complete filename of the NetCDF file containing the
ceilometer backscatter data.
NetCDF Variables required:
- time
- backscatter
@param numplots: The number of plot segments to split one day into. The
default 2 produces one 0-12 plot and one 12-24 plot with 2 thumbnails of
the same time separation.
@param title: Specify whether or not to have a title at the top of the
thumbnail images. The defualt False produces no title.
@param quality: Backscatter will be used every C{quality} piece of data.
The default C{1} uses every piece of data.
@param format: To be used with datestring to specify a format that
datestring follows. If no datestring is provided the current date will be
used.
@param datestring: Used with format to specify the date parsed from a
string following the format string of format.
@param dt: Datetime object to be used
@param backend: Specify the backend for matplotlib to use in plotting.
The default 'agg' is used for systems without displays.
@param blank: Create a blank when corrupt or no data is found for a
C{filename}; C{datestring} or datetime object required.
@type filename: str
@type numplots: integer
@type title: boolean
@type quality: integer
@type format: str
@type datestring: str
@type dt: datetime
@type backend: str
@type blank: boolean
"""
now = mytime.utc_now()
# Handle all of the parameters, Read Documentation for more information
if ncfile == None and dt == None:
if datestring == None:
datestring=now.strftime("%m/%d/%Y")
now = mytime.datetime.strptime(datestring, format)
ncfile = c.GET_NC_URL(now, site=site, description=description)
elif ncfile != None and dt != None:
raise ValueError("Specify either a ncfile or a datetime object, not both")
elif ncfile != None:
if datestring == None and blank:
raise ValueError("A datestring must be specified with blank or no blank plot can be created")
elif datestring != None:
now = mytime.datetime.strptime(datestring, format)
elif dt != None:
ncfile = c.GET_NC_URL(dt, site=site, description=description)
now = dt
datestring = now.strftime("%m/%d/%Y")
if numplots <= 0:
raise ValueError("numplots must be greater than zero")
d = []
try:
if UCDF:
d.append( CDF(ncfile) )
x = d[0].var(c.CDF_VARS['time_var']).get()
z = d[0].var(c.CDF_VARS['backscatter_var']).get()
fcbh = d[0].var(c.CDF_VARS['fcbh_var']).get()
scbh = d[0].var(c.CDF_VARS['scbh_var']).get()
tcbh = d[0].var(c.CDF_VARS['tcbh_var']).get()
else:
d.append( dap.open(ncfile) )
x = d[0][c.CDF_VARS['time_var']][::]
z = d[0][c.CDF_VARS['backscatter_var']][::]
fcbh = d[0][c.CDF_VARS['fcbh_var']][::]
scbh = d[0][c.CDF_VARS['scbh_var']][::]
tcbh = d[0][c.CDF_VARS['tcbh_var']][::]
x,y,z,fcbh,scbh,tcbh = _prepare_data(x, z, fcbh, scbh, tcbh)
except CDFError as e:
_log.exception("File Not Found or Not a properly formatted NetCDF file on %s" % ncfile)
if blank:
x = y = fcbh = scbh = tcbh = numpy.zeros((100))
z = numpy.zeros((100,100))
_log.warning("Blank plot will be created for %s" % ncfile)
else: raise e
finally:
if UCDF:
if d: [ cdf.close() for cdf in d ]
else: pass
created_files = []
try:
created_files = created_files + _plot_quicklook(plt, x, y, z, fcbh, scbh, tcbh, now, numplots, quality, c.GET_IMG_DIR(now), c.GET_IMG_FILENAME, site=site, description=description)
created_files = created_files + _plot_thumbnail(plt, x, y, z, now, numplots, quality, c.GET_IMG_DIR(now), c.GET_IMG_FILENAME, site=site, description=description)
except Exception as e:
_log.exception("Error in matplotlib or pylab during plotting")
raise e
finally:
# Just in case a figure was not closed
plt.close('all')
return created_files
def _prepare_data(x, z, fcbh, scbh, tcbh):
"""Prepare the x, y, and z data to be used in plotting
This private function should only be used by the make_plot() function.
This prepares data by doing the following:
1. X Data - Converted to a 0-24 scale
2. Y Data - A array of heights from 0 - 7 kilometers
3. Z Data - Masks data less than or equal to 0 and then finds the
absolute value of the log of the values.
@param x: The time data retrieved from the NetCDF file in make_plot()
@param z: The backscatter data retrieved from the NetCDF file in make_plot()
@type x: 1-D Numpy Array
@type z: 2-D Numpy Array
@return : x, y, z
"""
y = numpy.arange(0, z[0,:].size*30, 30)/1000.0
z = z.transpose()
x = x/3600.0
zshape = numpy.shape(z)
z = numpy.concatenate(z)
indp = numpy.nonzero(numpy.greater(z,c.BACK_VMIN))[0]
z[indp] = abs(numpy.log(z[indp]))
z = numpy.reshape(z,zshape)
z = numpy.ma.masked_where(z <= c.BACK_VMIN, z)
fcbh = numpy.ma.masked_where(fcbh == c.FCBH_VMIN, fcbh)
fcbh = fcbh/1000.0
scbh = numpy.ma.masked_where(scbh == c.SCBH_VMIN, scbh)
scbh = scbh/1000.0
tcbh = numpy.ma.masked_where(tcbh == c.TCBH_VMIN, tcbh)
tcbh = tcbh/1000.0
return x,y,z,fcbh,scbh,tcbh
def _plot_quicklook(plt, x, y, z, fcbh, scbh, tcbh, now, numplots, quality, image_dir, get_fn, site="rig", description=""):
"""Plot the large quicklook image
This private function should only be run from the make_plot() function.
Creates all large quicklook images so there are C{numplots} images.
See make_plot() for details.
@param x: The time data gathered from the NetCDF file in make_plot()
@param y: The altitude data created in the _prepare_Data()
@param z: The backscatter data gathered from the NetCDF file in make_plot()
@param now: The datetime object passed from make_plot()
@param numplots: The number of plots per day passed from make_plot()
@param quality: The number to take every other data point from make_plot()
@param image_dir: The directory to put the created image into.
@type x: 1-D Numpy Array
@type y: 1-D Numpy Array
@type z: 2-D Numpy Array
@type now: datetime object
@type numplots: int
@type quality: int
@type image_dir: str
"""
created_files = []
fig0 = plt.figure()
sp1 = plt.subplot(1,1,1,axisbg='black')
end = mytime.datetime(now.year, now.month, now.day, 24/numplots, 0, 0)
begin = end - mytime.timedelta(hours=24/numplots)
base_day = begin.day
if end.hour == 0:
end_hour = 24
else:
end_hour = end.hour
plt.pcolor(x[0:-1:quality], y, z[:,0:-1:quality], shading = 'flat', vmin=0, vmax=10)
plt.colorbar()
plt.plot(x[0:-1:quality], fcbh[0:-1:quality], marker=c.FCBH_MARKER,
color=c.FCBH_COLOR, markeredgecolor=c.FCBH_ECOLOR, linewidth=0,
markersize=c.FCBH_MSIZE)
plt.plot(x[0:-1:quality], scbh[0:-1:quality], marker=c.SCBH_MARKER,
color=c.SCBH_COLOR, markeredgecolor=c.SCBH_ECOLOR, linewidth=0,
markersize=c.SCBH_MSIZE)
plt.plot(x[0:-1:quality], tcbh[0:-1:quality], marker=c.TCBH_MARKER,
color=c.TCBH_COLOR, markeredgecolor=c.TCBH_ECOLOR, linewidth=0,
markersize=c.TCBH_MSIZE)
plt.axis([begin.hour,end_hour,0,7])
title = c.BACK_TITLE.replace("'date1'", "%02d/%02d/%04d" % (now.month,now.day,now.year))
plt.title(title)
plt.ylabel(c.BACK_YLABEL)
plt.xlabel(c.BACK_XLABEL)
if not os.path.exists(image_dir): os.makedirs(image_dir)
filename = get_fn(begin, end - mytime.timedelta(seconds=1), ptype=1, tag="", site=site, description=description)
file = os.path.join(image_dir, filename)
plt.savefig(file)
_log.debug("Ceilometer Backscatter Quicklook saved as %s" % file)
created_files.append(file)
end = end + mytime.timedelta(hours=24/numplots)
begin = end - mytime.timedelta(hours=24/numplots)
while begin.day == base_day:
if end.hour == 0:
end_hour = 24
else:
end_hour = end.hour
plt.axis([begin.hour,end_hour,0,7])
filename = get_fn(begin, end - mytime.timedelta(seconds=1), ptype=1, tag="", site=site, description=description)
file = os.path.join(image_dir, filename)
plt.savefig(file)
_log.debug("Ceilometer Backscatter Quicklook saved as %s" % file)
created_files.append(file)
end = end + mytime.timedelta(hours=24/numplots)
begin = end - mytime.timedelta(hours=24/numplots)
plt.close()
return created_files
def _plot_thumbnail(plt, x, y, z, now, numplots, quality, image_dir, get_fn, site="rig", description=""):
"""Plot the smaller thumbnail quicklook images
This private function should only be run from the make_plot() function.
Creates all thumbnail images so there are C{numplots} images.
See make_plot() for details.
@param x: The time data gathered from the NetCDF file in make_plot()
@param y: The altitude data created in the _prepare_Data()
@param z: The backscatter data gathered from the NetCDF file in make_plot()
@param now: The datetime object passed from make_plot()
@param numplots: The number of plots per day passed by make_plot()
@param quality: The number to take every other data point from make_plot()
@param image_dir: The directory to put the created image into
@param tn_size: The figure size of the thumbnail image
@param title: Specify to have a title or not
@type x: 1-D Numpy Array
@type y: 1-D Numpy Array
@type z: 2-D Numpy Array
@type now: datetime object
@type numplots: int
@type quality: int
@type image_dir: str
@type tn_size: 2-element integer tuple
@type title: boolean
"""
created_files = []
end = mytime.datetime(now.year, now.month, now.day, 24/numplots, 0, 0)
begin = end - mytime.timedelta(hours=24/numplots)
base_day = begin.day
fig1 = plt.figure(2, figsize=c.TN_SIZE)
spt = plt.subplot(1,1,1,axisbg='black')
plt.subplots_adjust(left=0,right=1,bottom=0,top=1)
if end.hour == 0:
end_hour = 24
else:
end_hour = end.hour
plt.pcolor(x[0:-1:quality], y, z[:,0:-1:quality], shading='flat', vmin=0, vmax=10)
plt.axis([begin.hour,end_hour,0,7])
tl = spt.yaxis.get_ticklines()
for t in tl:
t.set_visible(False)
if not os.path.exists(image_dir): os.makedirs(image_dir)
filename = get_fn(begin, end - mytime.timedelta(seconds=1), ptype=1, tag="tn", site=site, description=description)
file = os.path.join(image_dir, filename)
plt.savefig(file)
_log.debug("Ceilometer Backscatter Thumbnail saved as %s" % file)
created_files.append(file)
end = end + mytime.timedelta(hours=24/numplots)
begin = end - mytime.timedelta(hours=24/numplots)
while begin.day == base_day:
if end.hour == 0:
end_hour = 24
else:
end_hour = end.hour
plt.axis([begin.hour,end_hour,0,7])
filename = get_fn(begin, end - mytime.timedelta(seconds=1), ptype=1, tag="tn", site=site, description=description)
file = os.path.join(image_dir, filename)
plt.savefig(file)
_log.debug("Ceilometer Backscatter Thumbnail saved as %s" % file)
created_files.append(file)
end = end + mytime.timedelta(hours=24/numplots)
begin = end - mytime.timedelta(hours=24/numplots)
plt.close()
return created_files
def main():
"""Handles command line arguments if run as a script
Run first from command-line.
No Arguments or Run as Module
=============================
Creates, by default and by running the make_plot() function, full size quicklook plot of Ceilometer data
and a thumbnail quicklook image without a title of the same plot.
Run Script
==========
Usage::
quicklook_ceilo.py [-t | -f filename | --numplots=2 | --quality=1 | --date="MM/DD/YYYY" | --blank]
Try `python quicklook_ceilo.py -h' for more information.