diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 0000000000000000000000000000000000000000..19932bff5a7864fd22ac1439a9acbcf5a4bfc429
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1 @@
+include aossceilo/ceilo.ncml
diff --git a/README.rst b/README.rst
index b39f6c5a66652e5a98b59f8edf8adeef97830db7..9273b26c7eca374dfefe74723d61d95ef5080bde 100644
--- a/README.rst
+++ b/README.rst
@@ -6,4 +6,4 @@ and higher level product generation.
NOTE: Most of the code in this repository was copied from the old subversion
repository. This code was executed on python 2.6/2.7 environments, but will
-be run from python 3.6 from now on. Some scripts may not work out of the box.
+be run from python 3.6+ from now on. Some scripts may not work out of the box.
diff --git a/aossceilo/__init__.py b/aossceilo/__init__.py
index 885b15d64004148f7c4d30cf2415e21af0a1725c..e5f64fa9976d92c1f00564fd169e7467beb0b323 100644
--- a/aossceilo/__init__.py
+++ b/aossceilo/__init__.py
@@ -1,7 +1,6 @@
__import__('pkg_resources').declare_namespace(__name__)
__docformat__ = 'Epytext'
-import os
from aossceilo import CONFIG as c
instrument_name = 'ceilo'
diff --git a/aossceilo/message.py b/aossceilo/message.py
index 17e3c6c3116da3578af47a69bbcff82c0a9f4882..9d411049b2adf6c725b04d9c02f087f948a76f3e 100644
--- a/aossceilo/message.py
+++ b/aossceilo/message.py
@@ -4,8 +4,7 @@ For processing Ceilometer CT25K Messages
import logging
from calendar import timegm
from datetime import datetime
-#from urllib2 import urlopen
-from metobs.util.gen_open import open_ascii as urlopen
+from metobscommon.util.gen_open import open_ascii as urlopen
from numpy import array
@@ -82,7 +81,7 @@ ALARM_MSGS = [
"VOLTAGE FAILURE", "RECEIVER FAILURE", "LASER FAILURE",
"LASER TEMPERATURE SHUT_OFF"]
-class MessageError(StandardError):
+class MessageError(RuntimeError):
"""General message error."""
@@ -268,7 +267,7 @@ class Message2(object):
@return: 0 if the flag is not set, otherwise the value of the flag.
"""
- return long(self.status_string, 16) & flag
+ return int(self.status_string, 16) & flag
def get_status_messages(self):
"""
@@ -278,13 +277,13 @@ class Message2(object):
"""
messages = {'alarms':[], 'warnings':[], 'informational':[]}
for idx in range(len(WARNING_FLAGS)):
- if long(self.status_string, 16) & WARNING_FLAGS[idx]:
+ if int(self.status_string, 16) & WARNING_FLAGS[idx]:
messages['warnings'].append(WARNING_MSGS[idx])
for idx in range(len(ALARM_FLAGS)):
- if long(self.status_string, 16) & ALARM_FLAGS[idx]:
+ if int(self.status_string, 16) & ALARM_FLAGS[idx]:
messages['alarms'].append(ALARM_MSGS[idx])
for idx in range(len(INFO_FLAGS)):
- if long(self.status_string, 16) & INFO_FLAGS[idx]:
+ if int(self.status_string, 16) & INFO_FLAGS[idx]:
messages['informational'].append(INFO_MSGS[idx])
return messages
@@ -362,9 +361,9 @@ def parse_sky_condition(line, feet=False):
return sky_cond
-def parse_backscatter(data):
- if isinstance(data, (str, unicode)):
+def parse_backscatter(data):
+ if isinstance(data, str):
lines = data.split('\n')
else:
lines = data
@@ -459,7 +458,7 @@ def load_messages(url, on_error=None):
stamp = datetime.fromtimestamp(int(lines[0]))
try:
messages.append(Message2([l.strip() for l in lines[1:]], stamp))
- except Exception, e:
+ except Exception as e:
if hasattr(on_error, '__call__'):
on_error(lines, e)
# disregard possible garbage between messages
diff --git a/aossceilo/nc.py b/aossceilo/nc.py
index ad149ba36a5c273402cc4e594a9b2ba70fe97aef..dc9633fdc89c8b06940d8489e30f97911e7e37fc 100644
--- a/aossceilo/nc.py
+++ b/aossceilo/nc.py
@@ -12,7 +12,7 @@ from calendar import timegm
from datetime import datetime, timedelta
import pkg_resources
-from numpy import array, zeros, int32, int8, uint16
+from numpy import array, zeros, int32
from metobscommon.util import mytime
from metobscommon.util.gen_open import open_ascii as urlopen
@@ -214,7 +214,7 @@ def make_ceilo_files(begin, basedir=None, end=None, loc=ssec_loc, site="rig", de
url = get_ascii_url(dt, site=site, description=description)
try:
urlopen(url)
- except StandardError, e:
+ except Exception as e:
_log.warn('%s: %s', e, url)
continue
_log.debug('Creating %s', fpth)
diff --git a/aossceilo/quicklook/CONFIG.py b/aossceilo/quicklook/CONFIG.py
new file mode 100644
index 0000000000000000000000000000000000000000..9ec0d09751c16300d924fd2a3415520ad3121fd4
--- /dev/null
+++ b/aossceilo/quicklook/CONFIG.py
@@ -0,0 +1,111 @@
+"""Set of variables used by metobs.tower.quicklook to manipulate data and plot
+format.
+
+Any other options not seen in this CONFIG file are probably available in the
+metobs.tower.quicklook script flags.
+"""
+
+__author__ = 'David Hoese, SSEC'
+__docformat__ = 'Epytext'
+from aossceilo import CONFIG as c
+
+"""What visual backend matplotlib should use"""
+MATPLOTLIB_BACKEND='agg'
+"""Use pycdf instead of the opendap dap module"""
+USE_PYCDF = True
+
+"""Do not change the key(first) value of the dictionary values.
+ Will be used to get the data for each NetCDF variable.
+ If a variable is not present in the NetCDF file manually specify the "ptype"
+ with the proper script flag when running image generation.
+"""
+CDF_VARS = {'time_var':'time',
+ 'backscatter_var':'backscatter',
+ 'fcbh_var':'first_cbh',
+ 'scbh_var':'second_cbh',
+ 'tcbh_var':'third_cbh'}
+
+"""Valid min and max values are used to hide bad values that were measured by
+ an instrument. If there is an attribute of the NetCDF variable that has
+ this information set the value to a string of the attributes name. Otherwise
+ specify the integer or float value boundaries for the data. Valid min and
+ max must be in the units that the data was measured in.
+"""
+# Masked where data equals BACK_VMIN
+BACK_VMIN=0 # 'valid_min'
+# BACK_VMAX='valid_max'
+FCBH_VMIN=-9999
+#FCBH_VMAX=7000
+SCBH_VMIN=-9999
+#SCBH_VMAX=7000
+TCBH_VMIN=-9999
+#TCBH_VMAX=7000
+
+
+### Data Conversions ###
+"""Time can not be converted it will be calculated using epoch time and
+ taking (time - time.min())/3600.0 to give a 0 - 24 time range.
+"""
+"""The MEAS variables represent what the data was measured in, the CONV
+ is the units to convert the data to.
+"""
+"""'m' only"""
+# ALT_MEAS='m' # Meters
+# ALT_CONV='m' # Meters
+
+
+"""Make sure to label the plots correctly in the plot settings section below"""
+
+"""Thumbnail size is in inches."""
+TN_SIZE=(1,1)
+
+### Plot Settings ###
+
+### Backscatter Settings ###
+"""The title to place at the top of the backscatter figure
+
+ If you would like to place the date in the format MM/DD/YYYY put the string
+ in single quotes 'date1'
+
+ ex. BACK_TITLE="RIG Tower Meteorogram 'date1'"
+
+ These settings will only effect the quicklook because the thumbnail has no
+ labels.
+"""
+BACK_TITLE="Ceilometer Backscatter 'date1'"
+BACK_YLABEL="Altitude (km)"
+#BACK_YLABEL_SIZE=8
+BACK_XLABEL="Time (UTC)"
+#BACK_XLABEL_SIZE=8
+
+"""Cloud base height colors and edge colors.
+"""
+FCBH_MARKER = 'o'
+SCBH_MARKER = 'o'
+TCBH_MARKER = 'o'
+FCBH_COLOR = "red"
+SCBH_COLOR = "#003300" # Green
+TCBH_COLOR = "purple"
+FCBH_ECOLOR = "white"
+SCBH_ECOLOR = "white"
+TCBH_ECOLOR = "white"
+FCBH_MSIZE = 3
+SCBH_MSIZE = 3
+TCBH_MSIZE = 3
+
+### File Settings ###
+"""GET_IMG_DIR fucntion will be used to save the images to the directory:
+ RETURNED_PATH/image.png
+ the function must take a datetime object as its only parameter
+"""
+def get_img_dir(dt):
+ """Temp. function for returning the current image/cache directory"""
+ return c.get_cache_dir("img", dt)
+GET_IMG_DIR = get_img_dir
+
+def get_img_filename(begin, end, ptype=1, tag="", site="rig", description=""):
+ """Intermediate function for returning a standard image filename"""
+ return c.get_img_filename(begin, end, ptype, tag=tag, site=site, description=description)
+GET_IMG_FILENAME = get_img_filename
+
+GET_NC_URL = c.get_nc_url
diff --git a/aossceilo/quicklook/__init__.py b/aossceilo/quicklook/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..261cddc1a14cadd1d208f39bc1f6aa7c6e4f3057
--- /dev/null
+++ b/aossceilo/quicklook/__init__.py
@@ -0,0 +1,469 @@
+"""Create plots of the ceilometer instrument data
+
+
+Functions
+=========
+
+main()
+
+make_plot()
+
+@author: 'David Hoese'
+@organization: 'SSEC'
+@contact: 'david.hoese@ssec.wisc.edu'
+@version: '1.5'
+"""
+
+import sys, os, logging
+from metobscommon.util import mytime
+import aossceilo.quicklook.CONFIG as c
+import numpy
+
+if c.USE_PYCDF:
+ UCDF=True
+ from pycdf import CDF, CDFError
+else:
+ UCDF=False
+ from dap import client as dap
+
+# Must set backend for matplotlib before importing pylab
+import matplotlib as mpl
+mpl.use(c.MATPLOTLIB_BACKEND)
+import matplotlib.pylab as plt
+
+_log = logging.getLogger(__name__)
+
+def make_plot(ncfile=None, numplots=2, quality=1, site="rig", description="", format="%m/%d/%Y",
+ datestring=None, dt=None, blank=True):
+ """Creates a plot using ceilometer data from the NetCDF file or date
+ specified
+
+ If no arguments are provided plots of the current day will be created.
+ Specifying a datetime object for C{dt} AND a filename for C{filename} will
+ results in a C{ValueError}.
+ A blank plot will only be created if a datestring or datetime object is
+ specified otherwise there is no way of finding the date of the attempted
+ plot.
+
+ Logging
+ =======
+
+ Log messages will be logged to I{stdout}.
+
+ Log messages occur for the following:
+ - Specifying a date or filename for which the file does not exist
+ - Specifying a date or filename for a non-NetCDF file
+ - Specifying a date or filename for a NetCDF file that does not have
+ the correct variables
+ - Specifying a date or filename for a NetCDF file that has corrupt
+ data
+ - Any problem while plotting the data
+
+ Plotting
+ ========
+
+ This function will create 2 large quicklook images and 2 small
+ (1 in. x 1 in. default) thumbnail images by default with a black
+ background.
+ The number of plots can be changed by specifying a value for
+ C{numplots}. For example, if C{numplots=2}, 2 plots would be created
+ per day per type. So 2 larges quicklook images, 00:00:00 to 11:59:59
+ and 12:00:00 to 23:59:59 and the same time frames for the 2 thumbnail
+ images.
+
+ First the large quicklook images are created, in order from the
+ beginning of the day to the end, number of plots depends on the value of
+ C{numplots} as described above. After all large quicklook images are
+ created the thumbnail images are created in the same manner as the large
+ quicklook images.
+
+ Image Labels:
+
+ 1. The Y-axis:
+ - Units: Kilometers
+ - Range: 0 km to 7 km
+ 2. The X-axis:
+ - Units: Hours (UTC Timezone)
+ - Range: I{Varies according to C{numplots}}
+ 3. The Colorbar:
+ - Units: I{Arbitrary}
+ - Range: 0 to 10
+ 4. Title:
+ - Value: "Ceilometer Backscatter MM/DD/YYYY"
+ - I{Note: Thumbnails have no title by default and are different
+ than the title of the large quicklook images."}
+ - Set thumbnails to have a title by setting C{title} to
+ C{True}.
+
+ I{Note: Currently this code can only generate one day of data at a time}
+
+ @param filename: The complete filename of the NetCDF file containing the
+ ceilometer backscatter data.
+
+ NetCDF Variables required:
+ - time
+ - backscatter
+ @param numplots: The number of plot segments to split one day into. The
+ default 2 produces one 0-12 plot and one 12-24 plot with 2 thumbnails of
+ the same time separation.
+ @param title: Specify whether or not to have a title at the top of the
+ thumbnail images. The defualt False produces no title.
+ @param quality: Backscatter will be used every C{quality} piece of data.
+ The default C{1} uses every piece of data.
+ @param format: To be used with datestring to specify a format that
+ datestring follows. If no datestring is provided the current date will be
+ used.
+ @param datestring: Used with format to specify the date parsed from a
+ string following the format string of format.
+ @param dt: Datetime object to be used
+ @param backend: Specify the backend for matplotlib to use in plotting.
+ The default 'agg' is used for systems without displays.
+ @param blank: Create a blank when corrupt or no data is found for a
+ C{filename}; C{datestring} or datetime object required.
+ @type filename: str
+ @type numplots: integer
+ @type title: boolean
+ @type quality: integer
+ @type format: str
+ @type datestring: str
+ @type dt: datetime
+ @type backend: str
+ @type blank: boolean
+ """
+ now = mytime.utc_now()
+
+ # Handle all of the parameters, Read Documentation for more information
+ if ncfile == None and dt == None:
+ if datestring == None:
+ datestring=now.strftime("%m/%d/%Y")
+ now = mytime.datetime.strptime(datestring, format)
+ ncfile = c.GET_NC_URL(now, site=site, description=description)
+ elif ncfile != None and dt != None:
+ raise ValueError("Specify either a ncfile or a datetime object, not both")
+ elif ncfile != None:
+ if datestring == None and blank:
+ raise ValueError("A datestring must be specified with blank or no blank plot can be created")
+ elif datestring != None:
+ now = mytime.datetime.strptime(datestring, format)
+ elif dt != None:
+ ncfile = c.GET_NC_URL(dt, site=site, description=description)
+ now = dt
+ datestring = now.strftime("%m/%d/%Y")
+ if numplots <= 0:
+ raise ValueError("numplots must be greater than zero")
+
+ d = []
+ try:
+ if UCDF:
+ d.append( CDF(ncfile) )
+ x = d[0].var(c.CDF_VARS['time_var']).get()
+ z = d[0].var(c.CDF_VARS['backscatter_var']).get()
+ fcbh = d[0].var(c.CDF_VARS['fcbh_var']).get()
+ scbh = d[0].var(c.CDF_VARS['scbh_var']).get()
+ tcbh = d[0].var(c.CDF_VARS['tcbh_var']).get()
+ else:
+ d.append( dap.open(ncfile) )
+ x = d[0][c.CDF_VARS['time_var']][::]
+ z = d[0][c.CDF_VARS['backscatter_var']][::]
+ fcbh = d[0][c.CDF_VARS['fcbh_var']][::]
+ scbh = d[0][c.CDF_VARS['scbh_var']][::]
+ tcbh = d[0][c.CDF_VARS['tcbh_var']][::]
+ x,y,z,fcbh,scbh,tcbh = _prepare_data(x, z, fcbh, scbh, tcbh)
+ except CDFError as e:
+ _log.exception("File Not Found or Not a properly formatted NetCDF file on %s" % ncfile)
+ if blank:
+ x = y = fcbh = scbh = tcbh = numpy.zeros((100))
+ z = numpy.zeros((100,100))
+ _log.warning("Blank plot will be created for %s" % ncfile)
+ else: raise e
+ finally:
+ if UCDF:
+ if d: [ cdf.close() for cdf in d ]
+ else: pass
+
+ created_files = []
+ try:
+ created_files = created_files + _plot_quicklook(plt, x, y, z, fcbh, scbh, tcbh, now, numplots, quality, c.GET_IMG_DIR(now), c.GET_IMG_FILENAME, site=site, description=description)
+ created_files = created_files + _plot_thumbnail(plt, x, y, z, now, numplots, quality, c.GET_IMG_DIR(now), c.GET_IMG_FILENAME, site=site, description=description)
+ except Exception as e:
+ _log.exception("Error in matplotlib or pylab during plotting")
+ raise e
+ finally:
+ # Just in case a figure was not closed
+ plt.close('all')
+ return created_files
+
+def _prepare_data(x, z, fcbh, scbh, tcbh):
+ """Prepare the x, y, and z data to be used in plotting
+
+ This private function should only be used by the make_plot() function.
+ This prepares data by doing the following:
+
+ 1. X Data - Converted to a 0-24 scale
+ 2. Y Data - A array of heights from 0 - 7 kilometers
+ 3. Z Data - Masks data less than or equal to 0 and then finds the
+ absolute value of the log of the values.
+
+ @param x: The time data retrieved from the NetCDF file in make_plot()
+ @param z: The backscatter data retrieved from the NetCDF file in make_plot()
+ @type x: 1-D Numpy Array
+ @type z: 2-D Numpy Array
+ @return : x, y, z
+ """
+ y = numpy.arange(0, z[0,:].size*30, 30)/1000.0
+ z = z.transpose()
+ x = x/3600.0
+
+ zshape = numpy.shape(z)
+ z = numpy.concatenate(z)
+ indp = numpy.nonzero(numpy.greater(z,c.BACK_VMIN))[0]
+ z[indp] = abs(numpy.log(z[indp]))
+ z = numpy.reshape(z,zshape)
+ z = numpy.ma.masked_where(z <= c.BACK_VMIN, z)
+ fcbh = numpy.ma.masked_where(fcbh == c.FCBH_VMIN, fcbh)
+ fcbh = fcbh/1000.0
+ scbh = numpy.ma.masked_where(scbh == c.SCBH_VMIN, scbh)
+ scbh = scbh/1000.0
+ tcbh = numpy.ma.masked_where(tcbh == c.TCBH_VMIN, tcbh)
+ tcbh = tcbh/1000.0
+
+ return x,y,z,fcbh,scbh,tcbh
+
+def _plot_quicklook(plt, x, y, z, fcbh, scbh, tcbh, now, numplots, quality, image_dir, get_fn, site="rig", description=""):
+ """Plot the large quicklook image
+
+ This private function should only be run from the make_plot() function.
+ Creates all large quicklook images so there are C{numplots} images.
+ See make_plot() for details.
+
+ @param x: The time data gathered from the NetCDF file in make_plot()
+ @param y: The altitude data created in the _prepare_Data()
+ @param z: The backscatter data gathered from the NetCDF file in make_plot()
+ @param now: The datetime object passed from make_plot()
+ @param numplots: The number of plots per day passed from make_plot()
+ @param quality: The number to take every other data point from make_plot()
+ @param image_dir: The directory to put the created image into.
+ @type x: 1-D Numpy Array
+ @type y: 1-D Numpy Array
+ @type z: 2-D Numpy Array
+ @type now: datetime object
+ @type numplots: int
+ @type quality: int
+ @type image_dir: str
+ """
+ created_files = []
+ fig0 = plt.figure()
+ sp1 = plt.subplot(1,1,1,axisbg='black')
+ end = mytime.datetime(now.year, now.month, now.day, 24/numplots, 0, 0)
+ begin = end - mytime.timedelta(hours=24/numplots)
+ base_day = begin.day
+ if end.hour == 0:
+ end_hour = 24
+ else:
+ end_hour = end.hour
+
+ plt.pcolor(x[0:-1:quality], y, z[:,0:-1:quality], shading = 'flat', vmin=0, vmax=10)
+ plt.colorbar()
+ plt.plot(x[0:-1:quality], fcbh[0:-1:quality], marker=c.FCBH_MARKER,
+ color=c.FCBH_COLOR, markeredgecolor=c.FCBH_ECOLOR, linewidth=0,
+ markersize=c.FCBH_MSIZE)
+ plt.plot(x[0:-1:quality], scbh[0:-1:quality], marker=c.SCBH_MARKER,
+ color=c.SCBH_COLOR, markeredgecolor=c.SCBH_ECOLOR, linewidth=0,
+ markersize=c.SCBH_MSIZE)
+ plt.plot(x[0:-1:quality], tcbh[0:-1:quality], marker=c.TCBH_MARKER,
+ color=c.TCBH_COLOR, markeredgecolor=c.TCBH_ECOLOR, linewidth=0,
+ markersize=c.TCBH_MSIZE)
+ plt.axis([begin.hour,end_hour,0,7])
+ title = c.BACK_TITLE.replace("'date1'", "%02d/%02d/%04d" % (now.month,now.day,now.year))
+ plt.title(title)
+ plt.ylabel(c.BACK_YLABEL)
+ plt.xlabel(c.BACK_XLABEL)
+
+ if not os.path.exists(image_dir): os.makedirs(image_dir)
+
+ filename = get_fn(begin, end - mytime.timedelta(seconds=1), ptype=1, tag="", site=site, description=description)
+ file = os.path.join(image_dir, filename)
+ plt.savefig(file)
+ _log.debug("Ceilometer Backscatter Quicklook saved as %s" % file)
+ created_files.append(file)
+
+ end = end + mytime.timedelta(hours=24/numplots)
+ begin = end - mytime.timedelta(hours=24/numplots)
+ while begin.day == base_day:
+ if end.hour == 0:
+ end_hour = 24
+ else:
+ end_hour = end.hour
+ plt.axis([begin.hour,end_hour,0,7])
+ filename = get_fn(begin, end - mytime.timedelta(seconds=1), ptype=1, tag="", site=site, description=description)
+ file = os.path.join(image_dir, filename)
+ plt.savefig(file)
+ _log.debug("Ceilometer Backscatter Quicklook saved as %s" % file)
+ created_files.append(file)
+
+ end = end + mytime.timedelta(hours=24/numplots)
+ begin = end - mytime.timedelta(hours=24/numplots)
+ plt.close()
+ return created_files
+
+def _plot_thumbnail(plt, x, y, z, now, numplots, quality, image_dir, get_fn, site="rig", description=""):
+ """Plot the smaller thumbnail quicklook images
+
+ This private function should only be run from the make_plot() function.
+ Creates all thumbnail images so there are C{numplots} images.
+ See make_plot() for details.
+
+ @param x: The time data gathered from the NetCDF file in make_plot()
+ @param y: The altitude data created in the _prepare_Data()
+ @param z: The backscatter data gathered from the NetCDF file in make_plot()
+ @param now: The datetime object passed from make_plot()
+ @param numplots: The number of plots per day passed by make_plot()
+ @param quality: The number to take every other data point from make_plot()
+ @param image_dir: The directory to put the created image into
+ @param tn_size: The figure size of the thumbnail image
+ @param title: Specify to have a title or not
+ @type x: 1-D Numpy Array
+ @type y: 1-D Numpy Array
+ @type z: 2-D Numpy Array
+ @type now: datetime object
+ @type numplots: int
+ @type quality: int
+ @type image_dir: str
+ @type tn_size: 2-element integer tuple
+ @type title: boolean
+ """
+ created_files = []
+ end = mytime.datetime(now.year, now.month, now.day, 24/numplots, 0, 0)
+ begin = end - mytime.timedelta(hours=24/numplots)
+ base_day = begin.day
+
+ fig1 = plt.figure(2, figsize=c.TN_SIZE)
+ spt = plt.subplot(1,1,1,axisbg='black')
+ plt.subplots_adjust(left=0,right=1,bottom=0,top=1)
+ if end.hour == 0:
+ end_hour = 24
+ else:
+ end_hour = end.hour
+
+ plt.pcolor(x[0:-1:quality], y, z[:,0:-1:quality], shading='flat', vmin=0, vmax=10)
+ plt.axis([begin.hour,end_hour,0,7])
+ tl = spt.yaxis.get_ticklines()
+ for t in tl:
+ t.set_visible(False)
+
+ if not os.path.exists(image_dir): os.makedirs(image_dir)
+
+ filename = get_fn(begin, end - mytime.timedelta(seconds=1), ptype=1, tag="tn", site=site, description=description)
+ file = os.path.join(image_dir, filename)
+ plt.savefig(file)
+ _log.debug("Ceilometer Backscatter Thumbnail saved as %s" % file)
+ created_files.append(file)
+
+ end = end + mytime.timedelta(hours=24/numplots)
+ begin = end - mytime.timedelta(hours=24/numplots)
+ while begin.day == base_day:
+ if end.hour == 0:
+ end_hour = 24
+ else:
+ end_hour = end.hour
+ plt.axis([begin.hour,end_hour,0,7])
+ filename = get_fn(begin, end - mytime.timedelta(seconds=1), ptype=1, tag="tn", site=site, description=description)
+ file = os.path.join(image_dir, filename)
+ plt.savefig(file)
+ _log.debug("Ceilometer Backscatter Thumbnail saved as %s" % file)
+ created_files.append(file)
+
+ end = end + mytime.timedelta(hours=24/numplots)
+ begin = end - mytime.timedelta(hours=24/numplots)
+ plt.close()
+ return created_files
+
+def main():
+ """Handles command line arguments if run as a script
+
+ Run first from command-line.
+
+ No Arguments or Run as Module
+ =============================
+ Creates, by default and by running the make_plot() function, full size quicklook plot of Ceilometer data
+ and a thumbnail quicklook image without a title of the same plot.
+
+
+ Run Script
+ ==========
+ Usage::
+ quicklook_ceilo.py [-t | -f filename | --numplots=2 | --quality=1 | --date="MM/DD/YYYY" | --blank]
+ Try `python quicklook_ceilo.py -h' for more information.
+ Example: "python quicklook_ceilo.py -tf $HOME/my_ceilo_data.nc"
+
+ """
+
+ import optparse
+ from metobscommon.util import file
+ now = mytime.utc_now()
+
+ # Parse Arguments
+ usage="""ceilometer.py [-t | -f filename | --numplots=2 | --quality=1 | --date="MM/DD/YYYY" | --blank]
+ Try `python quicklook_ceilo.py -h' for more information.
+ Creates a full size quicklook plot of Ceilometer data and a thumbnail image of the same plot
+ Example: "python quicklook_ceilo.py -tf "$HOME/my_ceilo_data.nc"
+
+ Created by:
+ David Hoese
+ SSEC
+ Madison, WI 53706
+ """
+
+ parser = optparse.OptionParser(usage)
+ parser.add_option("-f", "--file", dest="filename",
+ default=None,
+ help="specify NetCDF file to plot")
+ parser.add_option("--numplots", dest="numplots",
+ default="2",
+ help="specify the number of the plots per day")
+ parser.add_option("--quality", dest="quality",
+ default="1",
+ help="specify the number of data points to skip by")
+ parser.add_option("--date", dest="datetoplot",
+ default=None,
+ help="specify the date of the data to be plotted")
+ parser.add_option("--blank", action="store_false", dest="blank", default=True,
+ help="create a blank plot if data file is not found")
+ parser.add_option("-l", dest="lockfile", default="ceilo_quicklook.lock",
+ help="specify the lockfile to use")
+ parser.add_option("--site", dest="site", default="rig",
+ help="the site name used in filenames")
+ parser.add_option("--description", dest="description", default="",
+ help="the description used in filenames")
+ (options, args) = parser.parse_args()
+
+ filename = options.filename
+ numplots = int(options.numplots)
+ quality = int(options.quality)
+ datetoplot = options.datetoplot
+ site = options.site
+ description = options.description
+
+ # If a filename is specified then a datetoplot string must also be specified
+ if filename != None and datetoplot == None:
+ _log.error("Filename must also be specified with a datetoplot")
+ sys.exit()
+
+ # Set up datetoplot string if there was nothing specified to plot (ie. Current Day)
+ if datetoplot == None and filename == None:
+ datetoplot = now.strftime("%m/%d/%Y")
+
+ try:
+ lock = file.FileLock(options.lockfile)
+ # Run the make_plot function to create the images with options and arguments specified at command-line
+ make_plot(filename, numplots, quality, site=site, description=description,
+ format="%m/%d/%Y", datestring=datetoplot, blank=options.blank)
+ except file.FileLockedError as e:
+ _log.warn("function already locked with %s", e.filename)
+ finally:
+ _log.warn("terminating", exc_info=1)
+ logging.shutdown()
+
+if __name__ == '__main__':
+ logging.basicConfig(level=logging.INFO)
+ sys.exit(main())
diff --git a/aossceilo/tidy.py b/aossceilo/tidy.py
index 71d75af047ae51d794494db5c88eca5f0dffdad0..2aecdaafccf27139e1a616b05c93b717e88b8ebc 100644
--- a/aossceilo/tidy.py
+++ b/aossceilo/tidy.py
@@ -21,11 +21,10 @@ Created by davidh on 2009-10-08.
Copyright (c) 2009 University of Wisconsin SSEC. All rights reserved.
"""
-import os, sys, logging, shutil
-from subprocess import check_call as sh
+import os, sys, logging
from datetime import datetime, timedelta
-from metobs.ceilo import quicklook, nc
-from metobs.ceilo import CONFIG as c
+from aossceilo import quicklook, nc
+from aossceilo import CONFIG as c
from metobscommon.util import raw_manager
rel = raw_manager.relativedir
import re
@@ -244,9 +243,9 @@ run "%prog help" to list commands
# print first line of docstring
for cmd in commands:
ds = commands[cmd].__doc__.split('\n')[0]
- print "%-16s %s" % (cmd,ds)
+ print("%-16s %s" % (cmd,ds))
else:
- print commands[command].__doc__
+ print(commands[command].__doc__)
def test():
"run tests"
diff --git a/environment.yaml b/environment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..9751c5f2932d9999e6cefee11138a78b96d663e9
--- /dev/null
+++ b/environment.yaml
@@ -0,0 +1,8 @@
+name: aoss-cameras
+channels:
+ - conda-forge
+dependencies:
+ - numpy
+ - pytz
+ - sh
+ - pyserial
diff --git a/scripts/archive_ceilo.sh b/scripts/archive_ceilo.sh
new file mode 100755
index 0000000000000000000000000000000000000000..c2e7bd2456410338d12cbb12e2998bbc9711c40d
--- /dev/null
+++ b/scripts/archive_ceilo.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Moves ceilometer data in raw (inst-data) and cache
+
+ENV=/opt/metobs/aoss_ceilo
+LOGDIR=$ENV/log
+LOCK=$ENV/lock/$(basename $0).lock
+
+source $ENV/scripts/metobs_config.sh
+
+(
+ flock -x -n 200 || exit $?
+ logfile=${LOGDIR}/ceilo-tidy-archive.log
+
+ # Run today
+ $ENV/bin/python -m aossceilo.tidy --debug incoming >> $logfile 2>&1
+
+) 200>$LOCK
+
diff --git a/scripts/do_ceilo_image_gen.sh b/scripts/do_ceilo_image_gen.sh
new file mode 100755
index 0000000000000000000000000000000000000000..e734817c4dd0c3efd4fec307690fae7c3a83d90c
--- /dev/null
+++ b/scripts/do_ceilo_image_gen.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Create ceilo images by running metobs.ceilo.tidy quicklook
+
+echo "NOT UPDATED FOR PYTHON 3"
+exit 1
+
+ENV=/data1/software/aoss-ceilo
+LOGDIR=$ENV/logs
+LOCK=$ENV/locks/$(basename $0).lock
+
+source $ENV/scripts/metobs_config.sh
+
+(
+ flock -x -n 200 || exit $?
+ logfile=${LOGDIR}/ceilo-tidy-quicklook.log
+
+ # Run Yesterday's Image Generation if the current time is 12:10 AM.
+ if [ `date +%H:%M` == "00:10" ]; then
+ $ENV/bin/python -m metobs.ceilo.tidy --debug --date=`date +%m/%d/%Y --date="yesterday"` quicklook >> $logfile 2>&1
+ fi
+
+ # Run metobs.ceilo.tidy quicklook
+ $ENV/bin/python -m metobs.ceilo.tidy --debug quicklook >> $logfile 2>&1
+
+) 200>$LOCK
+
diff --git a/scripts/do_ceilo_nc_gen.sh b/scripts/do_ceilo_nc_gen.sh
new file mode 100755
index 0000000000000000000000000000000000000000..88350560aab3bb36dd7cfd0e2d3995863c8364f5
--- /dev/null
+++ b/scripts/do_ceilo_nc_gen.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Generates NetCDF files of RIG Ceilometer data and puts them in the cache directory.
+
+echo "NOT UPDATED FOR PYTHON 3"
+exit 1
+
+ENV=/data1/software/aoss-ceilo
+LOGDIR=$ENV/logs
+LOCK=$ENV/locks/$(basename $0).lock
+
+source $ENV/scripts/metobs_config.sh
+
+(
+ flock -x -n 200 || exit $?
+ logfile=${LOGDIR}/ceilo-tidy-nc.log
+
+ # Runs metobs.ceilo.tidy nc
+ $ENV/bin/python -m metobs.ceilo.tidy --debug nc >> $logfile 2>&1
+
+) 200>$LOCK
+
diff --git a/scripts/metobs_config.sh b/scripts/metobs_config.sh
new file mode 100644
index 0000000000000000000000000000000000000000..0e4c6069cc3bd98b5546dc87137c411ccf2dfb92
--- /dev/null
+++ b/scripts/metobs_config.sh
@@ -0,0 +1,17 @@
+
+#
+# These values should override values from the python module metobs.ceilo.CONFIG
+#
+# where the ceilo ascii files get pushed to
+export CEILO_INCOMING_DIR=/data1/incoming/aoss/ceilo
+# The archived location
+export CEILO_PRAW_DIR=/data1/raw/aoss/ceilo
+# Where generated products go
+export CEILO_CACHE_DIR=/data1/cache/aoss/ceilo
+# Where the links for the latest images go
+export CEILO_LATEST_DIR=/data1/cache/aoss/ceilo
+# Where ASCII files are located
+export CEILO_ASCII_LOC=$CEILO_CACHE_DIR
+# Where NetCDF files are located
+export CEILO_NC_LOC=$CEILO_CACHE_DIR
+