Add old quicklook scripts and do further python 3 porting

MANIFEST.in

+include aossceilo/ceilo.ncml

README.rst

@@ -6,4 +6,4 @@ and higher level product generation.
 
 NOTE: Most of the code in this repository was copied from the old subversion
 repository. This code was executed on python 2.6/2.7 environments, but will
-be run from python 3.6 from now on. Some scripts may not work out of the box.
+be run from python 3.6+ from now on. Some scripts may not work out of the box.

aossceilo/__init__.py

 __import__('pkg_resources').declare_namespace(__name__)
 __docformat__ = 'Epytext'
 
-import os
 from aossceilo import CONFIG as c
 
 instrument_name = 'ceilo'

aossceilo/message.py

@@ -4,8 +4,7 @@ For processing Ceilometer CT25K Messages
 import logging
 from calendar import timegm
 from datetime import datetime
-#from urllib2 import urlopen
-from metobs.util.gen_open import open_ascii as urlopen
+from metobscommon.util.gen_open import open_ascii as urlopen
 
 from numpy import array
 
@@ -82,7 +81,7 @@ ALARM_MSGS = [
         "VOLTAGE FAILURE", "RECEIVER FAILURE", "LASER FAILURE",
         "LASER TEMPERATURE SHUT_OFF"]
 
-class MessageError(StandardError):
+class MessageError(RuntimeError):
     """General message error."""
 
 
@@ -268,7 +267,7 @@ class Message2(object):
 
         @return: 0 if the flag is not set, otherwise the value of the flag.
         """
-        return long(self.status_string, 16) & flag
+        return int(self.status_string, 16) & flag
     
     def get_status_messages(self):
         """
@@ -278,13 +277,13 @@ class Message2(object):
         """
         messages = {'alarms':[], 'warnings':[], 'informational':[]}
         for idx in range(len(WARNING_FLAGS)):
-            if long(self.status_string, 16) & WARNING_FLAGS[idx]:
+            if int(self.status_string, 16) & WARNING_FLAGS[idx]:
                 messages['warnings'].append(WARNING_MSGS[idx])
         for idx in range(len(ALARM_FLAGS)):
-            if long(self.status_string, 16) & ALARM_FLAGS[idx]:
+            if int(self.status_string, 16) & ALARM_FLAGS[idx]:
                 messages['alarms'].append(ALARM_MSGS[idx])
         for idx in range(len(INFO_FLAGS)):
-            if long(self.status_string, 16) & INFO_FLAGS[idx]:
+            if int(self.status_string, 16) & INFO_FLAGS[idx]:
                 messages['informational'].append(INFO_MSGS[idx])
         return messages
     
@@ -362,9 +361,9 @@ def parse_sky_condition(line, feet=False):
 
     return sky_cond
 
-def parse_backscatter(data):
 
-    if isinstance(data, (str, unicode)):
+def parse_backscatter(data):
+    if isinstance(data, str):
         lines = data.split('\n')
     else:
         lines = data
@@ -459,7 +458,7 @@ def load_messages(url, on_error=None):
         stamp = datetime.fromtimestamp(int(lines[0]))
         try:
             messages.append(Message2([l.strip() for l in lines[1:]], stamp))
-        except Exception, e:
+        except Exception as e:
             if hasattr(on_error, '__call__'):
                 on_error(lines, e)
         # disregard possible garbage between messages

aossceilo/nc.py

@@ -12,7 +12,7 @@ from calendar import timegm
 from datetime import datetime, timedelta
 
 import pkg_resources
-from numpy import array, zeros, int32, int8, uint16
+from numpy import array, zeros, int32
 
 from metobscommon.util import mytime
 from metobscommon.util.gen_open import open_ascii as urlopen
@@ -214,7 +214,7 @@ def make_ceilo_files(begin, basedir=None, end=None, loc=ssec_loc, site="rig", de
         url = get_ascii_url(dt, site=site, description=description)
         try:
             urlopen(url)
-        except StandardError, e:
+        except Exception as e:
             _log.warn('%s: %s', e, url)
             continue
         _log.debug('Creating %s', fpth)

aossceilo/quicklook/CONFIG.py

+"""Set of variables used by metobs.tower.quicklook to manipulate data and plot
+format.
+
+Any other options not seen in this CONFIG file are probably available in the
+metobs.tower.quicklook script flags.
+"""
+
+__author__ = 'David Hoese, SSEC'
+__docformat__ = 'Epytext'
+from aossceilo import CONFIG as c
+
+"""What visual backend matplotlib should use"""
+MATPLOTLIB_BACKEND='agg'
+"""Use pycdf instead of the opendap dap module"""
+USE_PYCDF = True
+
+"""Do not change the key(first) value of the dictionary values.
+ Will be used to get the data for each NetCDF variable.
+ If a variable is not present in the NetCDF file manually specify the "ptype"
+ with the proper script flag when running image generation.
+"""
+CDF_VARS = {'time_var':'time',
+            'backscatter_var':'backscatter',
+            'fcbh_var':'first_cbh',
+            'scbh_var':'second_cbh',
+            'tcbh_var':'third_cbh'}
+
+"""Valid min and max values are used to hide bad values that were measured by
+    an instrument.  If there is an attribute of the NetCDF variable that has
+    this information set the value to a string of the attributes name. Otherwise
+    specify the integer or float value boundaries for the data. Valid min and
+    max must be in the units that the data was measured in.
+"""
+# Masked where data equals BACK_VMIN
+BACK_VMIN=0 # 'valid_min'
+# BACK_VMAX='valid_max'
+FCBH_VMIN=-9999
+#FCBH_VMAX=7000
+SCBH_VMIN=-9999
+#SCBH_VMAX=7000
+TCBH_VMIN=-9999
+#TCBH_VMAX=7000
+
+
+### Data Conversions ###
+"""Time can not be converted it will be calculated using epoch time and
+    taking (time - time.min())/3600.0 to give a 0 - 24 time range.
+"""
+"""The MEAS variables represent what the data was measured in, the CONV
+    is the units to convert the data to.
+"""
+"""'m' only"""
+# ALT_MEAS='m' # Meters
+# ALT_CONV='m' # Meters
+
+
+"""Make sure to label the plots correctly in the plot settings section below"""
+
+"""Thumbnail size is in inches."""
+TN_SIZE=(1,1)
+
+### Plot Settings ###
+
+### Backscatter Settings ###
+"""The title to place at the top of the backscatter figure
+
+    If you would like to place the date in the format MM/DD/YYYY put the string
+    in single quotes 'date1'
+
+    ex. BACK_TITLE="RIG Tower Meteorogram 'date1'"
+
+    These settings will only effect the quicklook because the thumbnail has no
+    labels.
+"""
+BACK_TITLE="Ceilometer Backscatter 'date1'"
+BACK_YLABEL="Altitude (km)"
+#BACK_YLABEL_SIZE=8
+BACK_XLABEL="Time (UTC)"
+#BACK_XLABEL_SIZE=8
+
+"""Cloud base height colors and edge colors.
+"""
+FCBH_MARKER = 'o'
+SCBH_MARKER = 'o'
+TCBH_MARKER = 'o'
+FCBH_COLOR =  "red"
+SCBH_COLOR =  "#003300" # Green
+TCBH_COLOR =  "purple"
+FCBH_ECOLOR = "white"
+SCBH_ECOLOR = "white"
+TCBH_ECOLOR = "white"
+FCBH_MSIZE = 3
+SCBH_MSIZE = 3
+TCBH_MSIZE = 3
+
+### File Settings ###
+"""GET_IMG_DIR fucntion will be used to save the images to the directory:
+    RETURNED_PATH/image.png
+    the function must take a datetime object as its only parameter
+"""
+def get_img_dir(dt):
+    """Temp. function for returning the current image/cache directory"""
+    return c.get_cache_dir("img", dt)
+GET_IMG_DIR = get_img_dir
+
+def get_img_filename(begin, end, ptype=1, tag="", site="rig", description=""):
+    """Intermediate function for returning a standard image filename"""
+    return c.get_img_filename(begin, end, ptype, tag=tag, site=site, description=description)
+GET_IMG_FILENAME = get_img_filename
+
+GET_NC_URL = c.get_nc_url

aossceilo/quicklook/__init__.py

+"""Create plots of the ceilometer instrument data
+
+
+Functions
+=========
+
+main()
+
+make_plot()
+
+@author: 'David Hoese'
+@organization: 'SSEC'
+@contact: 'david.hoese@ssec.wisc.edu'
+@version: '1.5'
+"""
+
+import sys, os, logging
+from metobscommon.util import mytime
+import aossceilo.quicklook.CONFIG as c
+import numpy
+
+if c.USE_PYCDF:
+    UCDF=True
+    from pycdf import CDF, CDFError
+else:
+    UCDF=False
+    from dap import client as dap
+
+# Must set backend for matplotlib before importing pylab
+import matplotlib as mpl
+mpl.use(c.MATPLOTLIB_BACKEND)
+import matplotlib.pylab as plt
+
+_log = logging.getLogger(__name__)
+
+def make_plot(ncfile=None, numplots=2, quality=1, site="rig", description="", format="%m/%d/%Y",
+    datestring=None, dt=None, blank=True):
+    """Creates a plot using ceilometer data from the NetCDF file or date
+    specified
+
+    If no arguments are provided plots of the current day will be created.
+    Specifying a datetime object for C{dt} AND a filename for C{filename} will
+    results in a C{ValueError}.
+    A blank plot will only be created if a datestring or datetime object is
+    specified otherwise there is no way of finding the date of the attempted
+    plot.
+
+    Logging
+    =======
+
+        Log messages will be logged to I{stdout}.
+
+        Log messages occur for the following:
+            - Specifying a date or filename for which the file does not exist
+            - Specifying a date or filename for a non-NetCDF file
+            - Specifying a date or filename for a NetCDF file that does not have
+            the correct variables
+            - Specifying a date or filename for a NetCDF file that has corrupt
+            data
+            - Any problem while plotting the data
+
+    Plotting
+    ========
+
+        This function will create 2 large quicklook images and 2 small
+        (1 in. x 1 in. default) thumbnail images by default with a black
+        background.
+        The number of plots can be changed by specifying a value for
+        C{numplots}.  For example, if C{numplots=2}, 2 plots would be created
+        per day per type.  So 2 larges quicklook images, 00:00:00 to 11:59:59
+        and 12:00:00 to 23:59:59 and the same time frames for the 2 thumbnail
+        images.
+
+        First the large quicklook images are created, in order from the
+        beginning of the day to the end, number of plots depends on the value of
+        C{numplots} as described above.  After all large quicklook images are
+        created the thumbnail images are created in the same manner as the large
+        quicklook images.
+
+        Image Labels:
+
+            1. The Y-axis:
+                - Units: Kilometers
+                - Range: 0 km to 7 km
+            2. The X-axis:
+                - Units: Hours (UTC Timezone)
+                - Range: I{Varies according to C{numplots}}
+            3. The Colorbar:
+                - Units: I{Arbitrary}
+                - Range: 0 to 10
+            4. Title:
+                - Value: "Ceilometer Backscatter MM/DD/YYYY"
+                - I{Note: Thumbnails have no title by default and are different
+                than the title of the large quicklook images."}
+                    - Set thumbnails to have a title by setting C{title} to
+                    C{True}.
+
+        I{Note: Currently this code can only generate one day of data at a time}
+
+    @param filename: The complete filename of the NetCDF file containing the
+    ceilometer backscatter data.
+
+        NetCDF Variables required:
+            - time
+            - backscatter
+    @param numplots: The number of plot segments to split one day into. The
+    default 2 produces one 0-12 plot and one 12-24 plot with 2 thumbnails of
+    the same time separation.
+    @param title: Specify whether or not to have a title at the top of the
+    thumbnail images. The defualt False produces no title.
+    @param quality: Backscatter will be used every C{quality} piece of data.
+    The default C{1} uses every piece of data.
+    @param format: To be used with datestring to specify a format that
+    datestring follows. If no datestring is provided the current date will be
+    used.
+    @param datestring: Used with format to specify the date parsed from a
+    string following the format string of format.
+    @param dt: Datetime object to be used
+    @param backend: Specify the backend for matplotlib to use in plotting.
+    The default 'agg' is used for systems without displays.
+    @param blank: Create a blank when corrupt or no data is found for a
+    C{filename}; C{datestring} or datetime object required.
+    @type filename: str
+    @type numplots: integer
+    @type title: boolean
+    @type quality: integer
+    @type format: str
+    @type datestring: str
+    @type dt: datetime
+    @type backend: str
+    @type blank: boolean
+    """
+    now = mytime.utc_now()
+
+    # Handle all of the parameters, Read Documentation for more information
+    if ncfile == None and dt == None:
+        if datestring == None:
+            datestring=now.strftime("%m/%d/%Y")
+        now = mytime.datetime.strptime(datestring, format)
+        ncfile = c.GET_NC_URL(now, site=site, description=description)
+    elif ncfile != None and dt != None:
+        raise ValueError("Specify either a ncfile or a datetime object, not both")
+    elif ncfile != None:
+        if datestring == None and blank:
+            raise ValueError("A datestring must be specified with blank or no blank plot can be created")
+        elif datestring != None:
+                now = mytime.datetime.strptime(datestring, format)
+    elif dt != None:
+        ncfile = c.GET_NC_URL(dt, site=site, description=description)
+        now = dt
+        datestring = now.strftime("%m/%d/%Y")
+    if numplots <= 0:
+        raise ValueError("numplots must be greater than zero")
+
+    d = []
+    try:
+        if UCDF:
+            d.append( CDF(ncfile) )
+            x =    d[0].var(c.CDF_VARS['time_var']).get()
+            z =    d[0].var(c.CDF_VARS['backscatter_var']).get()
+            fcbh = d[0].var(c.CDF_VARS['fcbh_var']).get()
+            scbh = d[0].var(c.CDF_VARS['scbh_var']).get()
+            tcbh = d[0].var(c.CDF_VARS['tcbh_var']).get()
+        else:
+            d.append( dap.open(ncfile) )
+            x =    d[0][c.CDF_VARS['time_var']][::]
+            z =    d[0][c.CDF_VARS['backscatter_var']][::]
+            fcbh = d[0][c.CDF_VARS['fcbh_var']][::]
+            scbh = d[0][c.CDF_VARS['scbh_var']][::]
+            tcbh = d[0][c.CDF_VARS['tcbh_var']][::]
+        x,y,z,fcbh,scbh,tcbh = _prepare_data(x, z, fcbh, scbh, tcbh)
+    except CDFError as e:
+        _log.exception("File Not Found or Not a properly formatted NetCDF file on %s" % ncfile)
+        if blank:
+            x = y = fcbh = scbh = tcbh = numpy.zeros((100))
+            z = numpy.zeros((100,100))
+            _log.warning("Blank plot will be created for %s" % ncfile)
+        else: raise e
+    finally:
+        if UCDF:
+            if d: [ cdf.close() for cdf in d ]
+            else: pass
+
+    created_files = []
+    try:
+        created_files = created_files + _plot_quicklook(plt, x, y, z, fcbh, scbh, tcbh, now, numplots, quality, c.GET_IMG_DIR(now), c.GET_IMG_FILENAME, site=site, description=description)
+        created_files = created_files + _plot_thumbnail(plt, x, y, z, now, numplots, quality, c.GET_IMG_DIR(now), c.GET_IMG_FILENAME, site=site, description=description)
+    except Exception as e:
+        _log.exception("Error in matplotlib or pylab during plotting")
+        raise e
+    finally:
+        # Just in case a figure was not closed
+        plt.close('all')
+    return created_files
+
+def _prepare_data(x, z, fcbh, scbh, tcbh):
+    """Prepare the x, y, and z data to be used in plotting
+
+    This private function should only be used by the make_plot() function.
+    This prepares data by doing the following:
+
+        1. X Data - Converted to a 0-24 scale
+        2. Y Data - A array of heights from 0 - 7 kilometers
+        3. Z Data - Masks data less than or equal to 0 and then finds the
+        absolute value of the log of the values.
+
+    @param x: The time data retrieved from the NetCDF file in make_plot()
+    @param z: The backscatter data retrieved from the NetCDF file in make_plot()
+    @type x: 1-D Numpy Array
+    @type z: 2-D Numpy Array
+    @return : x, y, z
+    """
+    y = numpy.arange(0, z[0,:].size*30, 30)/1000.0
+    z = z.transpose()
+    x = x/3600.0
+
+    zshape = numpy.shape(z)
+    z = numpy.concatenate(z)
+    indp = numpy.nonzero(numpy.greater(z,c.BACK_VMIN))[0]
+    z[indp] = abs(numpy.log(z[indp]))
+    z = numpy.reshape(z,zshape)
+    z = numpy.ma.masked_where(z <= c.BACK_VMIN, z)
+    fcbh = numpy.ma.masked_where(fcbh == c.FCBH_VMIN, fcbh)
+    fcbh = fcbh/1000.0
+    scbh = numpy.ma.masked_where(scbh == c.SCBH_VMIN, scbh)
+    scbh = scbh/1000.0
+    tcbh = numpy.ma.masked_where(tcbh == c.TCBH_VMIN, tcbh)
+    tcbh = tcbh/1000.0
+
+    return x,y,z,fcbh,scbh,tcbh
+
+def _plot_quicklook(plt, x, y, z, fcbh, scbh, tcbh, now, numplots, quality, image_dir, get_fn, site="rig", description=""):
+    """Plot the large quicklook image
+
+    This private function should only be run from the make_plot() function.
+    Creates all large quicklook images so there are C{numplots} images.
+    See make_plot() for details.
+
+    @param x: The time data gathered from the NetCDF file in make_plot()
+    @param y: The altitude data created in the _prepare_Data()
+    @param z: The backscatter data gathered from the NetCDF file in make_plot()
+    @param now: The datetime object passed from make_plot()
+    @param numplots: The number of plots per day passed from make_plot()
+    @param quality: The number to take every other data point from make_plot()
+    @param image_dir: The directory to put the created image into.
+    @type x: 1-D Numpy Array
+    @type y: 1-D Numpy Array
+    @type z: 2-D Numpy Array
+    @type now: datetime object
+    @type numplots: int
+    @type quality: int
+    @type image_dir: str
+    """
+    created_files = []
+    fig0 = plt.figure()
+    sp1 = plt.subplot(1,1,1,axisbg='black')
+    end = mytime.datetime(now.year, now.month, now.day, 24/numplots, 0, 0)
+    begin = end - mytime.timedelta(hours=24/numplots)
+    base_day = begin.day
+    if end.hour == 0:
+        end_hour = 24
+    else:
+        end_hour = end.hour
+
+    plt.pcolor(x[0:-1:quality], y, z[:,0:-1:quality], shading = 'flat', vmin=0, vmax=10)
+    plt.colorbar()
+    plt.plot(x[0:-1:quality], fcbh[0:-1:quality], marker=c.FCBH_MARKER,
+            color=c.FCBH_COLOR, markeredgecolor=c.FCBH_ECOLOR, linewidth=0,
+            markersize=c.FCBH_MSIZE)
+    plt.plot(x[0:-1:quality], scbh[0:-1:quality], marker=c.SCBH_MARKER,
+            color=c.SCBH_COLOR, markeredgecolor=c.SCBH_ECOLOR, linewidth=0,
+            markersize=c.SCBH_MSIZE)
+    plt.plot(x[0:-1:quality], tcbh[0:-1:quality], marker=c.TCBH_MARKER,
+            color=c.TCBH_COLOR, markeredgecolor=c.TCBH_ECOLOR, linewidth=0,
+            markersize=c.TCBH_MSIZE)
+    plt.axis([begin.hour,end_hour,0,7])
+    title = c.BACK_TITLE.replace("'date1'", "%02d/%02d/%04d" % (now.month,now.day,now.year))
+    plt.title(title)
+    plt.ylabel(c.BACK_YLABEL)
+    plt.xlabel(c.BACK_XLABEL)
+
+    if not os.path.exists(image_dir): os.makedirs(image_dir)
+
+    filename = get_fn(begin, end - mytime.timedelta(seconds=1), ptype=1, tag="", site=site, description=description)
+    file = os.path.join(image_dir, filename)
+    plt.savefig(file)
+    _log.debug("Ceilometer Backscatter Quicklook saved as %s" % file)
+    created_files.append(file)
+
+    end = end + mytime.timedelta(hours=24/numplots)
+    begin = end - mytime.timedelta(hours=24/numplots)
+    while begin.day == base_day:
+        if end.hour == 0:
+            end_hour = 24
+        else:
+            end_hour = end.hour
+        plt.axis([begin.hour,end_hour,0,7])
+        filename = get_fn(begin, end - mytime.timedelta(seconds=1), ptype=1, tag="", site=site, description=description)
+        file = os.path.join(image_dir, filename)
+        plt.savefig(file)
+        _log.debug("Ceilometer Backscatter Quicklook saved as %s" % file)
+        created_files.append(file)
+
+        end =   end + mytime.timedelta(hours=24/numplots)
+        begin = end - mytime.timedelta(hours=24/numplots)
+    plt.close()
+    return created_files
+
+def _plot_thumbnail(plt, x, y, z, now, numplots, quality, image_dir, get_fn, site="rig", description=""):
+    """Plot the smaller thumbnail quicklook images
+
+    This private function should only be run from the make_plot() function.
+    Creates all thumbnail images so there are C{numplots} images.
+    See make_plot() for details.
+
+    @param x: The time data gathered from the NetCDF file in make_plot()
+    @param y: The altitude data created in the _prepare_Data()
+    @param z: The backscatter data gathered from the NetCDF file in make_plot()
+    @param now: The datetime object passed from make_plot()
+    @param numplots: The number of plots per day passed by make_plot()
+    @param quality: The number to take every other data point from make_plot()
+    @param image_dir: The directory to put the created image into
+    @param tn_size: The figure size of the thumbnail image
+    @param title: Specify to have a title or not
+    @type x: 1-D Numpy Array
+    @type y: 1-D Numpy Array
+    @type z: 2-D Numpy Array
+    @type now: datetime object
+    @type numplots: int
+    @type quality: int
+    @type image_dir: str
+    @type tn_size: 2-element integer tuple
+    @type title: boolean
+    """
+    created_files = []
+    end = mytime.datetime(now.year, now.month, now.day, 24/numplots, 0, 0)
+    begin = end - mytime.timedelta(hours=24/numplots)
+    base_day = begin.day
+    
+    fig1 = plt.figure(2, figsize=c.TN_SIZE)
+    spt = plt.subplot(1,1,1,axisbg='black')
+    plt.subplots_adjust(left=0,right=1,bottom=0,top=1)
+    if end.hour == 0:
+        end_hour = 24
+    else:
+        end_hour = end.hour
+
+    plt.pcolor(x[0:-1:quality], y, z[:,0:-1:quality], shading='flat', vmin=0, vmax=10)
+    plt.axis([begin.hour,end_hour,0,7])
+    tl = spt.yaxis.get_ticklines()
+    for t in tl:
+        t.set_visible(False)
+
+    if not os.path.exists(image_dir): os.makedirs(image_dir)
+
+    filename = get_fn(begin, end - mytime.timedelta(seconds=1), ptype=1, tag="tn", site=site, description=description)
+    file = os.path.join(image_dir, filename)
+    plt.savefig(file)
+    _log.debug("Ceilometer Backscatter Thumbnail saved as %s" % file)
+    created_files.append(file)
+
+    end = end + mytime.timedelta(hours=24/numplots)
+    begin = end - mytime.timedelta(hours=24/numplots)
+    while begin.day == base_day:
+        if end.hour == 0:
+            end_hour = 24
+        else:
+            end_hour = end.hour
+        plt.axis([begin.hour,end_hour,0,7])
+        filename = get_fn(begin, end - mytime.timedelta(seconds=1), ptype=1, tag="tn", site=site, description=description)
+        file = os.path.join(image_dir, filename)
+        plt.savefig(file)
+        _log.debug("Ceilometer Backscatter Thumbnail saved as %s" % file)
+        created_files.append(file)
+        
+        end = end + mytime.timedelta(hours=24/numplots)
+        begin = end - mytime.timedelta(hours=24/numplots)
+    plt.close()
+    return created_files
+
+def main():
+    """Handles command line arguments if run as a script
+
+    Run first from command-line.
+
+    No Arguments or Run as Module
+    =============================
+    Creates, by default and by running the make_plot() function, full size quicklook plot of Ceilometer data
+    and a thumbnail quicklook image without a title of the same plot.
+
+
+    Run Script
+    ==========
+    Usage::
+        quicklook_ceilo.py [-t | -f filename | --numplots=2 | --quality=1 | --date="MM/DD/YYYY" | --blank]
+        Try `python quicklook_ceilo.py -h' for more information.