diff --git a/aosstower/level_b1/nc.py b/aosstower/level_b1/nc.py
index 0f696a4e36817546e6450326e4850c4c1194f541..f4e472c915b22026a9cb8c875bbc8ed4d1ec7d34 100644
--- a/aosstower/level_b1/nc.py
+++ b/aosstower/level_b1/nc.py
@@ -151,8 +151,8 @@ def create_variables(nc_file, first_stamp, database, chunk_sizes=None, zlib=Fals
variable.ancillary_variables = 'base_time'
# CF default
- # if 'time' in key:
- # variable.calendar = 'gregorian'
+ if 'time' in key:
+ variable.calendar = 'gregorian'
for entry in sorted(database.keys()):
if entry == 'stamp':
diff --git a/aosstower/tower_quicklooks/create_quicklook.py b/aosstower/tower_quicklooks/create_quicklook.py
index 7873f03171e65aeaaa477b9f65d31480d3908768..e8da206127711847555ec727006368c150e68712 100644
--- a/aosstower/tower_quicklooks/create_quicklook.py
+++ b/aosstower/tower_quicklooks/create_quicklook.py
@@ -1,6 +1,6 @@
import os
-from datetime import datetime as dt
-from datetime import timedelta as delta
+import sys
+from datetime import datetime as dt, timedelta as delta
import logging
import pandas as pd
from netCDF4 import MFDataset, MFTime
@@ -8,99 +8,184 @@ import numpy as np
import matplotlib.pyplot as plt
import math
-CHOICES = ['air_temp', 'dewpoint',
- 'rh', 'pressure', 'wind_speed', 'wind_dir', 'accum_precip', 'solar_flux']
-
-TITLES = {'air_temp': 'Temperature and Dewpoint(°C)',
- 'rh': 'Relative Humidity(%)', 'pressure': 'Pressure(hpa)',
- 'wind_speed': 'Wind Speed(m/s)', 'wind_dir': 'Wind Direction(°)',
- 'accum_precip': 'Accumulated Precipitation Since 00Z (mm)',
- 'solar_flux': 'Solar Flux(W/m^2)'}
-
-IND_TITLES = {'air_temp': 'Temperature and Dewpoint',
- 'rh': 'Relative Humidity', 'pressure': 'Pressure',
- 'wind_speed': 'Wind Speed', 'wind_dir': 'Wind Direction',
- 'accum_precip': 'Accumulated Precipitation Since 00Z',
- 'solar_flux': 'Solar Flux'}
-
-# The purpose of this method is to take a string in the format
-# YYYY-MM-DDTHH:MM:SS and convert that to a datetime object
-# or a string in the format YY-MM-DD and convert that to a datetime object
-# used in coordination with argparse -s and -e params
-# @param datetime string
-# @return datetime object
-
-def _dt_convert(datetime_str):
- #parse datetime string, return datetime object
- try:
- return dt.strptime(datetime_str, '%Y-%m-%dT%H:%M:%S')
-
- except:
- return dt.strptime(datetime_str, '%Y-%m-%d')
-
-# The purpose of this method is to take a list of level_b1 filepaths
-# and convert those into a pandas frame with only good data
-# @param input_files - list of level_b1 filepaths
-# @return frame - pandas dataframe with only good data
+LOG = logging.getLogger(__name__)
+
+# names of the plots used in title (default is `.title()` of plot name)
+TITLES = {
+ 'air_temp': 'Air Temperature',
+ 'td': 'Air and Dewpoint Temperature',
+ 'rh': 'Relative Humidity',
+ 'wind_dir': 'Wind Direction',
+ 'accum_precip': 'Accumulated Precipitation Since 0Z',
+}
+
+
+class PlotMaker(object):
+ """Object for making plots and storing/validating plot metadata"""
+ def __init__(self, name, dependencies, title=None, units=None):
+ self.name = name
+ self.deps = dependencies
+ self._full_figure = None
+ if title is None:
+ title = "{title_prefix}{title_name}{units} {start_time:%Y-%m-%d}"
+ self._title = title
+ self.units = units
+
+ def missing_deps(self, frame):
+ """Get dependency variables missing from the provided frame"""
+ for var_name in self.deps:
+ if var_name not in frame:
+ yield var_name
+
+ def get_title(self, frame, is_subplot):
+ if self._title:
+ title_prefix = "AO&SS Building Tower " if not is_subplot else ''
+ title_name = TITLES.get(self.name, self.name.replace('_', ' ').title())
+ unit_str = '({})'.format(self.units) if self.units and is_subplot else ''
+ title = self._title.format(title_prefix=title_prefix,
+ title_name=title_name,
+ units=unit_str,
+ start_time=frame.index[0].to_pydatetime())
+ else:
+ title = ''
+ return title
+
+ def get_yticks(self, ymin, ymax, num_plots):
+ delta = math.ceil((ymax - ymin) / num_plots)
+ new_ticks = np.arange(ymin, (ymin + delta * num_plots), delta)
+ if not new_ticks:
+ return [ymin, ymin + 0.05, ymin + 0.1]
+ return new_ticks
+
+ def get_ylabel(self, is_subplot=False):
+ y_label = TITLES.get(self.name, self.name.replace('_', ' ').title())
+ if is_subplot:
+ return None
+ if self.units:
+ return "{} ({})".format(y_label, self.units)
+ return y_label
+
+ def create_plot(self, frame, fig, is_subplot=None, shared_x=None, title=None):
+ """
+
+ :param frame:
+ :param fig:
+ :param is_subplot: None or (num plots, num columns, num_rows)
+ :param shared_x:
+ :return:
+ """
+ if title is None:
+ title = self.get_title(frame, is_subplot)
+
+ if is_subplot:
+ ax = fig.add_subplot(*is_subplot, sharex=shared_x)
+ ax.set_title(title, x=0.5, y=get_subtitle_location(is_subplot[0]), fontsize=8)
+ else:
+ ax = fig.add_subplot(111, sharex=shared_x)
+ fig.suptitle(title, fontsize=13)
+ y_label = self.get_ylabel(is_subplot)
+ if y_label:
+ ax.set_ylabel(y_label)
+ plt.sca(ax)
+
+ # get the min for each column then combine them assuming we can
+ specific_frame = frame[[x for x in frame.columns if x in self.deps]]
+ ymin = np.floor(specific_frame.min().min())
+ ymax = np.ceil(specific_frame.max().max())
+
+ ax.plot(specific_frame.index, specific_frame, 'k')
+ if ymin == ymax:
+ ax.set_ylim(ymin, ymax + 0.1)
+ else:
+ ax.set_ylim(ymin, ymax)
-def get_data(input_files):
+ if is_subplot:
+ new_ticks = self.get_yticks(ymin, ymax, is_subplot[0])
+ ax.get_yaxis().get_major_ticks()[-1].set_visible(False)
+ ax.set_yticks(new_ticks)
+ else:
+ ax.set_xlabel('Time (UTC)')
+
+ return ax
+
+
+class MeteorogramPlotMaker(PlotMaker):
+ def __init__(self, name, dependencies, plot_deps=None, title=None):
+ self.plot_deps = plot_deps
+ super(MeteorogramPlotMaker, self).__init__(name, dependencies, title=title)
+
+ def create_plot(self, frame, fig, is_subplot=False, shared_x=None):
+ if is_subplot or shared_x:
+ raise ValueError("Meteorogram Plot can not be a subplot or share X-axis")
+
+ title = self.get_title(frame, False)
+ fig.suptitle(title, fontsize=13)
+
+ num_plots = len(self.plot_deps)
+ shared_x = None
+ for idx, plot_name in enumerate(self.plot_deps):
+ plot_maker = PLOT_TYPES.get(plot_name, PlotMaker(plot_name, (plot_name,)))
+ title_name = TITLES.get(plot_name, plot_name.replace('_', ' ').title())
+ ax = plot_maker.create_plot(frame, fig,
+ is_subplot=(num_plots, 1, idx + 1),
+ shared_x=shared_x,
+ title=title_name)
+ if idx == 0:
+ shared_x = ax
+ if idx != num_plots - 1:
+ # Disable the x-axis ticks so we don't interfere with other subplots
+ ax.get_xaxis().get_major_ticks()[-1].set_visible(False)
+ ax.get_xaxis().get_major_ticks()[0].set_visible(False)
+ ax.set_xlabel('Time (UTC)')
+
+ return ax
+
+# map plot name -> variable dependencies
+# if not listed then plot name is assumed to be the same as the variable needed
+PLOT_TYPES = {
+ 'meteorogram': MeteorogramPlotMaker('meteorogram',
+ ('air_temp', 'dewpoint', 'rh', 'wind_speed', 'wind_dir', 'accum_precip'),
+ ('td', 'rh', 'wind_speed', 'wind_dir', 'accum_precip')),
+ 'td': PlotMaker('td', ('air_temp', 'dewpoint'), units="°C"), # air_temp and dewpoint in one plot
+ 'wind_dir': PlotMaker('wind_dir', ('wind_dir',), units='°'), # special tick labels
+ 'rh': PlotMaker('rh', ('rh',), units='%'),
+ 'air_temp': PlotMaker('air_temp', ('air_temp',), units='°C'),
+ 'pressure': PlotMaker('pressure', ('pressure',), units='hpa'),
+ 'dewpoint': PlotMaker('dewpoint', ('air_temp',), units='°C'),
+ 'wind_speed': PlotMaker('wind_speed', ('wind_speed',), units='m/s'),
+ 'accum_precip': PlotMaker('accum_precip', ('accum_precip',), units='mm'),
+ 'solar_flux': PlotMaker('solar_flux', ('solar_flux',), units='W/m^2'),
+}
+
+
+def convert_to_thumbnail(fig):
+ # TODO
+ pass
+
+
+def get_data(input_files, columns):
+ data_dict = {}
files = MFDataset(input_files)
- #dictionary with var_name: good data pairs
- dataDict = {}
-
- #get the data from the files
- for name in CHOICES:
- data = files.variables[name][:]
-
- qc_data = files.variables['qc_' + name][:]
-
- dataDict[name] = data
- dataDict['qc_' + name] = qc_data
+ # get the data from the files
+ for name in columns:
+ if name not in files.variables:
+ LOG.warning("Unknown file variable: {}".format(name))
+ continue
+ data_dict[name] = files.variables[name][:]
+ data_dict['qc_' + name] = files.variables['qc_' + name][:]
- #for some reason, time has floating point precision issues
- #so I'm using base_time and offsets instead
- #might need to fix later
+ # convert base_time epoch format into date_time object
base_time = files.variables['base_time'][:]
-
- #convert base_time epoch format into date_time object
- base_time_obj = dt(1970,1,1) + delta(seconds=int(base_time))
+ base_time_obj = dt(1970, 1, 1) + delta(seconds=int(base_time))
- #get offsets from files
- off_sets = files.variables['time_offset']
-
- #convert that into offsets from the first file's base_time
- off_sets = MFTime(off_sets)[:]
-
- #for each offset, convert that into a datetime object
- stamps = [base_time_obj + delta(seconds=int(s)) for s in off_sets]
+ # convert per-file offsets to offsets based on the first file's base_time
+ offsets = MFTime(files.variables['time_offset'])[:]
+ # for each offset, convert that into a datetime object
+ data_dict['stamps'] = [base_time_obj + delta(seconds=int(s)) for s in offsets]
- #append all stamps into the data frame
- dataDict['stamps'] = stamps
+ return pd.DataFrame(data_dict).set_index(['stamps'])
- return pd.DataFrame(dataDict).set_index(['stamps'])
-
-# The purpose of this method is to use the qc mask
-# to only get the good data
-# @param qc_data - np array with nan for good values and various set bits for others
-# @param stamps - all stamps corresponding to data
-# @param data - data
-# @return good data and their corresponding stamps
-
-def get_good_data(qc_data, stamps, data):
- #all good data and stamps associated with them
- good_data = []
- good_stamps = []
-
- #get good indices
- good_indices = np.where(np.isnan(qc_data))[0]
-
- for idx in good_indices:
- #get good_data
- good_data.append(data[idx])
- good_stamps.append(stamps[idx])
-
- return [good_data, good_stamps]
# The purpose of this method is to determines all the 12:00:00 days
# within a start and end date
@@ -194,7 +279,7 @@ def thumbnail_plot(dates, data, ymin, ymax,
# @param create_air_dew_plot - boolean specifying if air temp and dewpoint
# @param output - output filename pattern
# should be in same plot
-def create_thumbnail(frame, ymin, ymax, create_air_dew_plot, output):
+def create_thumbnail(plot_names, frame, ymin, ymax, create_air_dew_plot, output):
#see if we're going to need subplots
#subplots needed when len is greater than 2 for one variable
#or greater than 4 for two variables
@@ -213,8 +298,7 @@ def create_thumbnail(frame, ymin, ymax, create_air_dew_plot, output):
numberPlots = len(list(frame.columns.values)) / 2
plotNumber = numberPlots * 100 + 10
- stamps = list(frame.index)
-
+ stamps = frame.index
#get dewpoint data
if create_air_dew_plot:
all_data = frame['dewpoint']
@@ -222,9 +306,9 @@ def create_thumbnail(frame, ymin, ymax, create_air_dew_plot, output):
good_list = get_good_data(qc_data, stamps, all_data)
dewpoint_data = good_list[0]
- dewpoint_stamps= good_list[1]
+ dewpoint_stamps = good_list[1]
- for name in CHOICES:
+ for name in plot_names:
if name == 'dewpoint' and create_air_dew_plot:
continue
@@ -257,18 +341,12 @@ def create_thumbnail(frame, ymin, ymax, create_air_dew_plot, output):
thumbnail_plot(good_stamps, good_data, ymin[name], ymax[name],
dewpoint_stamps, dewpoint_data, output, False)
- plt.savefig(output + '.thumbnail.png')
+ plt.savefig(output.format(plot_name='test', start_time=good_stamps[0]))
def full_plot_stamps(stamps):
- first_stamp = np.sort(stamps)[0]
-
- first_stamp = first_stamp.replace(hour=0, minute=0, second=0)
-
- #print([((s - first_stamp).total_seconds()/3600) for s in stamps])
-
- #print([(int((s - first_stamp).total_seconds()) / 3600) for s in stamps])
-
- return [((s - first_stamp).total_seconds()/3600) for s in stamps]
+ # assume the timestamps are monotonic
+ first_stamp = stamps[0].replace(hour=0, minute=0, second=0)
+ return [((s - first_stamp).total_seconds()/3600) for s in stamps]
# The purpose of this method is to change how the y ticks are shown
# @param min - minimum of the y range right now
@@ -277,8 +355,7 @@ def full_plot_stamps(stamps):
def get_new_labels(mini, maxi):
delta = math.ceil((maxi - mini) / 6)
-
- return np.arange(mini, (mini + delta*6), delta)
+ return np.arange(mini, (mini + delta*6), delta)
# The purpose of this method is to create a full_size plot
# @param frame - all data I need in a data frame
@@ -292,7 +369,7 @@ def get_new_labels(mini, maxi):
# @param wind_direction - says different yaxis labels
# @param accum_precip - different yaxis handling
# @param see if we need subplots
-def full_plot(dates, data, ymin, ymax,
+def full_plot(fig, axes, dates, data, ymin, ymax,
dewpoint_stamps, dewpoint_data, output, wind_dir,
accum_precip, need_subplots):
@@ -318,15 +395,13 @@ def full_plot(dates, data, ymin, ymax,
else:
+ # import ipdb; ipdb.set_trace()
plt.plot(dates, data, 'k')
- plt.axis([min(dates), max(dates), ymin, ymax])
+ # plt.axis([min(dates), max(dates), ymin, ymax])
if not need_subplots:
return
- #get axes
- axes = plt.gca()
-
#if not wind_direction, reset yaxis ticks
if not wind_dir:
if not accum_precip:
@@ -358,323 +433,207 @@ def full_plot(dates, data, ymin, ymax,
def get_subtitle_location(numSubPlots):
return 1 - 0.05*numSubPlots
-# The purpose of this method is to create a full size quicklook
-# @param frame - all data I need in a data frame
-# @param ymin - lower limit of the y axis if specified. set only when --daily is given
-# @param ymax - upper limit of yaxis if specified. Set only when --daily is given
-# @param create_air_dew_plot - boolean specifying if air temp and dewpoint
-# should be in same plot
-def create_full_plot(frame, ymin, ymax, create_air_dew_plot, output):
- #see if we're going to need subplots
- #subplots needed when len is greater than 2 for one variable
- #or greater than 4 for two variables
-
- need_subplots = len(list(frame.columns.values)) > 2 and not create_air_dew_plot
- need_subplots = need_subplots or len(list(frame.columns.values)) > 4 and create_air_dew_plot
-
- #get need supplot vars
- if need_subplots:
- plots_created = 1
-
- if create_air_dew_plot:
- numberPlots = (len(list(frame.columns.values)) - 2) / 2
- plotNumber = numberPlots * 100 + 10
-
- else:
- numberPlots = len(list(frame.columns.values)) / 2
- plotNumber = numberPlots * 100 + 10
-
- stamps = list(frame.index)
-
- #get dewpoint data
- if create_air_dew_plot:
- all_data = frame['dewpoint']
- qc_data = frame['qc_dewpoint']
-
- good_list = get_good_data(qc_data, stamps, all_data)
- dewpoint_data = good_list[0]
- dewpoint_stamps= good_list[1]
-
- for name in CHOICES:
- #if dewpoint with temp or a name not in frame
- # continue
- if name == 'dewpoint' and create_air_dew_plot:
- continue
-
- if name not in list(frame.columns.values):
- continue
-
- all_data = frame[name]
- qc_data = frame['qc_' + name]
-
- good_list = get_good_data(qc_data, stamps, all_data)
- good_data = good_list[0]
- good_stamps= good_list[1]
-
- dateString = good_stamps[0].strftime('%m/%d/%Y')
-
- # create plots
- if need_subplots:
- if plots_created == 1:
- plt.figure().suptitle('AO&SS Building Tower Meteorogram ' + dateString, fontsize=13)
- ax1 = plt.subplot(plotNumber + plots_created)
- ax1.set_title(TITLES[name], x=0.5, y=get_subtitle_location(numberPlots), fontsize=8)
-
- else:
- curr_plot = plt.subplot(plotNumber + plots_created, sharex=ax1)
- curr_plot.set_title(TITLES[name], x=0.5, y=get_subtitle_location(numberPlots), fontsize=8)
-
- plots_created += 1
-
- else:
- plt.figure().suptitle('AO&SS Building Tower ' + IND_TITLES[name] + ' ' + dateString, fontsize=13)
- plt.ylabel(TITLES[name])
-
- #if we don't need two lines in same plot, plot
- if not (create_air_dew_plot and name == 'air_temp'):
- full_plot(good_stamps, good_data, ymin[name], ymax[name],
- None, None, output, name == 'wind_dir',
- name == 'accum_precip', need_subplots)
-
- else:
- full_plot(good_stamps, good_data, ymin[name], ymax[name],
- dewpoint_stamps, dewpoint_data, output, False, False,
- need_subplots)
-
- if need_subplots:
- plt.subplots_adjust(hspace=0, bottom=0.125)
-
- plt.xlabel('Time (UTC)')
-
- #reverse code done in create_full_plot
- axes = plt.gca()
- axes.get_xaxis().get_major_ticks()[-1].set_visible(True)
- axes.get_xaxis().get_major_ticks()[0].set_visible(True)
+def create_full_plot(plot_names, frame, output, start_time=None, end_time=None):
+ """
- plt.savefig(output + '.png')
+ Args:
+ plot_names:
+ frame:
+ output:
+ start_time:
+ end_time:
+ daily: Whether or not this plot should represent one day of data
-# the purpose of this method is to use TW's algo to convert
-# tempC and relhum to dewpoint
-# @param tempC - temperature value in deg C
-# @param relhum - relative humidity value in %
+ Returns:
-def calcDewpoint(tempC, relhum):
"""
- Algorithm from Tom Whittaker tempC is the temperature in degrees Celsius,
- relhum is the relative humidity as a percentage.
-
- :param tempC: temperature in celsius
- :param relhum: relative humidity as a percentage
- """
- if tempC is None or relhum is None:
- return np.NaN
-
- gasconst = 461.5
- latheat = 2500800.0
-
- dp = 1.0 / (1.0 / (273.15 + tempC) - gasconst * np.log((0.0 + relhum) / 100) /
- (latheat - tempC * 2397.5))
-
- return min(dp - 273.15, tempC)
-
-# The purpose of this method is to check to see if we have dewpoint data
-# if we do not, fill in dewpoint data
-# @param frame - pandas frame of nc data
-# @return frame with dewpoint data and qc_dewpoint with only good data flagged
-
-def check_dewpoint(frame):
- dewpoint = frame['dewpoint']
- qc_dewpoint = frame['qc_dewpoint']
-
- good_list = get_good_data(qc_dewpoint.tolist(), list(frame.index), dewpoint.tolist())
-
- # no dewpoint data that is viable
- if len(good_list[0]) == 0:
- temp = frame['air_temp'].tolist()
- rel_hum = frame['rh'].tolist()
- qc_temp = frame['qc_air_temp'].tolist()
- qc_rel_hum = frame['qc_rh'].tolist()
- stamps = list(frame.index)
-
- newDewpoint = {}
-
- qc_dew = {}
-
- for idx, rh_quality in enumerate(qc_rel_hum):
- rh = rel_hum[idx]
- temp_quality = qc_temp[idx]
- air_temp = temp[idx]
- stamp = stamps[idx]
-
- if not math.isnan(temp_quality):
- air_temp = None
-
- elif not math.isnan(rh_quality):
- rh = None
+ #see if we're going to need subplots
+ #subplots needed when len is greater than 2 for one variable
+ #or greater than 4 for two variables
- dew = calcDewpoint(air_temp, rh)
+ # need_subplots = len(list(frame.columns.values)) > 2 and not create_air_dew_plot
+ # need_subplots = need_subplots or len(list(frame.columns.values)) > 4 and create_air_dew_plot
- newDewpoint[stamp] = dew
+ #get need supplot vars
+ # if need_subplots:
+ # plots_created = 1
+ #
+ # if create_air_dew_plot:
+ # numberPlots = (len(list(frame.columns.values)) - 2) / 2
+ # plotNumber = numberPlots * 100 + 10
+ #
+ # else:
+ # numberPlots = len(list(frame.columns.values)) / 2
+ # plotNumber = numberPlots * 100 + 10
+
+ for name in plot_names:
+ plot_maker = PLOT_TYPES.get(name, PlotMaker(name, (name,)))
+ var_names = []
+ for var_name in plot_maker.deps:
+ if var_name not in frame:
+ raise ValueError("Missing required variable '{}' for plot '{}'".format(var_name, name))
+ var_names.append(var_name)
+ # write NaNs where QC values are not 0
+ qc_name = 'qc_' + var_name
+ if qc_name in frame:
+ frame[var_name].mask(frame[qc_name] != 0)
+ var_names.append(qc_name)
+
+ # create a frame that doesn't include any of the bad values
+ plot_frame = frame[var_names]
+ plot_frame = plot_frame[~plot_frame.isnull().any(axis=1)]
+
+ fig = plt.figure()
+ ax = plot_maker.create_plot(plot_frame, fig)
+ ax.set_xlim(start_time, end_time)
+ import matplotlib.dates as md
+ xloc = md.AutoDateLocator(minticks=5, maxticks=8, interval_multiples=True)
+ xfmt = md.AutoDateFormatter(xloc)
+ def _fmt(interval, x, pos=None):
+ x_num = md.num2date(x).replace(tzinfo=None)
+ delta_seconds = (x_num - plot_frame.index[0].replace(hour=0, minute=0, second=0, microsecond=0)).total_seconds()
+ num_hours = delta_seconds / 3600.
+ if interval == md.HOURLY:
+ return "{:02.0f}".format(num_hours)
+ elif interval == md.MINUTELY:
+ num_minutes = delta_seconds / 60.
+ num_minutes -= int(num_hours) * 60.
+ return "{:02.0f}:{:02.0f}".format(int(num_hours), num_minutes)
+ else:
+ return x.strftime("{%Y-%m-%d}")
+ from functools import partial
+ xfmt.scaled[1. / md.MINUTES_PER_DAY] = plt.FuncFormatter(partial(_fmt, md.MINUTELY))
+ xfmt.scaled[1. / md.HOURS_PER_DAY] = plt.FuncFormatter(partial(_fmt, md.HOURLY))
+ ax.xaxis.set_major_locator(xloc)
+ ax.xaxis.set_major_formatter(xfmt)
+
+ out_fn = output.format(plot_name=name, start_time=plot_frame.index[0].to_pydatetime())
+ LOG.info("Saving plot '{}' to filename '{}'".format(name, out_fn))
+ fig.savefig(out_fn)
+
+ # all_data = frame[name]
+ # qc_data = frame['qc_' + name]
+ #
+ # good_list = get_good_data(qc_data, stamps, all_data)
+ # good_data = good_list[0]
+ # good_stamps = good_list[1]
+ #
+ # dateString = good_stamps[0].strftime('%m/%d/%Y')
+ #
+ # # create plots
+ # if need_subplots:
+ # if plots_created == 1:
+ # plt.figure().suptitle('AO&SS Building Tower Meteorogram ' + dateString, fontsize=13)
+ # ax1 = plt.subplot(plotNumber + plots_created)
+ # ax1.set_title(TITLES[name], x=0.5, y=get_subtitle_location(numberPlots), fontsize=8)
+ # else:
+ # curr_plot = plt.subplot(plotNumber + plots_created, sharex=ax1)
+ # curr_plot.set_title(TITLES[name], x=0.5, y=get_subtitle_location(numberPlots), fontsize=8)
+ #
+ # plots_created += 1
+ #
+ # else:
+ # plt.figure().suptitle('AO&SS Building Tower ' + IND_TITLES[name] + ' ' + dateString, fontsize=13)
+ # plt.ylabel(TITLES[name])
+ #
+ # #if we don't need two lines in same plot, plot
+ # if not (create_air_dew_plot and name == 'air_temp'):
+ # full_plot(good_stamps, good_data, ymin[name], ymax[name],
+ # None, None, output, name == 'wind_dir',
+ # name == 'accum_precip', need_subplots)
+ #
+ # else:
+ # full_plot(good_stamps, good_data, ymin[name], ymax[name],
+ # dewpoint_stamps, dewpoint_data, output, False, False,
+ # need_subplots)
+
+ # if need_subplots:
+ # plt.subplots_adjust(hspace=0, bottom=0.125)
+ #
+ # plt.xlabel('Time (UTC)')
- if math.isnan(dew):
- qc_dew[stamp] = 1
- else:
- qc_dew[stamp] = np.NaN
-
- newDewpoint = pd.Series(newDewpoint)
- qc_dew = pd.Series(qc_dew)
+def _dt_convert(datetime_str):
+ try:
+ return dt.strptime(datetime_str, '%Y-%m-%dT%H:%M:%S')
+ except ValueError:
+ return dt.strptime(datetime_str, '%Y-%m-%d')
- frame['dewpoint'] = newDewpoint
- frame['qc_dewpoint'] = qc_dew
-
- return frame
def main():
import argparse
-
- #argparse description
parser = argparse.ArgumentParser(description="Use data from level_b1 netCDF files to create netCDF files")
-
- #argparse verbosity info
- parser.add_argument('-v', '--verbose', action='count',
+ parser.add_argument('-v', '--verbose', action='count',
default=int(os.environ.get("VERBOSITY", 2)),
dest='verbosity',
help=('each occurence increases verbosity 1 level through'
+ ' ERROR-WARNING-INFO-DEBUG (default INFO)'))
-
- #argparse start and end times
parser.add_argument('-s', '--start-time', type=_dt_convert,
help="Start time of plot. If only -s is given, a plot of " +
"only that day is created. Formats allowed: \'YYYY-MM-DDTHH:MM:SS\', \'YYYY-MM-DD\'")
-
parser.add_argument('-e', '--end-time', type=_dt_convert,
help="End time of plot. If only -e is given, a plot of only that day is " +
"created. Formats allowed: \'YYYY-MM-DDTHH:MM:SS\', \'YYYY-MM-DD\'")
-
- #netcdf file paths
- parser.add_argument("input_files", nargs="+", help="aoss_tower_level_b1 paths")
-
- #output filename pattern
- parser.add_argument('-o', '--output', help="filename pattern")
-
- #thumbnail or full
- parser.add_argument('-t', '--thumb-nail', action='store_true', help="if specified, script creates a thumbnail")
-
- #plot names
- parser.add_argument('--varnames', nargs="+", choices=CHOICES,
- required=True,
- help="the variable names for the desired plot. Valid choices: air_temp, " +
- "dewpoint, rh, pressure, wind_speed, wind_dir, accum_precip, solar_flux")
-
- #--daily flag
- parser.add_argument('-d', '--daily', action='store_true',
- help="creates a plot for every day. Usually used to create plots " +
- "that will line up for aoss tower quicklooks page")
-
+ parser.add_argument('--met-plots', nargs='+',
+ help="Override plots to use in the combined meteorogram plot")
+ parser.add_argument("input_files", nargs="+", help="aoss_tower_level_b1 files")
+ parser.add_argument('-o', '--output', default="{plot_name}_{start_time:%Y%m%d_%H%M%S}.png", help="filename pattern")
+ parser.add_argument('-t', '--thumbnail', action='store_true', help="if specified, script creates a thumbnail")
+ parser.add_argument('-p', '--plot-names', nargs="+",
+ required=True,
+ help="the variable names or plot types to create")
+ parser.add_argument('-d', '--daily', action='store_true',
+ help="creates a plot for every day. Usually used to create plots " +
+ "that will line up for aoss tower quicklooks page")
args = parser.parse_args()
levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
-
level = levels[min(3, args.verbosity)]
-
logging.basicConfig(level=level)
- var_names = args.varnames
-
- create_air_dew_plot = False
-
- #see if we need to plot air_temp and dewpoint on the same plot
- if 'air_temp' in var_names and 'dewpoint' in var_names:
- create_air_dew_plot = True
- var_names.remove('dewpoint')
+ if not os.path.splitext(args.output)[-1]:
+ LOG.warning("File pattern provided does not have a file extension")
- #get data
- frame = get_data(args.input_files)
+ # check the dependencies for the meteorogram
+ if args.met_plots:
+ assert 'meteorogram' not in args.met_plots
+ PLOT_TYPES['meteorogram'].deps = args.met_plots
- frame = check_dewpoint(frame)
-
- #only have the data we need
- for name in CHOICES:
- if name == 'dewpoint' and create_air_dew_plot:
- continue
-
- if name not in var_names:
- del frame[name]
- del frame['qc_' + name]
+ plot_deps = [PLOT_TYPES[k].deps if k in PLOT_TYPES else (k,) for k in args.plot_names]
+ plot_deps = list(set(d for deps in plot_deps for d in deps))
+ frame = get_data(args.input_files, plot_deps)
+ bad_plot_names = set(args.plot_names) - (set(frame.columns) | set(PLOT_TYPES.keys()))
+ if bad_plot_names:
+ raise ValueError("Unknown plot name(s): {}".format(", ".join(bad_plot_names)))
#frame only contains data from start-end times
- if(args.start_time and args.end_time):
- start_filter = args.start_time.strftime('%Y-%m-%d %H:%M:%S')
- end_filter = args.end_time.strftime('%Y-%m-%d %H:%M:%S')
- frame = frame[start_filter: end_filter]
-
- #frame only contains data from start-end of that day
- elif(args.start_time):
+ if args.start_time and args.end_time:
+ frame = frame[args.start_time: args.end_time]
+ elif args.start_time:
+ #frame only contains data from start-end of that day
end_time = args.start_time.replace(hour=23, minute=59, second=59)
- start_filter = args.start_time.strftime('%Y-%m-%d %H:%M:%S')
- end_filter = end_time.strftime('%Y-%m-%d %H:%M:%S')
- frame = frame[start_filter: end_filter]
-
- ymin = {}
- ymax = {}
+ frame = frame[args.start_time: end_time]
if not args.daily:
- for name in CHOICES:
- if name == 'dewpoint' and create_air_dew_plot:
- continue
-
- if name not in frame:
- continue
-
- ymin[name] = None
- ymax[name] = None
-
- if(args.thumb_nail):
- create_thumbnail(frame, ymin, ymax, create_air_dew_plot, args.output)
-
- else:
- create_full_plot(frame, ymin, ymax, create_air_dew_plot, args.output)
-
+ # allow plotting methods to write inplace on a copy
+ frames = [frame.copy()]
else:
- for name in CHOICES:
- if name == 'dewpoint' and create_air_dew_plot:
- continue
-
- if name not in frame:
- continue
-
- if name == 'accum_precip':
- ymin[name] = frame[name].min()
- ymax[name] = frame[name].max()
-
- elif name == 'air_temp' and create_air_dew_plot:
- ymin[name] = math.floor(frame[name].min())
- ymax[name] = math.ceil(frame[name].max())
+ frames = (group[1] for group in frame.groupby(frame.index.day))
- dew_min = math.floor(frame['dewpoint'].min())
- dew_max = math.ceil(frame['dewpoint'].max())
-
- ymin[name] = min(dew_min, ymin[name])
- ymax[name] = max(dew_max, ymax[name])
-
-
- else:
- ymin[name] = math.floor(frame[name].min())
- ymax[name] = math.ceil(frame[name].max())
-
- frameList = [(group[1]) for group in frame.groupby(frame.index.day)]
-
- counter = 0
+ for frame in frames:
+ if args.daily:
+ # modify start and end time to the current day
+ start_time = frame.index[0].to_pydatetime().replace(hour=0, minute=0, second=0, microsecond=0)
+ end_time = frame.index[0].to_pydatetime().replace(hour=23, minute=59, second=59, microsecond=999999)
+ else:
+ start_time = args.start_time
+ end_time = args.end_time
- for frame in frameList:
- if args.thumb_nail:
- create_thumbnail(frame, ymin, ymax, create_air_dew_plot, args.output.format(str(counter)))
- else:
- create_full_plot(frame, ymin, ymax, create_air_dew_plot, args.output.format(str(counter)))
- counter += 1
- plt.gcf().clear()
+ if args.thumbnail:
+ create_thumbnail(args.plot_names, frame, args.output, start_time, end_time)
+ else:
+ create_full_plot(args.plot_names, frame, args.output, start_time, end_time)
if __name__ == "__main__":
- main()
+ sys.exit(main())