Add bash script for running level b1 processing

aosstower/level_b1/nc.py

@@ -2,13 +2,12 @@ import os
 import sys
 import logging
 import pandas as pd
-from datetime import datetime as dt
+from datetime import datetime
 from netCDF4 import Dataset
 import numpy as np
 import platform
 from aosstower import station, schema
 from aosstower.level_00 import parser
-from datetime import timedelta as delta
 from aosstower.level_b1 import calc
 
 LOG = logging.getLogger(__name__)
@@ -437,7 +436,7 @@ def create_giant_netcdf(input_files, output_fn, zlib, chunk_size,
     else:
         chunk_sizes = [frame.shape[0]]
 
-    first_stamp = dt.strptime(str(frame.index[0]), '%Y-%m-%d %H:%M:%S')
+    first_stamp = datetime.strptime(str(frame.index[0]), '%Y-%m-%d %H:%M:%S')
     # NETCDF4_CLASSIC was chosen so that MFDataset reading would work. See:
     # http://unidata.github.io/netcdf4-python/#netCDF4.MFDataset
     nc_file = Dataset(output_fn, 'w', format='NETCDF4_CLASSIC')
@@ -455,9 +454,12 @@ def create_giant_netcdf(input_files, output_fn, zlib, chunk_size,
 def _dt_convert(datetime_str):
     """Parse datetime string, return datetime object"""
     try:
-        return dt.strptime(datetime_str, '%Y-%m-%dT%H:%M:%S')
+        return datetime.strptime(datetime_str, '%Y%m%d')
     except ValueError:
-        return dt.strptime(datetime_str, '%Y-%m-%d')
+        try:
+            return datetime.strptime(datetime_str, '%Y-%m-%d')
+        except ValueError:
+            return datetime.strptime(datetime_str, '%Y-%m-%dT%H:%M:%S')
 
 
 def main():
@@ -477,9 +479,7 @@ def main():
                                 "\'YYYY-MM-DDTHH:MM:SS\', \'YYYY-MM-DD\'")
     parser.add_argument('-n', '--interval', default='1T',
                         help="""Width of the interval to average input data
-over in Pandas offset format. If not specified, 1 minute averages are used. If
-specified then '_high', '_mean', and '_low' versions of the data fields are
-written to the output NetCDF.
+over in Pandas offset format. If not specified, 1 minute averages are used.
 Use '1D' for daily or '5T' for 5 minute averages.
 See this page for more details:
 http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases""")
@@ -492,7 +492,7 @@ http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases""")
     parser.add_argument('--data-stream', help="'datastream' global attribute to put in output file")
 
     parser.add_argument('-i', '--input', dest='input_files', required=True, nargs="+",
-                        help="aoss_tower level_00 paths. Use @filename to red a list of paths from that file.")
+                        help="aoss_tower level_00 paths. Use @filename to read a list of paths from that filename.")
 
     parser.add_argument('-o', '--output', dest='output_files', required=True, nargs="+",
                         help="""NetCDF filename(s) to create from input. If one

aosstower/level_b1/quicklook.py

@@ -446,10 +446,14 @@ def create_plot(plot_names, frame, output,
 
 
 def _dt_convert(datetime_str):
+    """Parse datetime string, return datetime object"""
     try:
-        return datetime.strptime(datetime_str, '%Y-%m-%dT%H:%M:%S')
+        return datetime.strptime(datetime_str, '%Y%m%d')
     except ValueError:
-        return datetime.strptime(datetime_str, '%Y-%m-%d')
+        try:
+            return datetime.strptime(datetime_str, '%Y-%m-%d')
+        except ValueError:
+            return datetime.strptime(datetime_str, '%Y-%m-%dT%H:%M:%S')
 
 
 def main():
@@ -465,7 +469,7 @@ def main():
         "only that day is created. Formats allowed: \'YYYY-MM-DDTHH:MM:SS\', \'YYYY-MM-DD\'")
     parser.add_argument('-e', '--end-time', type=_dt_convert,
     help="End time of plot. If only -e is given, a plot of only that day is " +
-          "created. Formats allowed: \'YYYY-MM-DDTHH:MM:SS\', \'YYYY-MM-DD\'")
+          "created. Formats allowed: \'YYYY-MM-DDTHH:MM:SS\', \'YYYY-MM-DD\', \'YYYYMMDD\'")
     parser.add_argument('--met-plots', nargs='+',
                         help="Override plots to use in the combined meteorogram plot")
     parser.add_argument("input_files", nargs="+", help="aoss_tower_level_b1 files")

scripts/archive_tower.sh

@@ -29,7 +29,6 @@ fi
     fi
 
     log_info "$(date +%Y-%m-%dT%H:%M:%S): Running archive jobs for ${DATE}" >> $logfile
-    #log_info "$(date +%Y-%m-%dT%H:%M:%S): Running archive jobs for the past 3 days" >> $logfile
     $ENV/bin/python -m metobscommon.archive.incoming -vv -l $logfile --date=${DATE} aoss.tower
 
     log_info "Done"

scripts/metobs_config.sh

@@ -11,6 +11,8 @@ export TOWER_CACHE_DIR=/mnt/inst-data/cache/aoss/tower
 export ENV=/data1/software/aoss-tower3
 # Directory where logs will be stored
 export LOGDIR=$ENV/logs
+# Directory where work files can be placed (inside a separate temp directory)
+export WORKDIR=/mnt/inst-data/tmp
 
 log_info() {
     echo "INFO: $*" &>> $logfile
@@ -21,3 +23,26 @@ oops() {
     exit 1
 }
 
+day_before() {
+    date %Y%m%d --date "$1 -1 day"
+}
+
+work_dir() {
+    mktemp -d --tmpdir="$WORKDIR" "work_$1_"
+}
+
+cache_level_00_file() {
+    d=$1
+    year=${d:0:4}
+    month=${d:4:2}
+    day=${d:6:2}
+    echo "${TOWER_CACHE_DIR}/aoss/tower/level_00/version_00/${year}/${month}/${day}/aoss_tower.${year}-${month}-${day}.ascii"
+}
+
+cache_level_b1_file() {
+    d=$1
+    year=${d:0:4}
+    month=${d:4:2}
+    day=${d:6:2}
+    echo "${TOWER_CACHE_DIR}/aoss/tower/level_b1/version_00/${year}/${month}/${day}/aoss_tower.${year}-${month}-${day}.nc"
+}
\ No newline at end of file

scripts/run_tower_level_b1.sh

+#!/usr/bin/env bash
+
+# Description: Create Level b1 netcdf4 files and the corresponding quicklooks
+SCRIPT_HOME="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+SCRIPT_NAME=$(basename $0)
+SCRIPT_NAME=${SCRIPT_NAME/.sh/}
+
+# Get environment variables and common functions
+source $SCRIPT_HOME/metobs_config.sh
+
+DATE=$1
+if [ -z "$DATE" ]; then
+   DATE=`date +%Y%m%d`
+fi
+LOCK="${ENV}/locks/${SCRIPT_NAME}.lock"
+logfile="${LOGDIR}/${SCRIPT_NAME}.log"
+
+if [ ! -d $LOGDIR ]; then
+    oops "Log directory doesn't exist: $LOGDIR"
+    exit 1
+fi
+
+(
+    flock -x -n 200 || log_info "Script is already running, will not run again."
+
+    if [ ! -d $TOWER_CACHE_DIR ]; then
+        oops "Tower cache directory doesn't exist: $TOWER_CACHE_DIR"
+    fi
+
+    log_info "$(date +%Y-%m-%dT%H:%M:%S): Running level b1 jobs for ${DATE}" >> $logfile
+
+    tmp_dir=`work_dir "$DATE"`
+
+    ### NetCDF Generation ###
+    # properly generating a netcdf file for day X at least requires the data for day X-1 and X
+    previous_date=`day_before "$DATE"`
+    prev_file=`cache_level_00_file "$previous_date"`
+    curr_file=`cache_level_00_file "$DATE"`
+    out_file=`cache_level_b1_file "$DATE"`
+    out_fn=`basename "$out_file"`
+    tmp_out="$tmp_dir/$out_fn"
+    $ENV/bin/python -m aosstower.level_b1.nc -vv -z -i "$prev_file" "$curr_file" --date="${DATE}" -o "$tmp_out" >> $logfile
+    nc_status=$?
+    if [ $nc_status -ne 0 ]; then
+        oops "NetCDF generation failed for $DATE"
+    fi
+
+    echo "Moving NetCDF file from temp directory to cache" >> $logfile
+    $ENV/bin/python -m metobscommon.archive.incoming -vv -l $logfile --dates=${DATE} b1 aoss.tower "$tmp_out"
+
+    ### Quicklook Generation ###
+    # assumes that out_file is what the archive script wrote the file as
+    $ENV/bin/python -m aosstower.level_b1.quicklook -vv --thumbnail -s "$DATE" -i "$out_file" -o "$tmp_dir/aoss_tower.{plot_name}.{start_time:%Y-%m-%d}.png" -p meteorogram td pressure wind_speed wind_dir accum_precip solar_flux
+    quicklook_status=$?
+    if [ $quicklook_statis -ne 0 ]; then
+        oops "Quicklook generation failed for $DATE"
+    fi
+
+    echo "Moving Level B1 quicklooks from temp directory to cache" >> $logfile
+    $ENV/bin/python -m metobscommon.archive.incoming -vv -l $logfile --dates=${DATE} b1 aoss.tower "$tmp_dir/aoss_tower.*.png"
+
+    log_info "Done"
+
+) 200>$LOCK