Add bash scripts for archive processing to scripts directory

scripts/README.rst

+MetObs Archive Scripts
+======================
+
+The following directory contains bash and python scripts for simplifying
+calls to manage and process data files associated with the current environment.
+While some of these scripts may simply call a python module others provide
+complex locking, logging, and error handling that is best kept under revision
+control.
+
+This directory is meant to be softlinked (or git cloned) in a processing
+environment on the processing server. Normally this is something like:
+
+    /data1/software/<site>-<inst>/scripts -> repos/git/<Site><Inst>/scripts
+
+All scripts in this directory should be command line executable except
+for the "metobs_config.sh" script which is meant as a shared configuration
+script between all the processing scripts. See individual scripts for
+a better description of their purpose.
\ No newline at end of file

scripts/archive_tower.sh

+#!/usr/bin/env bash
+
+# Description: Pull data from the AOSS Rooftop Tower instrument for a specific day
+SCRIPT_HOME="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+SCRIPT_NAME=$(basename $0)
+SCRIPT_NAME=${SCRIPT_NAME/.sh/}
+
+# Get environment variables and common functions
+source $SCRIPT_HOME/metobs_config.sh
+
+# Assume we are in /data1/software/<site>-<inst>/repos/git/<Site><Inst>/scripts
+# and need to get to /data1/software/<site-<inst>
+ENV=`readlink -f ../../../../`
+LOGDIR=$ENV/logs
+
+DATE=$1
+if [ -z "$DATE" ]; then
+   DATE=`date +%Y%m%d`
+fi
+LOCK="${ENV}/locks/${SCRIPT_NAME}.lock"
+logfile="${LOGDIR}/${SCRIPT_NAME}.log"
+
+if [ ! -d $LOGDIR ]; then
+    oops "Log directory doesn't exist: $LOGDIR"
+    exit 1
+fi
+
+(
+    flock -x -n 200 || log_info "Script is already running, will not run again."
+
+    if [ ! -d $TOWER_INCOMING_DIR ]; then
+        log_info "Creating incoming directory $TOWER_INCOMING_DIR..."
+        mkdir -p $TOWER_INCOMING_DIR
+    fi
+
+    log_info "$(date +%Y-%m-%dT%H:%M:%S): Running archive jobs for ${DATE}" >> $logfile
+    #log_info "$(date +%Y-%m-%dT%H:%M:%S): Running archive jobs for the past 3 days" >> $logfile
+    $ENV/bin/python -m metobscommon.archive.incoming -vv -l $logfile --date=${DATE} aoss.tower
+
+    log_info "Done"
+
+) 200>$LOCK
\ No newline at end of file

scripts/metobs_config.sh

+#
+# These values should override values used in python or bash scripts
+#
+# where the tower ascii files get pushed to
+export TOWER_INCOMING_DIR=/mnt/inst-data/incoming/aoss/tower
+# The archived location (primary raw)
+export TOWER_PRAW_DIR=/mnt/inst-data/raw/aoss/tower
+# Where generated products go
+export TOWER_CACHE_DIR=/mnt/inst-data/cache/aoss/tower
+
+log_info() {
+    echo "INFO: $*" &>> $logfile
+}
+
+oops() {
+    echo "ERROR: $*" &>> $logfile
+    exit 1
+}
+