diff --git a/scripts/README.rst b/scripts/README.rst new file mode 100644 index 0000000000000000000000000000000000000000..ec660e15c3c6a9513da10682b0ddde3e1f2213eb --- /dev/null +++ b/scripts/README.rst @@ -0,0 +1,18 @@ +MetObs Archive Scripts +====================== + +The following directory contains bash and python scripts for simplifying +calls to manage and process data files associated with the current environment. +While some of these scripts may simply call a python module others provide +complex locking, logging, and error handling that is best kept under revision +control. + +This directory is meant to be softlinked (or git cloned) in a processing +environment on the processing server. Normally this is something like: + + /data1/software/<site>-<inst>/scripts -> repos/git/<Site><Inst>/scripts + +All scripts in this directory should be command line executable except +for the "metobs_config.sh" script which is meant as a shared configuration +script between all the processing scripts. See individual scripts for +a better description of their purpose. \ No newline at end of file diff --git a/scripts/archive_tower.sh b/scripts/archive_tower.sh new file mode 100755 index 0000000000000000000000000000000000000000..178e30988237b5f324a07ca16e505301c3d300be --- /dev/null +++ b/scripts/archive_tower.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# Description: Pull data from the AOSS Rooftop Tower instrument for a specific day +SCRIPT_HOME="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +SCRIPT_NAME=$(basename $0) +SCRIPT_NAME=${SCRIPT_NAME/.sh/} + +# Get environment variables and common functions +source $SCRIPT_HOME/metobs_config.sh + +# Assume we are in /data1/software/<site>-<inst>/repos/git/<Site><Inst>/scripts +# and need to get to /data1/software/<site-<inst> +ENV=`readlink -f ../../../../` +LOGDIR=$ENV/logs + +DATE=$1 +if [ -z "$DATE" ]; then + DATE=`date +%Y%m%d` +fi +LOCK="${ENV}/locks/${SCRIPT_NAME}.lock" +logfile="${LOGDIR}/${SCRIPT_NAME}.log" + +if [ ! -d $LOGDIR ]; then + oops "Log directory doesn't exist: $LOGDIR" + exit 1 +fi + +( + flock -x -n 200 || log_info "Script is already running, will not run again." + + if [ ! -d $TOWER_INCOMING_DIR ]; then + log_info "Creating incoming directory $TOWER_INCOMING_DIR..." + mkdir -p $TOWER_INCOMING_DIR + fi + + log_info "$(date +%Y-%m-%dT%H:%M:%S): Running archive jobs for ${DATE}" >> $logfile + #log_info "$(date +%Y-%m-%dT%H:%M:%S): Running archive jobs for the past 3 days" >> $logfile + $ENV/bin/python -m metobscommon.archive.incoming -vv -l $logfile --date=${DATE} aoss.tower + + log_info "Done" + +) 200>$LOCK \ No newline at end of file diff --git a/scripts/metobs_config.sh b/scripts/metobs_config.sh new file mode 100644 index 0000000000000000000000000000000000000000..ee1aa03a48a77a04d6f4bf9560537e2cfbc1299c --- /dev/null +++ b/scripts/metobs_config.sh @@ -0,0 +1,19 @@ +# +# These values should override values used in python or bash scripts +# +# where the tower ascii files get pushed to +export TOWER_INCOMING_DIR=/mnt/inst-data/incoming/aoss/tower +# The archived location (primary raw) +export TOWER_PRAW_DIR=/mnt/inst-data/raw/aoss/tower +# Where generated products go +export TOWER_CACHE_DIR=/mnt/inst-data/cache/aoss/tower + +log_info() { + echo "INFO: $*" &>> $logfile +} + +oops() { + echo "ERROR: $*" &>> $logfile + exit 1 +} +