reprocess_with_correction.sh

#!/usr/bin/env bash
# Tasks:
#   Copy new/incoming data to raw and archive locations
#   Start any processing that needs to be done on that data
# All location information should be controlled by the sparchive_archive and sparchive_post commands (via the sparc_instruments.ini file)
# Except for the initial "experiment exists" check

export PATH=/data1/opt/anaconda/bin:$PATH

if [ $# -ne 4 ]; then
    echo "Usage: ./reprocess_with_correction.sh <experiment name> <instrument> <post_script> <raw_correction_version>"
    exit 1
fi

if [ $USER != "metobs" ]; then
    echo "ERROR: This script should only be run by the 'metobs' user"
    exit 2
fi

source "/opt/metobs/sparchive/scripts/sparchive_env.sh"

EXP_NAME=$1
inst_name=$2
POST_SCRIPT=$3
CORRECTION_VERSION=$4
EXP_PATH="${SPARCHIVE_INCOMING_ROOT}/${EXP_NAME}"
ENV_NAME=`basename ${SPARCHIVE_ENV}`

# Let the environment variable set the past days (default 3)
script_name=$(basename $0)
script_name=${script_name/.sh/}
LOCK=${SPARCHIVE_ENV}/lock/${script_name}.lock
logfile=${SPARCHIVE_LOG_DIR}/${script_name}_${EXP_NAME}.log

log_info() {
    echo "INFO: $*" &>> $logfile
}

oops() {
    echo "ERROR: $*" &>> $logfile
    exit 1
}


find_seq() {
    exp=$1
    inst=$2
    # When the past days is 0 then we need to do a little more work to figure out the series of dates
    if [ ${SPARCHIVE_PAST_DAYS} -eq 0 ]; then
        for dated_dir in ${SPARCHIVE_RAW_ROOT}/${exp}/${inst}/20??????; do
            echo `basename $dated_dir`
        done
    else
        SEQ_START=$((SPARCHIVE_PAST_DAYS - 1))
        for days_ago in `seq ${SEQ_START} -1 0`; do
            date_yyyymmdd=`date -d "$days_ago days ago" +%Y%m%d`
            echo "$date_yyyymmdd"
        done
    fi
}


(
    flock -x -n 200 || log_info "Script is already running, will not run again."

    log_info "Running reprocess_with_correction.sh for experiment ${EXP_NAME}"

    # Load the anaconda environment
    log_info "Loading anaconda environment: ${ENV_NAME}"
    source activate ${ENV_NAME} &>> $logfile || oops "Could not load environment"

    # Run the experiment specific script for post processing (these should already have the environment loaded)
    if [ -f ${POST_SCRIPT} ]; then
        for date_yyyymmdd in `find_seq ${EXP_NAME} ${inst_name}`; do
            log_info "Running post processing script ${POST_SCRIPT} for ${date_yyyymmdd}"
            bash ${POST_SCRIPT} ${EXP_NAME} ${date_yyyymmdd} ${CORRECTION_VERSION} &>> $logfile || (echo "Post processing script failed: ${POST_SCRIPT}" &>> $logfile; continue)
        done
    else
        log_info "Processing script ${POST_SCRIPT} does not exist, nothing to run"
    fi

) 200>$LOCK