run.sh

#!/bin/bash -le
# Usage: run.sh
# Environment variables used for configuration:
# AMQPFIND_ARGS: Arguments to pass to amqpfind when listening for new input
#     events. Should not include the "-C" topic flag (see AMQPFIND_TOPIC).
#     Default: "-H cspp-geo-rabbit -X satellite -u guest -p guest"
# AMQPFIND_TOPIC: Topic to use for incoming data events.
#     Default: "data.goes.*.abi.*.l1b.netcdf.complete"
#     The first asterisk (3rd element) can limit processing to a particular
#     satellite (ex. `g16`). The second asterisk (5th element) can be used
#     to limit to a particular sector (choices: radf, radc, radm1, radm2)
# AMQPSEND_ARGS: Arguments to pass to amqpsend when sending out new data
#     events. Default: "-H cspp-geo-rabbit -X satellite -u guest -p guest"
# G2G_ARGS: Arguments to pass to geo2grid.sh. Should not include `-f` or `-p`
#     (see G2G_PRODUCTS).
#     Default: "-r abi_l1b -w geotiff --num-workers 8 --tiled --compress DEFLATE"
# G2G_PRODUCTS: Products to try to generate with Geo2Grid. Defaults to all 16
#     channels and the True Color composite.
# ADD_OVERVIEWS: Whether or not to add overview (lower resolution) images to
#     the produced geotiff. Default is "1" to create them, use "0" to turn
#     this off.
# OVERVIEW_RESAMPLE: Resampling method to use for "gdaladdo" call. Default
#     "average". See gdaladdo documentation for more options.
# RM_ON_FAILURE: Whether to delete temporary directories regardless of
#     processing success. Default is "1" to always delete, use "0" to turn
#     this off and only delete temporary directories on success.

# Verify that the data mount is available
test -d "/data"

# Change to /data volume to write our output
test -d "/dst"

export AMQPFIND_ARGS=${AMQPFIND_ARGS:-"-H cspp-geo-rabbit -X satellite -u guest -p guest"}
export AMQPSEND_ARGS=${AMQPSEND_ARGS:-"-H cspp-geo-rabbit -X satellite -u guest -p guest"}
export AMQPFIND_TOPIC=${AMQPFIND_TOPIC:-'data.goes.*.abi.*.l1b.netcdf.all.complete'}
export G2G_ARGS=${G2G_ARGS:-"-r abi_l1b -w geotiff --num-workers 8 --tiled --compress DEFLATE"}
export G2G_PRODUCTS=${G2G_PRODUCTS:-"C01 C02 C03 C04 C05 C06 C07 C08 C09 C10 C11 C12 C13 C14 C15 C16 true_color"}
export ADD_OVERVIEWS=${ADD_OVERVIEWS:-"1"}
export OVERVIEW_RESAMPLE=${OVERVIEW_RESAMPLE:-"average"}
export TMPDIR=${TMPDIR:-"/dst/tmp"}
export RM_ON_FAILURE=${RM_ON_FAILURE:-"1"}

run_geo2grid() {
    if [ $# -ne 5 ]; then
        echo "Unexpected number of arguments (expected 5): $#"
        return 1
    fi

    satellite_family=${1,,}
    satellite_id=${2,,}
    instrument=${3,,}
    data_type=${4,,}
    path="$5"
    echo "Starting Geo2Grid processing for ${path}"
    # convert path from a relative path to an absolute path
    path="/data/${path}"
    # Generate the path where we will save these geotiffs
    # by mimicing the source path
    dst_path=$(dirname "${path}")
    # change the source root /data with /dst
    dst_path=${dst_path/\/data/\/dst}
    mkdir -p "$dst_path"
    # run geo2grid in a temp directory
    mkdir -p "$TMPDIR"
    work_dir=$(mktemp --tmpdir -d "$(date +%Y%m%d_%H%M%S)_XXXXXXXX")
    cd $work_dir
    /usr/bin/time -v geo2grid.sh ${G2G_ARGS} -p ${G2G_PRODUCTS} -f $path
    g2g_status=$?
    if [[ $g2g_status -ne 0 ]]; then
        echo "ERROR: Geo2Grid processing failed"
        if [[ $RM_ON_FAILURE -eq 1 ]]; then
            rm -rf $work_dir
        fi
        return
    fi
    # remove nodata so fill value is not treated as transparency
    if [[ ${G2G_ARGS} =~ fill-value ]]; then
        source /work/geo2grid/bin/activate
        ls -1 *.tif | xargs -n1 -P0 gdal_edit.py -unsetnodata
    fi
    # Create overview (lower resolution) images
    if [[ ${ADD_OVERVIEWS} -eq 1 ]]; then
        echo "Adding overviews..."
        # default is to build overviews down to 256 pixels
        ls -1 *.tif | xargs -n1 -P0 gdaladdo -q -r ${OVERVIEW_RESAMPLE}
    fi
    # get the filename for the C07 geotiff that was created
    # this assumes there is only one timestep in this directory
    glob_pattern=$(echo *C07*.tif)
    # wildcard for all channel files and any other composites
    glob_pattern="${glob_pattern/C07/*}"
    # switch to where we want to actually place these files
    cd "$dst_path"
    # move the files from the temporary directory to the final directory
    echo "Moving files to destination..."
    mv -v ${work_dir}/${glob_pattern} .
    rm -rf $work_dir
    # make it a full path by adding the current directory
    glob_pattern="${PWD}/${glob_pattern}"
    # remove /dst from the path for "normalized" relative paths
    glob_pattern="${glob_pattern/\/dst\//}"
    amqpsend_topic="data.${satellite_family}.${satellite_id}.${instrument}.${data_type}.l1b.geotiff.all.complete"
    json_info="{path: ${glob_pattern}, satellite_family: ${satellite_family}, satellite_ID: ${satellite_id}, instrument: ${instrument}, data_type: ${data_type}}"
    json_info="{\"path\": \"${glob_pattern}\", \"satellite_family\": \"${satellite_family}\", \"satellite_ID\": \"${satellite_id}\", \"instrument\": \"${instrument}\", \"data_type\": \"${data_type}\"}"
    echo -e "[[\"$amqpsend_topic\", $json_info]]" | python /work/amqpfind/amqpsend.py ${AMQPSEND_ARGS}
    echo "AMQP message sent. Processing done."
}
export -f run_geo2grid
echo "Listening to AMQP messages with topic \"$AMQPFIND_TOPIC\""
python amqpfind/amqpfind.py ${AMQPFIND_ARGS} -C "${AMQPFIND_TOPIC}" -j "{satellite_family} {satellite_ID} {instrument} {data_type} \'{path}\'" | xargs -I{} -P3 -n1 bash -c "run_geo2grid {}"