run.sh

#!/bin/bash -le
# Usage: run.sh
# Environment variables used for configuration:
# AMQPFIND_ARGS: Arguments to pass to amqpfind when listening for new input
#     events. Should not include the "-C" topic flag (see AMQPFIND_TOPIC).
#     Default: "-H cspp-geo-rabbit -X satellite -u guest -p guest"
# AMQPFIND_TOPIC: Topic to use for incoming data events.
#     Default: "data.goes.*.abi.*.l1b.netcdf.complete"
#     The first asterisk (3rd element) can limit processing to a particular
#     satellite (ex. `g16`). The second asterisk (5th element) can be used
#     to limit to a particular sector (choices: radf, radc, radm1, radm2)
# AMQPSEND_ARGS: Arguments to pass to amqpsend when sending out new data
#     events. Default: "-H cspp-geo-rabbit -X satellite -u guest -p guest"

# Verify that the data mount is available
test -d "/data"

export AMQPFIND_ARGS=${AMQPFIND_ARGS:-"-H cspp-geo-rabbit -X satellite -u guest -p guest"}
export AMQPSEND_ARGS=${AMQPSEND_ARGS:-"-D 15 -H cspp-geo-rabbit -X satellite -u guest -p guest"}
export AMQPFIND_TOPIC=${AMQPFIND_TOPIC:-'data.goes.*.abi.*.l1b.geotiff.all.complete'}
export G2G_PRODUCTS=${G2G_PRODUCTS:-"C01 C02 C03 C04 C05 C06 C07 C08 C09 C10 C11 C12 C13 C14 C15 C16 true_color"}
export TILE_ARGS=${TILE_ARGS:-""}
export TMPDIR=${TMPDIR:-"/dst/tmp"}

run_tile_gen() {
    if [[ $# -ne 5 ]]; then
        echo "Unexpected number of arguments (expected 5): $#"
        return 1
    fi

    satellite_family=${1,,}
    satellite_id=${2,,}
    instrument=${3,,}
    data_type=${4,,}
    path="$5"
    echo "Starting tile generation processing for ${path}"
    # convert path from a relative path to an absolute path
    path="/data/${path}"

    # if /dst isn't defined then use /data
    if [[ -d "/dst" ]]; then
        dst_dir="/dst"
    else
        dst_dir="/data"
    fi

    # update shapefile in a temporary directory
    # and resample geotiff if necessary
    # FUTURE: TileDB will be updated in-place
    # generate_tiles.py will make a temporary directory
    out_dir="${dst_dir}/tiles/${satellite_id}/${instrument}/${data_type}"
    # if we are going to remap, separate the geotiffs and shapefile from unremapped
    if [[ ${TILE_ARGS} == *"--remap"* ]]; then
        echo "Adding '_ll' prefix to tile output directory because of remapping"
        out_dir="${out_dir}_ll"
        mkdir -p ${out_dir}
        # add string formatting portion to separate add 'product' sub-directory
        out_dir="${out_dir}/{product}_ll"
    else
        mkdir -p ${out_dir}
        # add string formatting portion to separate add 'product' sub-directory
        out_dir="${out_dir}/{product}"
    fi
    # Add optimization to only compute the bboxonce
    if ([ "$data_type" == "radf" ] || [ "$data_type" == "radc" ]) && [ $TILE_ARGS =~ "shared" ]; then
        TILE_ARGS="${TILE_ARGS} --shared-bbox"
    fi
    echo "Generating tiles in directory: ${out_dir}"
    for product in ${G2G_PRODUCTS}; do
        json_times_changed=$(python3 generate_tiles.py ${TILE_ARGS} --json-times -p ${product} -- ${out_dir} "${path}")
        # OUT/<product>/<product>.shp
        glob_pattern="${out_dir/\{product\}/${product}}/*.shp"
        # Remove the /data prefix
        glob_pattern="${glob_pattern/${dst_dir}\//}"

        amqpsend_topic="data.${satellite_family}.${satellite_id}.${instrument}.${data_type}.l1b.tiles.${product,,}.complete"
        json_info="{\"path\": \"${glob_pattern}\", \"satellite_family\": \"${satellite_family}\", \"satellite_ID\": \"${satellite_id}\", \"instrument\": \"${instrument}\", \"data_type\": \"${data_type}\", \"product\": \"${product}\"}"
        # append the JSON returned by the python (remove the curly braces at the ends)
        json_info="${json_info:0:-1}, ${json_times_changed:1}"
        echo -e "[[\"$amqpsend_topic\", $json_info]]"
        echo -e "[[\"$amqpsend_topic\", $json_info]]" | python3 /work/amqpfind/amqpsend.py ${AMQPSEND_ARGS}
    done
    echo "Done generating tiles for ${path}"
}

export -f run_tile_gen
echo "Listening to AMQP messages with topic \"$AMQPFIND_TOPIC\""
python3 amqpfind/amqpfind.py ${AMQPFIND_ARGS} -C "${AMQPFIND_TOPIC}" -j "{satellite_family} {satellite_ID} {instrument} {data_type} \'{path}\'" | xargs -I{} -P4 -n1 bash -c "run_tile_gen {}"