#!/bin/bash -le # Usage: run.sh # Environment variables used for configuration: # AMQPFIND_ARGS: Arguments to pass to amqpfind when listening for new input # events. Should not include the "-C" topic flag (see AMQPFIND_TOPIC). # Default: "-H cspp-geo-rabbit -X satellite -u guest -p guest" # AMQPFIND_TOPIC: Topic to use for incoming data events. # Default: "data.goes.*.abi.*.l1b.netcdf.complete" # The first asterisk (3rd element) can limit processing to a particular # satellite (ex. `g16`). The second asterisk (5th element) can be used # to limit to a particular sector (choices: radf, radc, radm1, radm2) # AMQPSEND_ARGS: Arguments to pass to amqpsend when sending out new data # events. Default: "-H cspp-geo-rabbit -X satellite -u guest -p guest" # Verify that the data mount is available test -d "/data" export AMQPFIND_ARGS=${AMQPFIND_ARGS:-"-H cspp-geo-rabbit -X satellite -u guest -p guest"} export AMQPSEND_ARGS=${AMQPSEND_ARGS:-"-H cspp-geo-rabbit -X satellite -u guest -p guest"} export AMQPFIND_TOPIC=${AMQPFIND_TOPIC:-'data.goes.*.abi.*.l1b.geotiff.complete'} export G2G_PRODUCTS=${G2G_PRODUCTS:-"C01 C02 C03 C04 C05 C06 C07 C08 C09 C10 C11 C12 C13 C14 C15 C16 true_color"} export TILE_ARGS=${TILE_ARGS:-""} export TMPDIR=${TMPDIR:-"/dst/tmp"} run_tile_gen() { if [[ $# -ne 5 ]]; then echo "Unexpected number of arguments (expected 5): $#" return 1 fi satellite_family=${1,,} satellite_id=${2,,} instrument=${3,,} data_type=${4,,} path="$5" echo "Starting Geo2Grid processing for ${path}" # convert path from a relative path to an absolute path path="/data/${path}" # if /dst isn't defined then use /data if [[ -d "/dst" ]]; then dst_dir="/dst" else dst_dir="/data" fi # update shapefile in a temporary directory # and resample geotiff if necessary # FUTURE: TileDB will be updated in-place # generate_tiles.py will make a temporary directory out_dir="${dst_dir}/tiles/${satellite_family}/${satellite_id}/${instrument}" mkdir -p ${out_dir} python3 generate_tiles.py ${TILE_ARGS} -p ${G2G_PRODUCTS} -- ${out_dir} ${path} # OUT/<product>/<product>.shp glob_pattern="${out_dir}/*/*.shp" # Remove the /data prefix glob_pattern="${glob_pattern/${dst_dir}\//}" amqpsend_topic="data.${satellite_family}.${satellite_id}.${instrument}.${data_type}.l1b.tiledb.complete" json_info="{path: ${glob_pattern}, satellite_family: ${satellite_family}, satellite_ID: ${satellite_id}, instrument: ${instrument}, data_type: ${data_type}}" json_info="{\"path\": \"${glob_pattern}\", \"satellite_family\": \"${satellite_family}\", \"satellite_ID\": \"${satellite_id}\", \"instrument\": \"${instrument}\", \"data_type\": \"${data_type}\"}" echo -e "[[\"$amqpsend_topic\", $json_info]]" | python /work/amqpfind/amqpsend.py ${AMQPSEND_ARGS} } export -f run_tile_gen echo "Listening to AMQP messages with topic \"$AMQPFIND_TOPIC\"" python amqpfind/amqpfind.py ${AMQPFIND_ARGS} -C "${AMQPFIND_TOPIC}" -j "{satellite_family} {satellite_ID} {instrument} {data_type} \'{path}\'" | xargs -I{} -P3 -n1 bash -c "run_tile_gen {}"