Commit 66675347 authored by Nick Bearson's avatar Nick Bearson
Browse files

bunch of logging changes & hopefully a fix for moving non-recognized files to quarantine still

parent 6264def4
...@@ -9,3 +9,9 @@ readonly LOGS=${EODN2REALEARTH}/logs ...@@ -9,3 +9,9 @@ readonly LOGS=${EODN2REALEARTH}/logs
readonly INCOMING=${EODN2REALEARTH}/incoming readonly INCOMING=${EODN2REALEARTH}/incoming
readonly PROCESSING=${EODN2REALEARTH}/processing readonly PROCESSING=${EODN2REALEARTH}/processing
readonly QUARANTINE=${EODN2REALEARTH}/quarantine readonly QUARANTINE=${EODN2REALEARTH}/quarantine
# general logging function for everything to use:
readonly TIMESTAMP=$(date +"[%Y-%m-%d %H:%M:%S]:")
log() {
echo "${TIMESTAMP} ${@}"
}
...@@ -15,18 +15,20 @@ exec >> $LOG 2>&1 ...@@ -15,18 +15,20 @@ exec >> $LOG 2>&1
filepath=$(readlink -f $1) filepath=$(readlink -f $1)
filename=$(basename $filepath) filename=$(basename $filepath)
recognized="yes"
# set $SCRIPT based on input type # set $SCRIPT based on input type
if [ ${filename:0:2} == "LC" ] && [ ${filename: -4} == ".zip" ]; then # landsat zip files if [ ${filename:0:2} == "LC" ] && [ ${filename: -4} == ".zip" ]; then # landsat zip files
echo "SAW LANDSAT ZIP: ${filename}" log "SAW LANDSAT ZIP: ${filename}"
SCRIPT="${SRC}/process-LCzip.sh" SCRIPT="${SRC}/process-LCzip.sh"
LOG_TO="${LOGS}/process-LCzip.log" LOG_TO="${LOGS}/process-LCzip.log"
elif [ ${filename:0:2} == "LC" ] && [ ${filename: -7} == ".tar.gz" ]; then # landsat zip files elif [ ${filename:0:2} == "LC" ] && [ ${filename: -7} == ".tar.gz" ]; then # landsat zip files
echo "SAW LANDSAT TAR.GZ: ${filename}" log "SAW LANDSAT TAR.GZ: ${filename}"
SCRIPT="${SRC}/process-LCtgz.sh" SCRIPT="${SRC}/process-LCtgz.sh"
LOG_TO="${LOGS}/process-LCtgz.log" LOG_TO="${LOGS}/process-LCtgz.log"
else # not recognized else # not recognized
echo "SAW (NOT RECOGNIZED): ${filename}" log "SAW (NOT RECOGNIZED): ${filename}"
exit 0 recognized="idk"
fi fi
# just quarantine everything for now in case we want it later # just quarantine everything for now in case we want it later
...@@ -34,6 +36,10 @@ fi ...@@ -34,6 +36,10 @@ fi
mkdir -p ${QUARANTINE} mkdir -p ${QUARANTINE}
mv -f $filepath ${QUARANTINE}/${filename} # overwrite so we don't crash if reprocessing mv -f $filepath ${QUARANTINE}/${filename} # overwrite so we don't crash if reprocessing
if [ "${recognized}" == "idk" ]; then
exit 0
fi
# set up processing directory # set up processing directory
mkdir -p ${PROCESSING}/${filename} mkdir -p ${PROCESSING}/${filename}
cd ${PROCESSING}/${filename} cd ${PROCESSING}/${filename}
...@@ -44,5 +50,5 @@ $SCRIPT ${PROCESSING}/${filename}/${filename} &>> ${LOG_TO} ...@@ -44,5 +50,5 @@ $SCRIPT ${PROCESSING}/${filename}/${filename} &>> ${LOG_TO}
if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
rm -r ${PROCESSING}/${filename} rm -r ${PROCESSING}/${filename}
else else
echo "ERROR [$?] while processing ${PROCESSING}/${filename}" # we shouldn't hit this due to `set -e` log "ERROR [$?] while processing ${PROCESSING}/${filename}" # we shouldn't hit this due to `set -e`
fi fi
...@@ -10,7 +10,8 @@ ...@@ -10,7 +10,8 @@
set -u set -u
set -e set -e
BANDS=(1 2 3 4 5 6 7 8 9 10 11) #BANDS=(1 2 3 4 5 6 7 8 9 10 11)
BANDS=(1 2 3 4) # nadir seems to be struggling, try just a few bands?
DIR="${BASH_SOURCE%/*}" DIR="${BASH_SOURCE%/*}"
if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
...@@ -19,9 +20,9 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi ...@@ -19,9 +20,9 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
WMSUPLOAD=${DIR}/wmsupload WMSUPLOAD=${DIR}/wmsupload
SCENE=$1 SCENE=$1
OPFLAG=$2 OPFLAG=${2:-""}
echo "PROCESSING: $1" log "PROCESSING: $1"
BASE=${SCENE##*/} # filename w/ extension BASE=${SCENE##*/} # filename w/ extension
STRIP=${BASE%%.tar.gz} # filename w/o extension STRIP=${BASE%%.tar.gz} # filename w/o extension
...@@ -34,10 +35,10 @@ PART=${STRIP:2:7} ...@@ -34,10 +35,10 @@ PART=${STRIP:2:7}
for band in ${BANDS[@]}; do for band in ${BANDS[@]}; do
MTL="${STRIP}_MTL.txt" MTL="${STRIP}_MTL.txt"
BN="${STRIP}_B${band}.TIF" BN="${STRIP}_B${band}.TIF"
echo "extracting band ${band}" log "extracting band ${band}"
tar xzf $SCENE $BN $MTL tar xzf $SCENE $BN $MTL
echo "calculating TOA radiance" log "calculating TOA radiance"
vmul=$(grep "RADIANCE_MULT_BAND_${band} =" $MTL | cut -d " " -f 7) vmul=$(grep "RADIANCE_MULT_BAND_${band} =" $MTL | cut -d " " -f 7)
vadd=$(grep "RADIANCE_ADD_BAND_${band} =" $MTL | cut -d " " -f 7) vadd=$(grep "RADIANCE_ADD_BAND_${band} =" $MTL | cut -d " " -f 7)
...@@ -47,7 +48,7 @@ for band in ${BANDS[@]}; do ...@@ -47,7 +48,7 @@ for band in ${BANDS[@]}; do
gdal_calc.py -A $BN --outfile=${product} --calc="(A*${vmul})+${vadd}" gdal_calc.py -A $BN --outfile=${product} --calc="(A*${vmul})+${vadd}"
if [ "${OPFLAG}" == "" ]; then if [ "${OPFLAG}" == "" ]; then
echo "uploading ${product}" log "uploading ${product}"
$WMSUPLOAD ${product} $WMSUPLOAD ${product}
fi fi
done done
...@@ -10,7 +10,7 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi ...@@ -10,7 +10,7 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
WMSUPLOAD=${DIR}/wmsupload WMSUPLOAD=${DIR}/wmsupload
SCENE=$1 SCENE=$1
echo "PROCESSING: $1" log "PROCESSING: $1"
YEAR=${SCENE:9:4} YEAR=${SCENE:9:4}
DOY=${SCENE:13:3} DOY=${SCENE:13:3}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment