Commit 5cfb88b6 authored by Nick Bearson's avatar Nick Bearson
Browse files

adding all the new eodn code, removal to come next

parent e73b4a72
Hard-coded paths currently expect that you've checked this out to:
/home/eodn/eodn2realearth
SETUP:
incrontab incrontab.table
./startup.sh
eodn source code in:
/lib/python2.7/site-packages/tools/
# 0 Monitor eodn incoming and trigger ingest
/home/eodn/eodn2realearth/incoming IN_CLOSE_WRITE,IN_MOVED_TO /home/eodn/eodn2realearth/src/ingest.sh $@/$#
#!/usr/bin/env bash
set -u
set -e
INCOMING=/home/eodn/eodn2realearth/incoming
PROCESSING=/home/eodn/eodn2realearth/processing
QUARANTINE=/home/eodn/eodn2realearth/quarantine
filepath=$(readlink -f $1)
filename=$(basename $filepath)
# set $SCRIPT based on input type
if [ ${filename:0:2} == "LC" ] && [ ${filename: -4} == ".zip" ]; then # landsat zip files
SCRIPT="bash -x /home/eodn/eodn2realearth/src/process-LCzip.sh"
else # not recognized
mkdir -p ${QUARANTINE}
mv $filepath ${QUARANTINE}/${filename}
exit 0
fi
mkdir -p ${PROCESSING}/${filename}
cd ${PROCESSING}/${filename}
mv $filepath ${filename}
$SCRIPT ${filename}
if [ $? -eq 0 ]; then
rm -r ${PROCESSING}/${filename}
fi
#!/bin/bash
set -u
set -e
WMSUPLOAD=/home/eodn/eodn2realearth/src/wmsupload
SCENE=$1
YEAR=${SCENE:9:4}
DOY=${SCENE:13:3}
YMD=`date -d "Jan 1 $YEAR + $DOY days - 1 day" "+%Y%m%d"`
PART=${SCENE:2:7}
BASE=${SCENE##*/} # filename w/ extension
STRIP=${BASE%%.zip} # filename w/o extension
TMP=tmp
mkdir -p $TMP
unzip $SCENE -d $TMP
cd $TMP
sed -i s:\&quot\;:\":g *.aux.xml # swap " for actual quotes
for REF in "" _QB _TIR; do
case $REF in
_QB )
IMG=${STRIP}${REF}.png
TAG=QB
;;
_TIR )
IMG=${STRIP}${REF}.jpg
TAG=TIR
;;
"" )
IMG=${STRIP}${REF}.jpg
TAG=FC
;;
esac
if [ ! -f $IMG ]; then # some zips don't have all the image types - skip any that don't exist
continue
fi
grep SRS ${IMG}.aux.xml > WKT # extract the WKT line from XML
sed -i s:\ \ \<SRS\>::g WKT # remove XML tags from WKT
sed -i s:\<\/SRS\>::g WKT
TIF=EODN-${TAG}_${YMD}_120000.part${PART}.tif
gdal_translate -of GTiff -a_srs WKT $IMG $TIF
$WMSUPLOAD $TIF
done
#!/bin/bash
set -u
set -e
SCENE=$1
URL=$2
TMP=$PWD/temp
FILE="scene.zip"
BASE=${SCENE##*/} # filename w/ extension
STRIP=${BASE%%.zip} # filename w/o extension
YEAR=${SCENE:9:4}
DOY=${SCENE:13:3}
YMD=`date -d "Jan 1 $YEAR + $DOY days - 1 day" "+%Y%m%d"`
PART=${SCENE:2:7}
function download() {
URL=$1
FILE=$2
time lors_download -t 10 -b 10m $URL -o $FILE
# for testing...
# cp $URL $FILE
}
function process() {
FILE=$1
mkdir -p $TMP
echo "Unpacking ${SCENE}..."
unzip $FILE -d $TMP
cd $TMP
sed -i s:\&quot\;:\":g *.aux.xml # swap &quot; for actual quotes
for REF in "" _QB _TIR; do
case $REF in
_QB )
IMG=${STRIP}${REF}.png
TAG=QB
;;
_TIR )
IMG=${STRIP}${REF}.jpg
TAG=TIR
;;
"" )
IMG=${STRIP}${REF}.jpg
TAG=FC
;;
esac
if [ ! -f $IMG ]; then # some zips don't have all the image types - skip any that don't exist
continue
fi
grep SRS ${IMG}.aux.xml > WKT # extract the WKT line from XML
sed -i s:\ \ \<SRS\>::g WKT # remove XML tags from WKT
sed -i s:\<\/SRS\>::g WKT
TIF=EODN-${TAG}_${YMD}_120000.part${PART}.tif
gdal_translate -of GTiff -a_srs WKT $IMG $TIF
wmsupload $TIF
done
rm -r $TMP
}
download $URL $FILE
if [ ! -f $FILE ]; then
echo "Could not download $URL"
exit
fi
( process $FILE )
rm $FILE
#!/usr/bin/env bash
echo "
This script should:
+ check that incron is running
+ start eodn_feed
- daemonize it
... anything else? create incoming?
"
#!/bin/sh
################################################################################
# DO NOT ALTER THIS LINE
VERSION=15
################################################################################
SERVER=realearth.ssec.wisc.edu:80
CURL=`which curl 2>/dev/null`
TIMEOUT_BIN=`which timeout 2>/dev/null`
TIMEOUT=
if [ -n "${TIMEOUT_BIN}" ]; then
TIMEOUT="${TIMEOUT_BIN} -s 9 600"
fi
showHelp() {
echo ""
echo "$0 [-hufprv] [-s server:port] file [name] [date] [time]"
echo ""
echo " -h: Show help"
echo " -u: Check for update"
echo ""
echo " -f: Force overwrite if there is a conflict"
echo " -p: Designate file as part of a larger product"
echo ""
echo " -r: Send through head node proxy"
echo " -s: Specify the server and port ([server:port] required)"
echo ""
echo " -v: Be verbose"
echo ""
echo " file: Path to file"
echo " Format: /path/to/[name]_[YYYYMMDD]_[HHMMSS].???"
echo ""
echo " name: Specify the product name"
echo " Required when the file name does not contain [name]"
echo " Format: Cannot contain '_'"
echo " date: Specify the date"
echo " Required when the file name does not contain [date]"
echo " Format: YYYYMMDD"
echo " time: Specify the time"
echo " Required when the file name does not contain [time]"
echo " Format: HHMMSS"
echo ""
}
checkUpdate() {
DL_URL="http://${SERVER}/wmsupload"
URL="$DL_URL?version=${VERSION}"
VERSION_CHECK=`${CURL} -s ${URL}`
if [ -n "${VERSION_CHECK}" ]; then
echo "A new version is available at:"
echo " ${DL_URL}"
echo "Download with:"
echo " ${CURL} ${DL_URL} -o $0"
else
echo "This version is up to date"
fi
}
# Get the options from the command line
WMS_FORCE=0
WMS_PART=0
WMS_PROXY=0
WMS_VERBOSE=0
if [ -z "$1" ]; then
showHelp
exit 0
fi
while getopts hufprvs: o; do
case "$o" in
h) showHelp
exit 0
;;
u) checkUpdate
exit 0
;;
f) WMS_FORCE=1
;;
p) WMS_PART=1
;;
r) WMS_PROXY=1
;;
v) WMS_VERBOSE=1
;;
s) SERVER=${OPTARG}
;;
*) showHelp
exit 1
;;
esac
done
shift $((${OPTIND} -1))
# Set our variables
WMS_FILE=$1
WMS_NAME=$2
WMS_DATE=$3
WMS_TIME=$4
# Does the file exist?
if [ ! -f "${WMS_FILE}" ]; then
echo "ERROR: Could not find file: ${WMS_FILE}"
exit 1
fi
# Set the defaults for sending
WMS_FILE_DIR=`dirname "${WMS_FILE}"`
WMS_FILE_NAME=`basename "${WMS_FILE}"`
# Verify the product name
if [ -n "${WMS_NAME}" ]; then
match=`expr "${WMS_NAME}" : '\([a-zA-Z0-9\-]\{1,\}\)'`
if [ "$match" != "${WMS_NAME}" ]; then
echo ""
echo "ERROR: Invalid product name"
showHelp
exit 1
fi
fi
# Verify the product date
if [ -n "${WMS_DATE}" ]; then
match=`expr "${WMS_DATE}" : '\([0-9]\{8\}\)'`
if [ "$match" != "${WMS_DATE}" ]; then
echo ""
echo "ERROR: Invalid product date"
showHelp
exit 1
fi
fi
# Verify the product time
if [ -n "${WMS_TIME}" ]; then
match=`expr "${WMS_TIME}" : '\([0-9]\{6\}\)'`
if [ "$match" != "${WMS_TIME}" ]; then
echo ""
echo "ERROR: Invalid product time"
showHelp
exit 1
fi
fi
# Get the direct upload name (unless -r was specified)
if [ ${WMS_PROXY} -eq 0 ]; then
SERVER_DIRECT=
if [ -n "${WMS_NAME}" ]; then
SERVER_DIRECT=`${CURL} -s http://${SERVER}/wmsupload?name=${WMS_NAME}`
else
SERVER_DIRECT=`${CURL} -s http://${SERVER}/wmsupload?file=${WMS_FILE_NAME}`
fi
if [ -n "${SERVER_DIRECT}" ]; then
SERVER=${SERVER_DIRECT}
else
if [ ${WMS_VERBOSE} -ne 0 ]; then
echo "WARNING: Could not determine the head node URL for proxy upload"
fi
fi
fi
# Change to the dir with the file
cd "${WMS_FILE_DIR}"
echo "Connecting to ${SERVER}..."
# Check if the server is ready to receive the file
if [ ${WMS_VERBOSE} -ne 0 ]; then
echo "Checking upload availability"
fi
BYTES=`/bin/ls -Lln "${WMS_FILE_NAME}" |awk '{print $5}'`
COMMAND="${CURL} -s http://${SERVER}/wmsupload?bytes=${BYTES}"
SUCCESS=`${COMMAND} -o - 2>/dev/null |head -1`
if [ "${SUCCESS}" -eq "${SUCCESS}" ] 2>/dev/null; then
if [ "${SUCCESS}" = "-1" ]; then
echo " Server cannot accept the file, it is too large!"
exit 3
fi
while [ "${SUCCESS}" != "1" ]; do
echo " Server cannot accept the file at this time, trying again in 15 seconds..."
sleep 15
SUCCESS=`${COMMAND} -o - |head -1`
done
else
if [ ${WMS_VERBOSE} -ne 0 ]; then
echo " Server does not understand file size check, continuing..."
fi
fi
# Send the file
echo "Sending ${WMS_FILE_NAME}"
COMMAND="${TIMEOUT} ${CURL} --max-time 600 -s http://${SERVER}/upload/ -F file=@${WMS_FILE_NAME}"
if [ -n "${WMS_NAME}" ]; then
COMMAND="${COMMAND} -F name=${WMS_NAME}"
fi
if [ -n "${WMS_DATE}" ]; then
COMMAND="${COMMAND} -F date=${WMS_DATE}"
fi
if [ -n "${WMS_TIME}" ]; then
COMMAND="${COMMAND} -F time=${WMS_TIME}"
fi
if [ ${WMS_PART} -ne 0 ]; then
COMMAND="${COMMAND} -F part=1"
fi
if [ ${WMS_FORCE} -ne 0 ]; then
COMMAND="${COMMAND} -F force=1"
fi
# Retry a few times...
RETRIES=3
${COMMAND}
LASTEXIT=$?
while [ ${LASTEXIT} -ne 0 -a ${RETRIES} -gt 0 ]; do
echo "Curl command failed: ${LASTEXIT}"
SLEEP=$((${RANDOM} * 30 / 32767))
echo "Sleeping for ${SLEEP} seconds..."
sleep ${SLEEP}
echo "Trying again..."
${COMMAND}
LASTEXIT=$?
RETRIES=$((${RETRIES} - 1))
done
if [ ${LASTEXIT} -eq 0 ]; then
echo "Done"
else
echo "Giving up"
fi
exit ${LASTEXIT}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment