Commit ba671cb5 authored by Nick Bearson's avatar Nick Bearson
Browse files

removing old eodn stuff

parent 5cfb88b6
#!/usr/bin/env python
import websocket
import re, os
import json
import argparse
from subprocess import call
FILERE = re.compile(r'LC[0-9]{14}LGN00.zip')
BIN = os.path.dirname(os.path.abspath(__file__))
PROCESS = os.path.join(BIN, "process.sh")
def on_message(ws, message):
js = json.loads(message)
print "scene : %s" % js['properties']['metadata']['scene_id']
print "Filename: %s" % js['name']
print "Size : %s" % js['size']
print "URL : %s" % js['selfRef']
if (FILERE.match(js['name'])):
print "\n### Matching filename found, processing...."
try:
results = call([PROCESS, js['name'], js['selfRef']])
except Exception as e:
print "ERROR calling process.sh %s" % e
def on_error(ws, error):
print(error)
def on_close(ws):
print("### closed ###")
def on_open(ws):
pass
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Listen for and then process a particular LANDSAT scene")
parser.add_argument('-H', '--host', type=str, help='The Exnode service',
default="ws://dev.incntre.iu.edu:8888/subscribe/exnode")
args = parser.parse_args()
host = args.host
ws = websocket.WebSocketApp(host,
on_message = on_message,
on_error = on_error,
on_close = on_close)
ws.on_open = on_open
ws.run_forever()
#!/bin/bash
set -u
set -e
SCENE=$1
URL=$2
TMP=$PWD/temp
FILE="scene.zip"
BASE=${SCENE##*/} # filename w/ extension
STRIP=${BASE%%.zip} # filename w/o extension
YEAR=${SCENE:9:4}
DOY=${SCENE:13:3}
YMD=`date -d "Jan 1 $YEAR + $DOY days - 1 day" "+%Y%m%d"`
PART=${SCENE:2:7}
function download() {
URL=$1
FILE=$2
time lors_download -t 10 -b 10m $URL -o $FILE
# for testing...
# cp $URL $FILE
}
function process() {
FILE=$1
mkdir -p $TMP
echo "Unpacking ${SCENE}..."
unzip $FILE -d $TMP
cd $TMP
sed -i s:\&quot\;:\":g *.aux.xml # swap " for actual quotes
for REF in "" _QB _TIR; do
case $REF in
_QB )
IMG=${STRIP}${REF}.png
TAG=QB
;;
_TIR )
IMG=${STRIP}${REF}.jpg
TAG=TIR
;;
"" )
IMG=${STRIP}${REF}.jpg
TAG=FC
;;
esac
if [ ! -f $IMG ]; then # some zips don't have all the image types - skip any that don't exist
continue
fi
grep SRS ${IMG}.aux.xml > WKT # extract the WKT line from XML
sed -i s:\ \ \<SRS\>::g WKT # remove XML tags from WKT
sed -i s:\<\/SRS\>::g WKT
TIF=EODN-${TAG}_${YMD}_120000.part${PART}.tif
gdal_translate -of GTiff -a_srs WKT $IMG $TIF
wmsupload $TIF
done
rm -r $TMP
}
download $URL $FILE
if [ ! -f $FILE ]; then
echo "Could not download $URL"
exit
fi
( process $FILE )
rm $FILE
Requirements in $PATH:
+ gdal
+ lors_download (http://loci.cs.utk.edu/lors/tutorial/download.php)
+ wmsupload (http://realearth.ssec.wisc.edu/wmsupload)
Also requires a python 2.7+ install with:
+ websocket-client (https://pypi.python.org/pypi/websocket-client)
eodn source code in:
/lib/python2.7/site-packages/tools/
#!/bin/bash
set -u
set -e
SCENE=$1
URL=$2
TMP=$PWD/temp
FILE="scene.zip"
BASE=${SCENE##*/} # filename w/ extension
STRIP=${BASE%%.zip} # filename w/o extension
YEAR=${SCENE:9:4}
DOY=${SCENE:13:3}
YMD=`date -d "Jan 1 $YEAR + $DOY days - 1 day" "+%Y%m%d"`
PART=${SCENE:2:7}
function download() {
URL=$1
FILE=$2
time lors_download -t 10 -b 10m $URL -o $FILE
# for testing...
# cp $URL $FILE
}
function process() {
FILE=$1
mkdir -p $TMP
echo "Unpacking ${SCENE}..."
unzip $FILE -d $TMP
cd $TMP
sed -i s:\&quot\;:\":g *.aux.xml # swap &quot; for actual quotes
for REF in "" _QB _TIR; do
case $REF in
_QB )
IMG=${STRIP}${REF}.png
TAG=QB
;;
_TIR )
IMG=${STRIP}${REF}.jpg
TAG=TIR
;;
"" )
IMG=${STRIP}${REF}.jpg
TAG=FC
;;
esac
if [ ! -f $IMG ]; then # some zips don't have all the image types - skip any that don't exist
continue
fi
grep SRS ${IMG}.aux.xml > WKT # extract the WKT line from XML
sed -i s:\ \ \<SRS\>::g WKT # remove XML tags from WKT
sed -i s:\<\/SRS\>::g WKT
TIF=EODN-${TAG}_${YMD}_120000.part${PART}.tif
gdal_translate -of GTiff -a_srs WKT $IMG $TIF
wmsupload $TIF
done
rm -r $TMP
}
download $URL $FILE
if [ ! -f $FILE ]; then
echo "Could not download $URL"
exit
fi
( process $FILE )
rm $FILE
#!/bin/sh
################################################################################
# DO NOT ALTER THIS LINE
VERSION=11
################################################################################
SERVER=realearth.ssec.wisc.edu:80
showHelp() {
echo ""
echo "$0 [-hufprv] [-s server:port] file [name] [date] [time]"
echo ""
echo " -h: Show help"
echo " -u: Check for update"
echo ""
echo " -f: Force overwrite if there is a conflict"
echo " -p: Designate file as part of a larger product"
echo ""
echo " -r: Send through head node proxy"
echo " -s: Specify the server and port ([server:port] required)"
echo ""
echo " -v: Be verbose"
echo ""
echo " file: Path to file"
echo " Format: /path/to/[name]_[YYYYMMDD]_[HHMMSS].???"
echo ""
echo " name: Specify the product name"
echo " Required when the file name does not contain [name]"
echo " Format: Cannot contain '_'"
echo " date: Specify the date"
echo " Required when the file name does not contain [date]"
echo " Format: YYYYMMDD"
echo " time: Specify the time"
echo " Required when the file name does not contain [time]"
echo " Format: HHMMSS"
echo ""
}
checkUpdate() {
DL_URL="http://${SERVER}/wmsupload"
URL="$DL_URL?version=${VERSION}"
VERSION_CHECK=`curl -s ${URL}`
if [ -n "${VERSION_CHECK}" ]; then
echo "A new version is available at:"
echo " ${DL_URL}"
echo "Download with:"
echo " curl ${DL_URL} -o $0"
else
echo "This version is up to date"
fi
}
# Get the options from the command line
WMS_FORCE=0
WMS_PART=0
WMS_PROXY=0
WMS_VERBOSE=0
if [ -z "$1" ]; then
showHelp
exit 0
fi
while getopts hufprvs: o; do
case "$o" in
h) showHelp
exit 0
;;
u) checkUpdate
exit 0
;;
f) WMS_FORCE=1
;;
p) WMS_PART=1
;;
r) WMS_PROXY=1
;;
v) WMS_VERBOSE=1
;;
s) SERVER=${OPTARG}
;;
*) showHelp
exit 1
;;
esac
done
shift $((${OPTIND} -1))
# Set our variables
WMS_FILE=$1
WMS_NAME=$2
WMS_DATE=$3
WMS_TIME=$4
# Does the file exist?
if [ ! -f "${WMS_FILE}" ]; then
echo "ERROR: Could not find file: ${WMS_FILE}"
exit 1
fi
# Set the defaults for sending
WMS_FILE_DIR=`dirname "${WMS_FILE}"`
WMS_FILE_NAME=`basename "${WMS_FILE}"`
# Verify the product name
if [ -n "${WMS_NAME}" ]; then
match=`expr "${WMS_NAME}" : '\([a-zA-Z0-9\-]\{1,\}\)'`
if [ "$match" != "${WMS_NAME}" ]; then
echo ""
echo "ERROR: Invalid product name"
showHelp
exit 1
fi
fi
# Verify the product date
if [ -n "${WMS_DATE}" ]; then
match=`expr "${WMS_DATE}" : '\([0-9]\{8\}\)'`
if [ "$match" != "${WMS_DATE}" ]; then
echo ""
echo "ERROR: Invalid product date"
showHelp
exit 1
fi
fi
# Verify the product time
if [ -n "${WMS_TIME}" ]; then
match=`expr "${WMS_TIME}" : '\([0-9]\{6\}\)'`
if [ "$match" != "${WMS_TIME}" ]; then
echo ""
echo "ERROR: Invalid product time"
showHelp
exit 1
fi
fi
# Get the direct upload name (unless -r was specified)
if [ ${WMS_PROXY} -eq 0 ]; then
SERVER_DIRECT=
if [ -n "${WMS_NAME}" ]; then
SERVER_DIRECT=`curl -s http://${SERVER}/wmsupload?name=${WMS_NAME}`
else
SERVER_DIRECT=`curl -s http://${SERVER}/wmsupload?file=${WMS_FILE_NAME}`
fi
if [ -n "${SERVER_DIRECT}" ]; then
SERVER=${SERVER_DIRECT}
else
if [ ${WMS_VERBOSE} -ne 0 ]; then
echo "WARNING: Could not determine the head node URL for proxy upload"
fi
fi
fi
# Change to the dir with the file
cd "${WMS_FILE_DIR}"
echo "Connecting to ${SERVER}..."
# Check if the server is ready to receive the file
if [ ${WMS_VERBOSE} -ne 0 ]; then
echo "Checking upload availability"
fi
BYTES=`/bin/ls -Ll "${WMS_FILE_NAME}" |/bin/awk '{print $5}'`
COMMAND="curl -s http://${SERVER}/wmsupload?bytes=${BYTES}"
SUCCESS=`${COMMAND} -o - 2>/dev/null |head -1`
if [ "${SUCCESS}" -eq "${SUCCESS}" ] 2>/dev/null; then
if [ "${SUCCESS}" = "-1" ]; then
echo " Server cannot accept the file, it is too large!"
exit 3
fi
while [ "${SUCCESS}" != "1" ]; do
echo " Server cannot accept the file at this time, trying again in 15 seconds..."
sleep 15
SUCCESS=`${COMMAND} -o - |head -1`
done
else
if [ ${WMS_VERBOSE} -ne 0 ]; then
echo " Server does not understand file size check, continuing..."
fi
fi
# Send the file
echo "Sending ${WMS_FILE_NAME}"
COMMAND="curl -s http://${SERVER}/upload/ -F file=@${WMS_FILE_NAME}"
if [ -n "${WMS_NAME}" ]; then
COMMAND="${COMMAND} -F name=${WMS_NAME}"
fi
if [ -n "${WMS_DATE}" ]; then
COMMAND="${COMMAND} -F date=${WMS_DATE}"
fi
if [ -n "${WMS_TIME}" ]; then
COMMAND="${COMMAND} -F time=${WMS_TIME}"
fi
if [ ${WMS_PART} -ne 0 ]; then
COMMAND="${COMMAND} -F part=1"
fi
if [ ${WMS_FORCE} -ne 0 ]; then
COMMAND="${COMMAND} -F force=1"
fi
${COMMAND}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment