initializing, please wait...

LCzips/listen.py

+#!/usr/bin/env python
+import websocket
+
+import re, os
+import json
+import argparse
+from subprocess import call
+
+FILERE = re.compile(r'LC[0-9]{14}LGN00.zip')
+BIN = os.path.dirname(os.path.abspath(__file__))
+PROCESS = os.path.join(BIN, "process.sh")
+
+def on_message(ws, message):
+    js = json.loads(message)    
+    print "scene   : %s" % js['properties']['metadata']['scene_id']
+    print "Filename: %s" % js['name']
+    print "Size    : %s" % js['size']
+    print "URL     : %s" % js['selfRef']
+
+    if (FILERE.match(js['name'])):
+        print "\n### Matching filename found, processing...."
+        try:
+            results = call([PROCESS, js['name'], js['selfRef']])
+        except Exception as e:
+            print "ERROR calling process.sh %s" % e
+
+def on_error(ws, error):
+    print(error)
+
+def on_close(ws):
+    print("### closed ###")
+
+def on_open(ws):
+    pass
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(
+        description="Listen for and then process a particular LANDSAT scene")
+    parser.add_argument('-H', '--host', type=str, help='The Exnode service',
+                        default="ws://dev.incntre.iu.edu:8888/subscribe/exnode")
+
+    args = parser.parse_args()
+
+    host = args.host
+
+    ws = websocket.WebSocketApp(host,
+                                on_message = on_message,
+                                on_error = on_error,
+                                on_close = on_close)
+    ws.on_open = on_open
+    ws.run_forever()
+    
+

LCzips/process.sh

+#!/bin/bash
+
+set -u
+set -e
+
+SCENE=$1
+URL=$2
+TMP=$PWD/temp
+FILE="scene.zip"
+
+BASE=${SCENE##*/} # filename w/ extension
+STRIP=${BASE%%.zip} # filename w/o extension
+
+YEAR=${SCENE:9:4}
+DOY=${SCENE:13:3}
+YMD=`date -d "Jan 1 $YEAR + $DOY days - 1 day" "+%Y%m%d"`
+PART=${SCENE:2:7}
+
+function download() {
+  URL=$1
+  FILE=$2
+  time lors_download -t 10 -b 10m $URL -o $FILE
+# for testing...
+#  cp $URL $FILE
+}
+
+function process() {
+  FILE=$1
+  mkdir -p $TMP
+  echo "Unpacking ${SCENE}..."
+  unzip $FILE -d $TMP
+  cd $TMP
+  
+  sed -i s:\&quot\;:\":g *.aux.xml # swap " for actual quotes
+  
+  for REF in "" _QB _TIR; do
+    case $REF in
+      _QB )
+        IMG=${STRIP}${REF}.png
+        TAG=QB
+        ;;
+      _TIR )
+        IMG=${STRIP}${REF}.jpg
+        TAG=TIR
+        ;;
+      "" )
+        IMG=${STRIP}${REF}.jpg
+        TAG=FC
+        ;;
+    esac
+  
+    if [ ! -f $IMG ]; then  # some zips don't have all the image types - skip any that don't exist
+      continue
+    fi
+  
+    grep SRS ${IMG}.aux.xml > WKT # extract the WKT line from XML
+    sed -i s:\ \ \<SRS\>::g WKT # remove XML tags from WKT
+    sed -i s:\<\/SRS\>::g WKT
+  
+    TIF=EODN-${TAG}_${YMD}_120000.part${PART}.tif
+  
+    gdal_translate -of GTiff -a_srs WKT $IMG $TIF
+    wmsupload $TIF
+  done
+  
+  rm -r $TMP
+}
+
+
+download $URL $FILE 
+
+if [ ! -f $FILE ]; then
+    echo "Could not download $URL"
+    exit
+fi
+
+( process $FILE )
+rm $FILE

README

+Requirements in $PATH:
+
++ gdal
++ lors_download (http://loci.cs.utk.edu/lors/tutorial/download.php)
++ wmsupload (http://realearth.ssec.wisc.edu/wmsupload)
+
+Also requires a python 2.7+ install with:
+
++ websocket-client (https://pypi.python.org/pypi/websocket-client)

wmsupload

+#!/bin/sh
+################################################################################
+# DO NOT ALTER THIS LINE
+VERSION=11
+################################################################################
+
+SERVER=realearth.ssec.wisc.edu:80
+
+showHelp() {
+	echo ""
+	echo "$0 [-hufprv] [-s server:port] file [name] [date] [time]"
+	echo ""
+	echo "   -h: Show help"
+	echo "   -u: Check for update"
+	echo ""
+	echo "   -f: Force overwrite if there is a conflict"
+	echo "   -p: Designate file as part of a larger product"
+	echo ""
+	echo "   -r: Send through head node proxy"
+	echo "   -s: Specify the server and port ([server:port] required)"
+	echo ""
+	echo "   -v: Be verbose"
+	echo ""
+	echo " file: Path to file"
+	echo "       Format: /path/to/[name]_[YYYYMMDD]_[HHMMSS].???"
+	echo ""
+	echo " name: Specify the product name"
+	echo "       Required when the file name does not contain [name]"
+	echo "       Format: Cannot contain '_'"
+	echo " date: Specify the date"
+	echo "       Required when the file name does not contain [date]"
+	echo "       Format: YYYYMMDD"
+	echo " time: Specify the time"
+	echo "       Required when the file name does not contain [time]"
+	echo "       Format: HHMMSS"
+	echo ""
+}
+
+checkUpdate() {
+	DL_URL="http://${SERVER}/wmsupload"
+	URL="$DL_URL?version=${VERSION}"
+	VERSION_CHECK=`curl -s ${URL}`
+	if [ -n "${VERSION_CHECK}" ]; then
+		echo "A new version is available at:"
+		echo "    ${DL_URL}"
+		echo "Download with:"
+		echo "    curl ${DL_URL} -o $0"
+	else
+		echo "This version is up to date"
+	fi
+}
+
+# Get the options from the command line
+WMS_FORCE=0
+WMS_PART=0
+WMS_PROXY=0
+WMS_VERBOSE=0
+if [ -z "$1" ]; then
+	showHelp
+	exit 0
+fi
+while getopts hufprvs: o; do
+	case "$o" in
+	h)	showHelp
+		exit 0
+		;;
+	u)	checkUpdate
+		exit 0
+		;;
+	f)	WMS_FORCE=1
+		;;
+	p)	WMS_PART=1
+		;;
+	r)	WMS_PROXY=1
+		;;
+	v)	WMS_VERBOSE=1
+		;;
+	s)	SERVER=${OPTARG}
+		;;
+	*)	showHelp
+		exit 1
+		;;
+	esac
+done
+shift $((${OPTIND} -1))
+
+# Set our variables
+WMS_FILE=$1
+WMS_NAME=$2
+WMS_DATE=$3
+WMS_TIME=$4
+
+# Does the file exist?
+if [ ! -f "${WMS_FILE}" ]; then
+	echo "ERROR: Could not find file: ${WMS_FILE}"
+	exit 1
+fi
+
+# Set the defaults for sending
+WMS_FILE_DIR=`dirname "${WMS_FILE}"`
+WMS_FILE_NAME=`basename "${WMS_FILE}"`
+
+# Verify the product name
+if [ -n "${WMS_NAME}" ]; then
+	match=`expr "${WMS_NAME}" : '\([a-zA-Z0-9\-]\{1,\}\)'`
+	if [ "$match" != "${WMS_NAME}" ]; then
+		echo ""
+		echo "ERROR: Invalid product name"
+		showHelp
+		exit 1
+	fi
+fi
+
+# Verify the product date
+if [ -n "${WMS_DATE}" ]; then
+	match=`expr "${WMS_DATE}" : '\([0-9]\{8\}\)'`
+	if [ "$match" != "${WMS_DATE}" ]; then
+		echo ""
+		echo "ERROR: Invalid product date"
+		showHelp
+		exit 1
+	fi
+fi
+
+# Verify the product time
+if [ -n "${WMS_TIME}" ]; then
+	match=`expr "${WMS_TIME}" : '\([0-9]\{6\}\)'`
+	if [ "$match" != "${WMS_TIME}" ]; then
+		echo ""
+		echo "ERROR: Invalid product time"
+		showHelp
+		exit 1
+	fi
+fi
+
+# Get the direct upload name (unless -r was specified)
+if [ ${WMS_PROXY} -eq 0 ]; then
+	SERVER_DIRECT=
+	if [ -n "${WMS_NAME}" ]; then
+		SERVER_DIRECT=`curl -s http://${SERVER}/wmsupload?name=${WMS_NAME}`
+	else
+		SERVER_DIRECT=`curl -s http://${SERVER}/wmsupload?file=${WMS_FILE_NAME}`
+	fi
+	if [ -n "${SERVER_DIRECT}" ]; then
+		SERVER=${SERVER_DIRECT}
+	else
+		if [ ${WMS_VERBOSE} -ne 0 ]; then
+			echo "WARNING: Could not determine the head node URL for proxy upload"
+		fi
+	fi
+fi
+
+# Change to the dir with the file
+cd "${WMS_FILE_DIR}"
+echo "Connecting to ${SERVER}..."
+
+# Check if the server is ready to receive the file
+if [ ${WMS_VERBOSE} -ne 0 ]; then
+	echo "Checking upload availability"
+fi
+BYTES=`/bin/ls -Ll "${WMS_FILE_NAME}" |/bin/awk '{print $5}'`
+COMMAND="curl -s http://${SERVER}/wmsupload?bytes=${BYTES}"
+SUCCESS=`${COMMAND} -o - 2>/dev/null |head -1`
+if [ "${SUCCESS}" -eq "${SUCCESS}" ] 2>/dev/null; then
+	if [ "${SUCCESS}" = "-1" ]; then
+		echo "  Server cannot accept the file, it is too large!"
+		exit 3
+	fi
+	while [ "${SUCCESS}" != "1" ]; do
+		echo "  Server cannot accept the file at this time, trying again in 15 seconds..."
+		sleep 15
+		SUCCESS=`${COMMAND} -o - |head -1`
+	done
+else
+	if [ ${WMS_VERBOSE} -ne 0 ]; then
+		echo "  Server does not understand file size check, continuing..."
+	fi
+fi
+
+# Send the file
+echo "Sending ${WMS_FILE_NAME}"
+COMMAND="curl -s http://${SERVER}/upload/ -F file=@${WMS_FILE_NAME}"
+if [ -n "${WMS_NAME}" ]; then
+	COMMAND="${COMMAND} -F name=${WMS_NAME}"
+fi
+if [ -n "${WMS_DATE}" ]; then
+	COMMAND="${COMMAND} -F date=${WMS_DATE}"
+fi
+if [ -n "${WMS_TIME}" ]; then
+	COMMAND="${COMMAND} -F time=${WMS_TIME}"
+fi
+if [ ${WMS_PART} -ne 0 ]; then
+	COMMAND="${COMMAND} -F part=1"
+fi
+if [ ${WMS_FORCE} -ne 0 ]; then
+	COMMAND="${COMMAND} -F force=1"
+fi
+${COMMAND}