adding all the new eodn code, removal to come next

src/README

+Hard-coded paths currently expect that you've checked this out to:
+
+  /home/eodn/eodn2realearth
+
+SETUP:
+
+incrontab incrontab.table
+./startup.sh

src/SOURCE

+eodn source code in:
+/lib/python2.7/site-packages/tools/

src/incrontab.table

+# 0 Monitor eodn incoming and trigger ingest
+/home/eodn/eodn2realearth/incoming IN_CLOSE_WRITE,IN_MOVED_TO /home/eodn/eodn2realearth/src/ingest.sh $@/$#
+

src/ingest.sh

+#!/usr/bin/env bash
+
+set -u
+set -e
+
+INCOMING=/home/eodn/eodn2realearth/incoming
+PROCESSING=/home/eodn/eodn2realearth/processing
+QUARANTINE=/home/eodn/eodn2realearth/quarantine
+
+filepath=$(readlink -f $1)
+filename=$(basename $filepath)
+
+# set $SCRIPT based on input type
+if [ ${filename:0:2} == "LC" ] && [ ${filename: -4} == ".zip" ]; then  # landsat zip files
+  SCRIPT="bash -x /home/eodn/eodn2realearth/src/process-LCzip.sh"
+else # not recognized
+  mkdir -p ${QUARANTINE}
+  mv $filepath ${QUARANTINE}/${filename}
+  exit 0
+fi
+
+mkdir -p ${PROCESSING}/${filename}
+cd ${PROCESSING}/${filename}
+mv $filepath ${filename}
+$SCRIPT ${filename} 
+if [ $? -eq 0 ]; then
+  rm -r ${PROCESSING}/${filename}
+fi

src/process-LCzip.sh

+#!/bin/bash
+
+set -u
+set -e
+
+WMSUPLOAD=/home/eodn/eodn2realearth/src/wmsupload
+SCENE=$1
+
+YEAR=${SCENE:9:4}
+DOY=${SCENE:13:3}
+YMD=`date -d "Jan 1 $YEAR + $DOY days - 1 day" "+%Y%m%d"`
+PART=${SCENE:2:7}
+
+BASE=${SCENE##*/} # filename w/ extension
+STRIP=${BASE%%.zip} # filename w/o extension
+
+TMP=tmp
+mkdir -p $TMP
+unzip $SCENE -d $TMP
+cd $TMP
+
+sed -i s:\&quot\;:\":g *.aux.xml # swap " for actual quotes
+
+for REF in "" _QB _TIR; do
+  case $REF in
+    _QB )
+      IMG=${STRIP}${REF}.png
+      TAG=QB
+      ;;
+    _TIR )
+      IMG=${STRIP}${REF}.jpg
+      TAG=TIR
+      ;;
+    "" )
+      IMG=${STRIP}${REF}.jpg
+      TAG=FC
+      ;;
+  esac
+
+  if [ ! -f $IMG ]; then  # some zips don't have all the image types - skip any that don't exist
+    continue
+  fi
+
+  grep SRS ${IMG}.aux.xml > WKT # extract the WKT line from XML
+  sed -i s:\ \ \<SRS\>::g WKT # remove XML tags from WKT
+  sed -i s:\<\/SRS\>::g WKT
+
+  TIF=EODN-${TAG}_${YMD}_120000.part${PART}.tif
+
+  gdal_translate -of GTiff -a_srs WKT $IMG $TIF
+  $WMSUPLOAD $TIF
+done

src/process.sh

+#!/bin/bash
+
+set -u
+set -e
+
+SCENE=$1
+URL=$2
+TMP=$PWD/temp
+FILE="scene.zip"
+
+BASE=${SCENE##*/} # filename w/ extension
+STRIP=${BASE%%.zip} # filename w/o extension
+
+YEAR=${SCENE:9:4}
+DOY=${SCENE:13:3}
+YMD=`date -d "Jan 1 $YEAR + $DOY days - 1 day" "+%Y%m%d"`
+PART=${SCENE:2:7}
+
+function download() {
+  URL=$1
+  FILE=$2
+  time lors_download -t 10 -b 10m $URL -o $FILE
+# for testing...
+#  cp $URL $FILE
+}
+
+function process() {
+  FILE=$1
+  mkdir -p $TMP
+  echo "Unpacking ${SCENE}..."
+  unzip $FILE -d $TMP
+  cd $TMP
+  
+  sed -i s:\&quot\;:\":g *.aux.xml # swap " for actual quotes
+  
+  for REF in "" _QB _TIR; do
+    case $REF in
+      _QB )
+        IMG=${STRIP}${REF}.png
+        TAG=QB
+        ;;
+      _TIR )
+        IMG=${STRIP}${REF}.jpg
+        TAG=TIR
+        ;;
+      "" )
+        IMG=${STRIP}${REF}.jpg
+        TAG=FC
+        ;;
+    esac
+  
+    if [ ! -f $IMG ]; then  # some zips don't have all the image types - skip any that don't exist
+      continue
+    fi
+  
+    grep SRS ${IMG}.aux.xml > WKT # extract the WKT line from XML
+    sed -i s:\ \ \<SRS\>::g WKT # remove XML tags from WKT
+    sed -i s:\<\/SRS\>::g WKT
+  
+    TIF=EODN-${TAG}_${YMD}_120000.part${PART}.tif
+  
+    gdal_translate -of GTiff -a_srs WKT $IMG $TIF
+    wmsupload $TIF
+  done
+  
+  rm -r $TMP
+}
+
+
+download $URL $FILE 
+
+if [ ! -f $FILE ]; then
+    echo "Could not download $URL"
+    exit
+fi
+
+( process $FILE )
+rm $FILE

src/startup.sh

+#!/usr/bin/env bash
+
+echo "
+This script should:
+ + check that incron is running
+ + start eodn_feed
+   - daemonize it
+
+ ... anything else? create incoming?
+"

src/wmsupload

+#!/bin/sh
+################################################################################
+# DO NOT ALTER THIS LINE
+VERSION=15
+################################################################################
+
+SERVER=realearth.ssec.wisc.edu:80
+
+CURL=`which curl 2>/dev/null`
+TIMEOUT_BIN=`which timeout 2>/dev/null`
+TIMEOUT=
+if [ -n "${TIMEOUT_BIN}" ]; then
+	TIMEOUT="${TIMEOUT_BIN} -s 9 600"
+fi
+
+showHelp() {
+	echo ""
+	echo "$0 [-hufprv] [-s server:port] file [name] [date] [time]"
+	echo ""
+	echo "   -h: Show help"
+	echo "   -u: Check for update"
+	echo ""
+	echo "   -f: Force overwrite if there is a conflict"
+	echo "   -p: Designate file as part of a larger product"
+	echo ""
+	echo "   -r: Send through head node proxy"
+	echo "   -s: Specify the server and port ([server:port] required)"
+	echo ""
+	echo "   -v: Be verbose"
+	echo ""
+	echo " file: Path to file"
+	echo "       Format: /path/to/[name]_[YYYYMMDD]_[HHMMSS].???"
+	echo ""
+	echo " name: Specify the product name"
+	echo "       Required when the file name does not contain [name]"
+	echo "       Format: Cannot contain '_'"
+	echo " date: Specify the date"
+	echo "       Required when the file name does not contain [date]"
+	echo "       Format: YYYYMMDD"
+	echo " time: Specify the time"
+	echo "       Required when the file name does not contain [time]"
+	echo "       Format: HHMMSS"
+	echo ""
+}
+
+checkUpdate() {
+	DL_URL="http://${SERVER}/wmsupload"
+	URL="$DL_URL?version=${VERSION}"
+	VERSION_CHECK=`${CURL} -s ${URL}`
+	if [ -n "${VERSION_CHECK}" ]; then
+		echo "A new version is available at:"
+		echo "    ${DL_URL}"
+		echo "Download with:"
+		echo "    ${CURL} ${DL_URL} -o $0"
+	else
+		echo "This version is up to date"
+	fi
+}
+
+# Get the options from the command line
+WMS_FORCE=0
+WMS_PART=0
+WMS_PROXY=0
+WMS_VERBOSE=0
+if [ -z "$1" ]; then
+	showHelp
+	exit 0
+fi
+while getopts hufprvs: o; do
+	case "$o" in
+	h)	showHelp
+		exit 0
+		;;
+	u)	checkUpdate
+		exit 0
+		;;
+	f)	WMS_FORCE=1
+		;;
+	p)	WMS_PART=1
+		;;
+	r)	WMS_PROXY=1
+		;;
+	v)	WMS_VERBOSE=1
+		;;
+	s)	SERVER=${OPTARG}
+		;;
+	*)	showHelp
+		exit 1
+		;;
+	esac
+done
+shift $((${OPTIND} -1))
+
+# Set our variables
+WMS_FILE=$1
+WMS_NAME=$2
+WMS_DATE=$3
+WMS_TIME=$4
+
+# Does the file exist?
+if [ ! -f "${WMS_FILE}" ]; then
+	echo "ERROR: Could not find file: ${WMS_FILE}"
+	exit 1
+fi
+
+# Set the defaults for sending
+WMS_FILE_DIR=`dirname "${WMS_FILE}"`
+WMS_FILE_NAME=`basename "${WMS_FILE}"`
+
+# Verify the product name
+if [ -n "${WMS_NAME}" ]; then
+	match=`expr "${WMS_NAME}" : '\([a-zA-Z0-9\-]\{1,\}\)'`
+	if [ "$match" != "${WMS_NAME}" ]; then
+		echo ""
+		echo "ERROR: Invalid product name"
+		showHelp
+		exit 1
+	fi
+fi
+
+# Verify the product date
+if [ -n "${WMS_DATE}" ]; then
+	match=`expr "${WMS_DATE}" : '\([0-9]\{8\}\)'`
+	if [ "$match" != "${WMS_DATE}" ]; then
+		echo ""
+		echo "ERROR: Invalid product date"
+		showHelp
+		exit 1
+	fi
+fi
+
+# Verify the product time
+if [ -n "${WMS_TIME}" ]; then
+	match=`expr "${WMS_TIME}" : '\([0-9]\{6\}\)'`
+	if [ "$match" != "${WMS_TIME}" ]; then
+		echo ""
+		echo "ERROR: Invalid product time"
+		showHelp
+		exit 1
+	fi
+fi
+
+# Get the direct upload name (unless -r was specified)
+if [ ${WMS_PROXY} -eq 0 ]; then
+	SERVER_DIRECT=
+	if [ -n "${WMS_NAME}" ]; then
+		SERVER_DIRECT=`${CURL} -s http://${SERVER}/wmsupload?name=${WMS_NAME}`
+	else
+		SERVER_DIRECT=`${CURL} -s http://${SERVER}/wmsupload?file=${WMS_FILE_NAME}`
+	fi
+	if [ -n "${SERVER_DIRECT}" ]; then
+		SERVER=${SERVER_DIRECT}
+	else
+		if [ ${WMS_VERBOSE} -ne 0 ]; then
+			echo "WARNING: Could not determine the head node URL for proxy upload"
+		fi
+	fi
+fi
+
+# Change to the dir with the file
+cd "${WMS_FILE_DIR}"
+echo "Connecting to ${SERVER}..."
+
+# Check if the server is ready to receive the file
+if [ ${WMS_VERBOSE} -ne 0 ]; then
+	echo "Checking upload availability"
+fi
+BYTES=`/bin/ls -Lln "${WMS_FILE_NAME}" |awk '{print $5}'`
+COMMAND="${CURL} -s http://${SERVER}/wmsupload?bytes=${BYTES}"
+SUCCESS=`${COMMAND} -o - 2>/dev/null |head -1`
+if [ "${SUCCESS}" -eq "${SUCCESS}" ] 2>/dev/null; then
+	if [ "${SUCCESS}" = "-1" ]; then
+		echo "  Server cannot accept the file, it is too large!"
+		exit 3
+	fi
+	while [ "${SUCCESS}" != "1" ]; do
+		echo "  Server cannot accept the file at this time, trying again in 15 seconds..."
+		sleep 15
+		SUCCESS=`${COMMAND} -o - |head -1`
+	done
+else
+	if [ ${WMS_VERBOSE} -ne 0 ]; then
+		echo "  Server does not understand file size check, continuing..."
+	fi
+fi
+
+# Send the file
+echo "Sending ${WMS_FILE_NAME}"
+COMMAND="${TIMEOUT} ${CURL} --max-time 600 -s http://${SERVER}/upload/ -F file=@${WMS_FILE_NAME}"
+if [ -n "${WMS_NAME}" ]; then
+	COMMAND="${COMMAND} -F name=${WMS_NAME}"
+fi
+if [ -n "${WMS_DATE}" ]; then
+	COMMAND="${COMMAND} -F date=${WMS_DATE}"
+fi
+if [ -n "${WMS_TIME}" ]; then
+	COMMAND="${COMMAND} -F time=${WMS_TIME}"
+fi
+if [ ${WMS_PART} -ne 0 ]; then
+	COMMAND="${COMMAND} -F part=1"
+fi
+if [ ${WMS_FORCE} -ne 0 ]; then
+	COMMAND="${COMMAND} -F force=1"
+fi
+
+# Retry a few times...
+RETRIES=3
+${COMMAND}
+LASTEXIT=$?
+while [ ${LASTEXIT} -ne 0 -a ${RETRIES} -gt 0 ]; do
+	echo "Curl command failed: ${LASTEXIT}"
+	SLEEP=$((${RANDOM} * 30 / 32767))
+	echo "Sleeping for ${SLEEP} seconds..."
+	sleep ${SLEEP}
+	echo "Trying again..."
+	${COMMAND}
+	LASTEXIT=$?
+	RETRIES=$((${RETRIES} - 1))
+done
+if [ ${LASTEXIT} -eq 0 ]; then
+	echo "Done"
+else
+	echo "Giving up"
+fi
+exit ${LASTEXIT}