Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Nick Bearson
eodn2realearth
Commits
e73b4a72
Commit
e73b4a72
authored
Mar 15, 2015
by
Nick Bearson
Browse files
initializing, please wait...
parents
Changes
4
Hide whitespace changes
Inline
Side-by-side
LCzips/listen.py
0 → 100755
View file @
e73b4a72
#!/usr/bin/env python
import
websocket
import
re
,
os
import
json
import
argparse
from
subprocess
import
call
FILERE
=
re
.
compile
(
r
'LC[0-9]{14}LGN00.zip'
)
BIN
=
os
.
path
.
dirname
(
os
.
path
.
abspath
(
__file__
))
PROCESS
=
os
.
path
.
join
(
BIN
,
"process.sh"
)
def
on_message
(
ws
,
message
):
js
=
json
.
loads
(
message
)
print
"scene : %s"
%
js
[
'properties'
][
'metadata'
][
'scene_id'
]
print
"Filename: %s"
%
js
[
'name'
]
print
"Size : %s"
%
js
[
'size'
]
print
"URL : %s"
%
js
[
'selfRef'
]
if
(
FILERE
.
match
(
js
[
'name'
])):
print
"
\n
### Matching filename found, processing...."
try
:
results
=
call
([
PROCESS
,
js
[
'name'
],
js
[
'selfRef'
]])
except
Exception
as
e
:
print
"ERROR calling process.sh %s"
%
e
def
on_error
(
ws
,
error
):
print
(
error
)
def
on_close
(
ws
):
print
(
"### closed ###"
)
def
on_open
(
ws
):
pass
if
__name__
==
"__main__"
:
parser
=
argparse
.
ArgumentParser
(
description
=
"Listen for and then process a particular LANDSAT scene"
)
parser
.
add_argument
(
'-H'
,
'--host'
,
type
=
str
,
help
=
'The Exnode service'
,
default
=
"ws://dev.incntre.iu.edu:8888/subscribe/exnode"
)
args
=
parser
.
parse_args
()
host
=
args
.
host
ws
=
websocket
.
WebSocketApp
(
host
,
on_message
=
on_message
,
on_error
=
on_error
,
on_close
=
on_close
)
ws
.
on_open
=
on_open
ws
.
run_forever
()
LCzips/process.sh
0 → 100755
View file @
e73b4a72
#!/bin/bash
set
-u
set
-e
SCENE
=
$1
URL
=
$2
TMP
=
$PWD
/temp
FILE
=
"scene.zip"
BASE
=
${
SCENE
##*/
}
# filename w/ extension
STRIP
=
${
BASE
%%.zip
}
# filename w/o extension
YEAR
=
${
SCENE
:9:4
}
DOY
=
${
SCENE
:13:3
}
YMD
=
`
date
-d
"Jan 1
$YEAR
+
$DOY
days - 1 day"
"+%Y%m%d"
`
PART
=
${
SCENE
:2:7
}
function
download
()
{
URL
=
$1
FILE
=
$2
time
lors_download
-t
10
-b
10m
$URL
-o
$FILE
# for testing...
# cp $URL $FILE
}
function
process
()
{
FILE
=
$1
mkdir
-p
$TMP
echo
"Unpacking
${
SCENE
}
..."
unzip
$FILE
-d
$TMP
cd
$TMP
sed
-i
s:
\&
quot
\;
:
\"
:g
*
.aux.xml
# swap " for actual quotes
for
REF
in
""
_QB _TIR
;
do
case
$REF
in
_QB
)
IMG
=
${
STRIP
}${
REF
}
.png
TAG
=
QB
;;
_TIR
)
IMG
=
${
STRIP
}${
REF
}
.jpg
TAG
=
TIR
;;
""
)
IMG
=
${
STRIP
}${
REF
}
.jpg
TAG
=
FC
;;
esac
if
[
!
-f
$IMG
]
;
then
# some zips don't have all the image types - skip any that don't exist
continue
fi
grep
SRS
${
IMG
}
.aux.xml
>
WKT
# extract the WKT line from XML
sed
-i
s:
\ \ \<
SRS
\>
::g WKT
# remove XML tags from WKT
sed
-i
s:
\<\/
SRS
\>
::g WKT
TIF
=
EODN-
${
TAG
}
_
${
YMD
}
_120000.part
${
PART
}
.tif
gdal_translate
-of
GTiff
-a_srs
WKT
$IMG
$TIF
wmsupload
$TIF
done
rm
-r
$TMP
}
download
$URL
$FILE
if
[
!
-f
$FILE
]
;
then
echo
"Could not download
$URL
"
exit
fi
(
process
$FILE
)
rm
$FILE
README
0 → 100644
View file @
e73b4a72
Requirements in $PATH:
+ gdal
+ lors_download (http://loci.cs.utk.edu/lors/tutorial/download.php)
+ wmsupload (http://realearth.ssec.wisc.edu/wmsupload)
Also requires a python 2.7+ install with:
+ websocket-client (https://pypi.python.org/pypi/websocket-client)
wmsupload
0 → 100755
View file @
e73b4a72
#!/bin/sh
################################################################################
# DO NOT ALTER THIS LINE
VERSION
=
11
################################################################################
SERVER
=
realearth.ssec.wisc.edu:80
showHelp
()
{
echo
""
echo
"
$0
[-hufprv] [-s server:port] file [name] [date] [time]"
echo
""
echo
" -h: Show help"
echo
" -u: Check for update"
echo
""
echo
" -f: Force overwrite if there is a conflict"
echo
" -p: Designate file as part of a larger product"
echo
""
echo
" -r: Send through head node proxy"
echo
" -s: Specify the server and port ([server:port] required)"
echo
""
echo
" -v: Be verbose"
echo
""
echo
" file: Path to file"
echo
" Format: /path/to/[name]_[YYYYMMDD]_[HHMMSS].???"
echo
""
echo
" name: Specify the product name"
echo
" Required when the file name does not contain [name]"
echo
" Format: Cannot contain '_'"
echo
" date: Specify the date"
echo
" Required when the file name does not contain [date]"
echo
" Format: YYYYMMDD"
echo
" time: Specify the time"
echo
" Required when the file name does not contain [time]"
echo
" Format: HHMMSS"
echo
""
}
checkUpdate
()
{
DL_URL
=
"http://
${
SERVER
}
/wmsupload"
URL
=
"
$DL_URL
?version=
${
VERSION
}
"
VERSION_CHECK
=
`
curl
-s
${
URL
}
`
if
[
-n
"
${
VERSION_CHECK
}
"
]
;
then
echo
"A new version is available at:"
echo
"
${
DL_URL
}
"
echo
"Download with:"
echo
" curl
${
DL_URL
}
-o
$0
"
else
echo
"This version is up to date"
fi
}
# Get the options from the command line
WMS_FORCE
=
0
WMS_PART
=
0
WMS_PROXY
=
0
WMS_VERBOSE
=
0
if
[
-z
"
$1
"
]
;
then
showHelp
exit
0
fi
while
getopts
hufprvs: o
;
do
case
"
$o
"
in
h
)
showHelp
exit
0
;;
u
)
checkUpdate
exit
0
;;
f
)
WMS_FORCE
=
1
;;
p
)
WMS_PART
=
1
;;
r
)
WMS_PROXY
=
1
;;
v
)
WMS_VERBOSE
=
1
;;
s
)
SERVER
=
${
OPTARG
}
;;
*
)
showHelp
exit
1
;;
esac
done
shift
$((${
OPTIND
}
-
1
))
# Set our variables
WMS_FILE
=
$1
WMS_NAME
=
$2
WMS_DATE
=
$3
WMS_TIME
=
$4
# Does the file exist?
if
[
!
-f
"
${
WMS_FILE
}
"
]
;
then
echo
"ERROR: Could not find file:
${
WMS_FILE
}
"
exit
1
fi
# Set the defaults for sending
WMS_FILE_DIR
=
`
dirname
"
${
WMS_FILE
}
"
`
WMS_FILE_NAME
=
`
basename
"
${
WMS_FILE
}
"
`
# Verify the product name
if
[
-n
"
${
WMS_NAME
}
"
]
;
then
match
=
`
expr
"
${
WMS_NAME
}
"
:
'\([a-zA-Z0-9\-]\{1,\}\)'
`
if
[
"
$match
"
!=
"
${
WMS_NAME
}
"
]
;
then
echo
""
echo
"ERROR: Invalid product name"
showHelp
exit
1
fi
fi
# Verify the product date
if
[
-n
"
${
WMS_DATE
}
"
]
;
then
match
=
`
expr
"
${
WMS_DATE
}
"
:
'\([0-9]\{8\}\)'
`
if
[
"
$match
"
!=
"
${
WMS_DATE
}
"
]
;
then
echo
""
echo
"ERROR: Invalid product date"
showHelp
exit
1
fi
fi
# Verify the product time
if
[
-n
"
${
WMS_TIME
}
"
]
;
then
match
=
`
expr
"
${
WMS_TIME
}
"
:
'\([0-9]\{6\}\)'
`
if
[
"
$match
"
!=
"
${
WMS_TIME
}
"
]
;
then
echo
""
echo
"ERROR: Invalid product time"
showHelp
exit
1
fi
fi
# Get the direct upload name (unless -r was specified)
if
[
${
WMS_PROXY
}
-eq
0
]
;
then
SERVER_DIRECT
=
if
[
-n
"
${
WMS_NAME
}
"
]
;
then
SERVER_DIRECT
=
`
curl
-s
http://
${
SERVER
}
/wmsupload?name
=
${
WMS_NAME
}
`
else
SERVER_DIRECT
=
`
curl
-s
http://
${
SERVER
}
/wmsupload?file
=
${
WMS_FILE_NAME
}
`
fi
if
[
-n
"
${
SERVER_DIRECT
}
"
]
;
then
SERVER
=
${
SERVER_DIRECT
}
else
if
[
${
WMS_VERBOSE
}
-ne
0
]
;
then
echo
"WARNING: Could not determine the head node URL for proxy upload"
fi
fi
fi
# Change to the dir with the file
cd
"
${
WMS_FILE_DIR
}
"
echo
"Connecting to
${
SERVER
}
..."
# Check if the server is ready to receive the file
if
[
${
WMS_VERBOSE
}
-ne
0
]
;
then
echo
"Checking upload availability"
fi
BYTES
=
`
/bin/ls
-Ll
"
${
WMS_FILE_NAME
}
"
|/bin/awk
'{print $5}'
`
COMMAND
=
"curl -s http://
${
SERVER
}
/wmsupload?bytes=
${
BYTES
}
"
SUCCESS
=
`
${
COMMAND
}
-o
- 2>/dev/null |head
-1
`
if
[
"
${
SUCCESS
}
"
-eq
"
${
SUCCESS
}
"
]
2>/dev/null
;
then
if
[
"
${
SUCCESS
}
"
=
"-1"
]
;
then
echo
" Server cannot accept the file, it is too large!"
exit
3
fi
while
[
"
${
SUCCESS
}
"
!=
"1"
]
;
do
echo
" Server cannot accept the file at this time, trying again in 15 seconds..."
sleep
15
SUCCESS
=
`
${
COMMAND
}
-o
- |head
-1
`
done
else
if
[
${
WMS_VERBOSE
}
-ne
0
]
;
then
echo
" Server does not understand file size check, continuing..."
fi
fi
# Send the file
echo
"Sending
${
WMS_FILE_NAME
}
"
COMMAND
=
"curl -s http://
${
SERVER
}
/upload/ -F file=@
${
WMS_FILE_NAME
}
"
if
[
-n
"
${
WMS_NAME
}
"
]
;
then
COMMAND
=
"
${
COMMAND
}
-F name=
${
WMS_NAME
}
"
fi
if
[
-n
"
${
WMS_DATE
}
"
]
;
then
COMMAND
=
"
${
COMMAND
}
-F date=
${
WMS_DATE
}
"
fi
if
[
-n
"
${
WMS_TIME
}
"
]
;
then
COMMAND
=
"
${
COMMAND
}
-F time=
${
WMS_TIME
}
"
fi
if
[
${
WMS_PART
}
-ne
0
]
;
then
COMMAND
=
"
${
COMMAND
}
-F part=1"
fi
if
[
${
WMS_FORCE
}
-ne
0
]
;
then
COMMAND
=
"
${
COMMAND
}
-F force=1"
fi
${
COMMAND
}
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment