Skip to content
Snippets Groups Projects
Commit bbe87cbc authored by Ray Garcia's avatar Ray Garcia :scream_cat:
Browse files

Merge remote-tracking branch 'origin/develop'

parents e764d4fd 74469d55
No related branches found
No related tags found
No related merge requests found
#!/bin/bash
#
# Example script demonstrating automation of SCMI tile generation using AXI-Tools.
#
# Requires inotify-tools (available as part of CentOS) be present.
# Requires these environment variables:
# GRB_PRODUCTS input L1B directory
# AXI_TOOLS AXI-Tools location
# SCENE ABI scene to filter on, e.g. RadC, RadM1, RadM2, RadF
# SCMI_REGION Region code to use in output SCMI filenames, e.g. ECONUS, EMESO1, EMESO2, EFD
# SCMI_ENV Environment and data type code to use in output SCMI filename, e.g. OT, OR, DT
#
# Invocation:
# /path/to/grb2scmi.sh
#
# Output is written to the working directory.
oops() {
echo "ERROR: $*"
exit 1
}
# configurable values provided in environment
test -d "$GRB_PRODUCTS" || oops "please set GRB_PRODUCTS environment to directory of arriving GRB products"
test -n "$AXI_TOOLS" || oops "Please set AXI_TOOLS in environment to indicate location of the AXI-Tools software"
test -x "$(which inotifywait)" || oops "inotify-tools is needed by this script."
export PY3=$AXI_TOOLS/ShellB3/bin/python3
test -x "$PY3" || oops "Could not find $PY3. Check environment variable AXI_TOOLS"
export CMI_CHANGER=$AXI_TOOLS/bin/cmi_changer.sh
test -f "$CMI_CHANGER" || oops "Could not find $CMI_CHANGER. Check environment variable AXI_TOOLS"test -n "$SCENE" || oops "please set SCENE in environment to indicate the scene to filter incoming files on (e.g. \"RadC\")" # e.g. "RadC" "RadM1", "RadM2", "RadF"
test -n "$SCMI_REGION" || oops "please set SCMI_REGION in environment to indicate the region to use in output files (e.g. \"ECONUS\")" # TCONUS, TMESO1, TMESO2, TFD for lack of a better guess
test -n "$SCMI_ENV" || oops "please set SCMI_ENV in environment to indicate the environment and data type to use in output files (e.g. \"OT\")" # OR, DT, etc
cmi_changer() {
# echo "+++ creating SCMI tiles using $*"
$CMI_CHANGER -E $SCMI_ENV -R $SCMI_REGION "$@"
}
export -f cmi_changer # needed when using with xargs and other forms of sub-shell like $(...)
# OR_ABI-L1b-RadM1-M3C
inotify_scene() {
scene="$1"
inotifywait -m -r -e close_write -e moved_to --format "${GRB_PRODUCTS}/%f" "$GRB_PRODUCTS" \
|grep --line-buffered -E ".*?_ABI-L1b-${scene}-M.*\.nc$"
}
inotify_main() {
inotify_scene $SCENE \
|xargs -P20 -L1 -I {} /bin/nice /bin/bash -c "cmi_changer {}"
}
while true; do
inotify_main &>${SCENE}-$(date '+%Y%m%dT%H%M%S').log || echo "ERROR: message loop exited"
echo "ERROR: timeout at $(date); restarting in 30s"
sleep 30
done
......@@ -95,7 +95,7 @@ inotify_scene() {
if [ -n "$HCAST_SECTION_TIMEOUT" ]; then
inotifywait -m -r -e close_write -e moved_to --format "${HCAST_PRODUCTS}/%f" "$HCAST_PRODUCTS" \
|grep --line-buffered -E ".*?${scene}.*$" \
|$PY3 -m goesr.consolidate --hcast-sections --timeout=$HCAST_SECTION_TIMEOUT \
|$PY3 -u -m goesr.consolidate --hcast-sections --timeout=$HCAST_SECTION_TIMEOUT \
|sed -u 's/\*$//g'
else
inotifywait -m -r -e close_write -e moved_to --format "${HCAST_PRODUCTS}/%f" "$HCAST_PRODUCTS" \
......
......@@ -338,9 +338,11 @@ def _debug(type_, value, tb):
pdb.post_mortem(tb) # more “modern”
def _line_thread_main(queue: Queue, fob):
# renamed _enqueue_lines since _line_thread_main had block buffering issue
# but is keyed-monkeypatched by some versions of hcast2scmi.sh in nws_glue repo
def _enqueue_lines(queue: Queue, fob):
try:
for line in fob.readlines():
for line in iter(fob.readline, ''):
queue.put(line)
queue.put(None) # signal done
except Exception as exc:
......@@ -350,9 +352,8 @@ def _line_thread_main(queue: Queue, fob):
def threaded_line_reader(fob, delay_seconds=5.0):
"""yield lines from file, or None every delay_seconds of idle
"""
Q = Queue()
T = Thread(target=_line_thread_main, args=(Q, fob))
T = Thread(target=_enqueue_lines, args=(Q, fob))
T.daemon = True
T.start()
while True:
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment