diff --git a/mapserver/Dockerfile b/mapserver/Dockerfile
index 4982922f612c61545f4c07e9763f387df8934e28..75d218c700991b6f5bc99f3af66bc90981619310 100644
--- a/mapserver/Dockerfile
+++ b/mapserver/Dockerfile
@@ -54,6 +54,7 @@ COPY cgi-bin/* /usr/lib/cgi-bin/
# Point apache to the mapserver binary
RUN ln -s /usr/local/bin/mapserv /usr/lib/cgi-bin/mapserv && \
+ ln -s /usr/local/bin/mapserv /usr/lib/cgi-bin/mapserv.fcgi && \
chown ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/* && \
chown -h ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/*
@@ -66,6 +67,9 @@ COPY html/ /var/www/html/
COPY sql/ /work/sql/
RUN sqlite3 -init /work/sql/goesr_crs.sql /usr/local/share/proj/proj.db
+# Check the config before we finish
+RUN apache2ctl configtest
+
# https://httpd.apache.org/docs/2.4/stopping.html#gracefulstop
STOPSIGNAL WINCH
diff --git a/mapserver/mapfiles/g16_abi_radf_l1b.map b/mapserver/mapfiles/g16_abi_radf_l1b.map
index d989dc4c112317afecca7c6e324e3ec384f5784b..384b7034056b514e4bd5868d9f991b70147f13d5 100644
--- a/mapserver/mapfiles/g16_abi_radf_l1b.map
+++ b/mapserver/mapfiles/g16_abi_radf_l1b.map
@@ -1,7 +1,8 @@
MAP
IMAGETYPE PNG
SIZE 256 256
- EXTENT -180 -90 180 90
+ EXTENT -5434894.885 -5434894.885 5434894.885 5434894.885
+ #EXTENT -180 -90 180 90
PROJECTION
"init=epsg:930916"
diff --git a/mapserver/site-conf b/mapserver/site-conf
index 0c812d2e18e49146c1dfb537f702802b4f65e8f5..b83c316d3b5534c4601d956644a1e4fbfd7fbea4 100644
--- a/mapserver/site-conf
+++ b/mapserver/site-conf
@@ -28,16 +28,25 @@
#Include conf-available/serve-cgi-bin.conf
# Custom CSPP Geo
- # LoadModule rewrite_module modules/mod_rewrite.so
RewriteEngine on
# /wms/goes16/abi/fldk/
RewriteRule "^/wms/([^/]+)/([^/]+)/([^/]+)/l1b?(.*)" "/cgi-bin/mapserv?map=/work/mapfiles/$1_$2_$3_l1b.map&$4" [PT,QSA]
+ # FCGI version of the above
+ RewriteRule "^/fwms/([^/]+)/([^/]+)/([^/]+)/l1b?(.*)" "/cgi-bin/mapserv.fcgi?map=/work/mapfiles/$1_$2_$3_l1b.map&$4" [PT,QSA]
# /wms_times/g16/abi/radf/true_color
RewriteRule "^/wms_times/([^/]+)/([^/]+)/([^/]+)/([^/]+)" "/cgi-bin/layer_times.py?layer=/data/tiles/$1/$2/$3/$4/$4.shp" [PT,QSA]
# FIXME: We need to include the sector
# /data/goes/grb/goes16/2020/2020_01_21_021/abi/L1b/RadF/GOES-16_ABI_RadF_C01_20200121_000016_GOES-East.tif
# "/data/tiles/g16/abi/radf/true_color/true_color"
LogLevel alert rewrite:trace6
+
+ # handler should be added by the enabled module
+ #AddHandler fcgid-script fcgi
+ <IfModule mod_fcgid.c>
+ FcgidMaxProcessesPerClass 30
+ FcgidInitialEnv PROJ_LIB /usr/local/share/proj
+ FcgidInitialEnv LD_LIBRARY_PATH "/usr/local/lib:/usr/local/pgsql/lib:/usr3/pkg3/oracle9/lib"
+ </IfModule>
</VirtualHost>
# vim: syntax=apache ts=4 sw=4 sts=4 sr noet
\ No newline at end of file
diff --git a/tile_gen/Dockerfile b/tile_gen/Dockerfile
index 125540c43684370b59e725755a438fbf2982bb40..e51a66e8f631d3acc843953c71520fdc614d53c3 100644
--- a/tile_gen/Dockerfile
+++ b/tile_gen/Dockerfile
@@ -2,7 +2,6 @@ FROM tiledb/tiledb-geospatial:latest
WORKDIR /work
-# TODO may need the unzip command to be installed if not already
RUN apt-get update && apt-get install -y unzip && \
wget http://ssec.wisc.edu/~rayg/pub/amqpfind.zip && \
unzip amqpfind.zip && \
diff --git a/tile_gen/generate_tiles.py b/tile_gen/generate_tiles.py
index 2312f6b1c800bf047d9eab36a0b3a8306e004793..0a5c7e8c5158f07c994cbacc6c636f6e3c486f17 100644
--- a/tile_gen/generate_tiles.py
+++ b/tile_gen/generate_tiles.py
@@ -6,6 +6,7 @@ import warnings
import logging
import subprocess
import shutil
+import json
from glob import glob
import tile_index
@@ -71,6 +72,11 @@ def main():
help="Shapefile filename pattern to use and placed "
"in the output directory. "
"(default: '{product}.shp')")
+ parser.add_argument('--json-times', action='store_true',
+ help="Print a JSON dict to stdout containing removed "
+ "and added times (ex. '{\'times_removed\': [], "
+ "\'times_added\': []}'). Times are in the form: "
+ "YYYY-MM-DDTHH:MM:SS.")
parser.add_argument('out_dir',
help="Output path to save tile information to (ex. '/data/tiles/{product}')")
parser.add_argument('input_files', nargs="+",
@@ -78,6 +84,8 @@ def main():
args = parser.parse_args()
groups = group_files(args.products, args.input_files)
+ all_added = set()
+ all_removed = set()
for prod, prod_files in groups.items():
out_dir = args.out_dir.format(product=prod)
os.makedirs(out_dir, exist_ok=True)
@@ -97,8 +105,14 @@ def main():
# create shape file
LOG.info("Rebuilding shapefile index with:\n\t{}".format(", ".join(all_prod_files)))
- tile_index.index(all_prod_files, shp_pathname)
-
+ removed_times, added_times = tile_index.index(all_prod_files, shp_pathname)
+ all_removed.update(removed_times)
+ all_added.update(added_times)
+
+ if args.json_times:
+ all_removed = [x.isoformat() for x in sorted(all_removed)]
+ all_added = [x.isoformat() for x in sorted(all_added)]
+ print(json.dumps({'removed_times': all_removed, 'added_times': all_added}))
if __name__ == "__main__":
diff --git a/tile_gen/run.sh b/tile_gen/run.sh
index 449169bbe594746a7cd09bca1f245ccb7579816b..7501bea2c61206b36cd6fc710e237427b5054253 100755
--- a/tile_gen/run.sh
+++ b/tile_gen/run.sh
@@ -16,7 +16,7 @@
test -d "/data"
export AMQPFIND_ARGS=${AMQPFIND_ARGS:-"-H cspp-geo-rabbit -X satellite -u guest -p guest"}
-export AMQPSEND_ARGS=${AMQPSEND_ARGS:-"-H cspp-geo-rabbit -X satellite -u guest -p guest"}
+export AMQPSEND_ARGS=${AMQPSEND_ARGS:-"-D 15 -H cspp-geo-rabbit -X satellite -u guest -p guest"}
export AMQPFIND_TOPIC=${AMQPFIND_TOPIC:-'data.goes.*.abi.*.l1b.geotiff.complete'}
export G2G_PRODUCTS=${G2G_PRODUCTS:-"C01 C02 C03 C04 C05 C06 C07 C08 C09 C10 C11 C12 C13 C14 C15 C16 true_color"}
export TILE_ARGS=${TILE_ARGS:-""}
@@ -62,16 +62,20 @@ run_tile_gen() {
out_dir="${out_dir}/{product}"
fi
echo "Generating tiles in directory: ${out_dir}"
- python3 generate_tiles.py ${TILE_ARGS} -p ${G2G_PRODUCTS} -- ${out_dir} ${path}
- # OUT/<product>/<product>.shp
- glob_pattern="${out_dir}/*/*.shp"
- # Remove the /data prefix
- glob_pattern="${glob_pattern/${dst_dir}\//}"
+ for product in ${G2G_PRODUCTS}; do
+ json_times_changed=$(python3 generate_tiles.py ${TILE_ARGS} --json-times -p ${product} -- ${out_dir} ${path})
+ # OUT/<product>/<product>.shp
+ glob_pattern="${out_dir}/${product}/*.shp"
+ # Remove the /data prefix
+ glob_pattern="${glob_pattern/${dst_dir}\//}"
- amqpsend_topic="data.${satellite_family}.${satellite_id}.${instrument}.${data_type}.l1b.tiledb.complete"
-# json_info="{path: ${glob_pattern}, satellite_family: ${satellite_family}, satellite_ID: ${satellite_id}, instrument: ${instrument}, data_type: ${data_type}}"
- json_info="{\"path\": \"${glob_pattern}\", \"satellite_family\": \"${satellite_family}\", \"satellite_ID\": \"${satellite_id}\", \"instrument\": \"${instrument}\", \"data_type\": \"${data_type}\"}"
- echo -e "[[\"$amqpsend_topic\", $json_info]]" | python3 /work/amqpfind/amqpsend.py ${AMQPSEND_ARGS}
+ amqpsend_topic="data.${satellite_family}.${satellite_id}.${instrument}.${data_type}.l1b.tiles.${product}.complete"
+ json_info="{\"path\": \"${glob_pattern}\", \"satellite_family\": \"${satellite_family}\", \"satellite_ID\": \"${satellite_id}\", \"instrument\": \"${instrument}\", \"data_type\": \"${data_type}\", \"product\": \"${product}\"}"
+ # append the JSON returned by the python (remove the curly braces at the ends)
+ json_info="${json_info:0:-1}, ${json_times_changed:1}"
+ echo -e "[[\"$amqpsend_topic\", $json_info]]"
+ echo -e "[[\"$amqpsend_topic\", $json_info]]" | python3 /work/amqpfind/amqpsend.py ${AMQPSEND_ARGS}
+ done
echo "Done generating tiles for ${path}"
}
diff --git a/tile_gen/tile_index.py b/tile_gen/tile_index.py
index 6ff1769ef093322586201601cdc4432279c55e6f..2514c4b29d1fe7b4d676fbb787aff190a4a28d38 100644
--- a/tile_gen/tile_index.py
+++ b/tile_gen/tile_index.py
@@ -41,6 +41,18 @@ possible_time_regex = (
)
+def get_current_times(shp_filename):
+ """Get currently available times in the shapefile."""
+ if not os.path.isfile(shp_filename):
+ # no shapefile, no times
+ return
+
+ with fiona.open(shp_filename, 'r') as shp_file:
+ for time_step in shp_file:
+ t = time_step['properties']['time']
+ yield datetime.datetime.strptime(t, '%Y-%m-%dT%H:%M:%S')
+
+
def get_file_time(fn):
for regex, time_fmt in possible_time_regex:
matches = regex.findall(fn)
@@ -57,16 +69,23 @@ def index(input_files, output_shapefile):
once. Repeated calls to this function will overwrite existing
shapefile information.
+ Returns:
+ (times_removed, times_added): Two lists of times removed by this index
+ being updated and times added by this index being updated.
+
"""
import tempfile
out_dir, shp_fn = os.path.split(output_shapefile)
tmp_dir = tempfile.mkdtemp("_tile_index")
tmp_shapefile = os.path.join(tmp_dir, shp_fn)
+ current_times = set(get_current_times(output_shapefile))
+ new_times = set()
with fiona.open(tmp_shapefile, 'w', driver='ESRI Shapefile',
schema=temporal_schema) as output:
for f in input_files:
try:
dt = get_file_time(f)
+ new_times.add(dt)
except ValueError:
logger.error(f"Can't time for file {f}")
continue
@@ -90,6 +109,10 @@ def index(input_files, output_shapefile):
# we don't need the temporary directory anymore
shutil.rmtree(tmp_dir, ignore_errors=True)
+ times_removed = current_times - new_times
+ times_added = new_times = current_times
+ return sorted(times_removed), sorted(times_added)
+
# def index(src_dir, output):
# files = glob.glob(os.path.join(src_dir, '*.tif'))