Add ability to seed cache when mapcache times are added

mapcache/Dockerfile

@@ -10,21 +10,6 @@ RUN apt-get -y update && \
     apt-get -y install gcc g++ cmake sqlite3 libsqlite3-dev pkg-config bash-completion curl subversion && \
     apt-get -y clean
 
-
-# FIXME: Remove if/when upstream image builds with fcgi
-# Install Mapserver (unstable)
-#RUN mkdir -p /build_deps && cd /build_deps \
-#  && apt-get -y update \
-##  && apt-get install -y libprotobuf17 zlib1g-dev libpng-dev libjpeg-turbo8 libfreetype6 libfribidi-dev libharfbuzz-dev libcairo2-dev libfcgi-dev libgeos++-dev postgresql postgis libxml2-dev libgif-dev libjpeg-turbo8-dev libprotobuf-dev protobuf-compiler libprotobuf-c-dev libprotobuf-c1 libprotobuf-dev protobuf-c-compiler \
-#  && apt-get install -y zlib1g-dev libpng-dev libjpeg-turbo8 libfreetype6 libfribidi-dev libharfbuzz-dev libcairo2-dev libfcgi-dev libgeos++-dev postgresql postgis libxml2-dev libgif-dev libjpeg-turbo8-dev libprotobuf-dev protobuf-compiler libprotobuf-c-dev libprotobuf-c1 libprotobuf-dev protobuf-c-compiler \
-#  && apt-get -y clean \
-#  && git clone https://github.com/mapserver/mapserver.git && cd mapserver \
-#  && git checkout 0fcc810f0b559c800f950db78a79fa6574799f23 \
-#  && mkdir -p build && cd build \
-#  && cmake .. -DWITH_PROTOBUFC=OFF -DWITH_POSTGIS=OFF \
-#  && make -j$(nproc) \
-#  && make install
-
 # apache
 # http://www.inanzzz.com/index.php/post/rhsb/running-apache-server-as-foreground-on-ubuntu-with-dockerfile
 # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added
@@ -66,16 +51,12 @@ COPY custom-conf /etc/apache2/conf-available/cspp_geo.conf
 # disable the default which would conflict with our custom
 RUN a2enconf cspp_geo
 
-# Point apache to the mapserver binary
-#RUN ln -s /usr/local/bin/mapserv /usr/lib/cgi-bin/mapserv && \
-#    chown ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/* && \
-#    chown -h ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/*
-
 RUN pip3 install jinja2
 COPY render.py /work/
 COPY mapcache_template.xml /work/
-COPY run.sh /work/
+COPY run.sh render_config.sh /work/
 COPY html/ /var/www/html/
+ENV PATH /work:$PATH
 
 # Add our own custom EPSG codes (HACK)
 # GOES-16 ABI Full Disk = EPSG:930916

mapcache/render_config.sh

+#!/usr/bin/env bash
+
+if [[ $# -eq 2 ]]; then
+    export WMS_HOST=$1
+    export WMS_PORT=$2
+else
+    echo "Usage: ./run.sh <wms_hostname> <wms_port>"
+    exit 1
+fi
+
+# Load environment variable options to overwrite in the config
+export CACHE_DISK_BASE=${CACHE_DISK_BASE:-"/data/tile_cache"}
+
+# Update the mapcache.xml file with the real host and port name
+mc_tmpl="/work/mapcache_template.xml"
+mc_xml="/work/mapcache/mapcache.xml"
+mkdir -p $(dirname $mc_xml)
+python3 /work/render.py $mc_tmpl > $mc_xml
+
+# Create the tile directory as root
+if [[ ! -d ${CACHE_DISK_BASE} ]]; then
+    mkdir -p ${CACHE_DISK_BASE} && chown www-data:www-data ${CACHE_DISK_BASE} \
+      && chmod 775 ${CACHE_DISK_BASE}
+fi
+

mapcache/run.sh

 #!/bin/bash -le
 
-if [[ $# -eq 2 ]]; then
-    export WMS_HOST=$1
-    export WMS_PORT=$2
-else
-    echo "Usage: ./run.sh <wms_hostname> <wms_port>"
-    exit 1
-fi
-
-# Load environment variable options to overwrite in the config
-export CACHE_DISK_BASE=${CACHE_DISK_BASE:-"/data/tile_cache"}
-
-# Update the mapcache.xml file with the real host and port name
-mc_tmpl="/work/mapcache_template.xml"
-mc_xml="/work/mapcache/mapcache.xml"
-mkdir -p $(dirname $mc_xml)
-python3 /work/render.py $mc_tmpl > $mc_xml
-
-# Create the tile directory as root
-mkdir -p ${CACHE_DISK_BASE} && chown www-data:www-data ${CACHE_DISK_BASE} \
-  && chmod 775 ${CACHE_DISK_BASE}
-
+# Render the template config to a full config
+render_config.sh "$@"
+# Make sure that the config is reasonable and print out any errors
 /usr/sbin/apache2ctl configtest
-
+# Start apache in the foreground
 /usr/sbin/apache2ctl -DFOREGROUND

mapcache_timedb/Dockerfile

-FROM ubuntu:18.04
+FROM gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapcache:latest
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV TZ=GMT
@@ -6,7 +6,7 @@ ENV LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH}
 ENV LANG=C.UTF-8
 ENV LC_ALL=C.UTF-8
 
-RUN apt-get update && apt-get -y upgrade && apt-get install -y \
+RUN apt-get -y update && apt-get -y upgrade && apt-get install -y \
   gosu \
   pwgen \
   tzdata \
@@ -32,10 +32,10 @@ WORKDIR /work
 
 # Install amqpfind
 RUN apt-get update && apt-get install -y unzip && \
-    wget http://ssec.wisc.edu/~rayg/pub/amqpfind.zip && \
+    wget https://www.ssec.wisc.edu/~rayg/pub/amqpfind.zip && \
     unzip amqpfind.zip && \
     rm amqpfind.zip && \
     rm -rf /var/lib/apt/lists/*
 
-COPY time_updater.py bin/
-COPY run.sh bin/
+COPY time_updater.py /work
+COPY updater.sh /work

mapcache_timedb/README.md

@@ -9,7 +9,7 @@ docker build -t gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapcache:
 ## Usage
 
 ```bash
-docker run -d --rm --name cspp-geo-mapcache-timedb --network cspp-geo-rabbit -v cspp-geo-tile-cache:/data gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapcache_timedb:latest ./bin/run.sh cspp-geo-mapserver 80
+docker run -d --rm --name cspp-geo-mapcache-timedb --network cspp-geo-rabbit -v cspp-geo-tile-cache:/data gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapcache_timedb:latest ./bin/updater.sh cspp-geo-mapserver 80
 ```
 
 Note the host name and port of the MapServer passed to the `run.sh` at the end.

mapcache_timedb/run.sh → mapcache_timedb/updater.sh

@@ -23,6 +23,9 @@ export G2G_PRODUCTS=${G2G_PRODUCTS:-"C01 C02 C03 C04 C05 C06 C07 C08 C09 C10 C11
 export SATELLITES=${SATELLITES:-"g16 g17"}
 export INSTRUMENTS=${INSTRUMENTS:-"abi"}
 export SECTORS=${SECTORS:-"radf radc radm1 radm2"}
+export SEED=${SEED:-"0"}  # 0 (False) or anything else (True)
+export SEED_OVERLAYS=${SEED_OVERLAYS:-"1"}  # 0 (False) or anything else (True)
+export SEED_ARGS=${SEED_ARGS:-""}
 
 json_list_to_bash() {
     blist=$1
@@ -33,6 +36,24 @@ json_list_to_bash() {
 }
 export -f json_list_to_bash
 
+run_mapcache_seed() {
+    CONFIG="/work/mapcache/mapcache.xml"
+    NPROCS="4"
+    TILESET=$1  # g16_abi_radf_l1b_true_color
+    GRID=$2  # grid_g16_abi_radf
+    # optional (borders/overlays don't have a time dimension)
+    T=$3  # YYYY-MM-DDTHH:MM:SS (Z will be added, assumes UTC)
+    if [[ $T == "" ]]; then
+        TIME=""
+    else
+        TIME="-D TIME=${T}Z"  # Add Z timezone
+    fi
+    set -x
+    su -l www-data -s /bin/bash -c "export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH; time mapcache_seed -c $CONFIG -t $TILESET -n $NPROCS -g $GRID $TIME $SEED_ARGS"
+    set +x
+}
+export -f run_mapcache_seed
+
 run_time_updater() {
     if [[ $# -ne 7 ]]; then
         echo "Unexpected number of arguments (expected 7): $#"
@@ -43,7 +64,7 @@ run_time_updater() {
     satellite_id=${2,,}
     instrument=${3,,}
     data_type=${4,,}
-    product=${5,,}
+    product=${5}
     removed_times="$6"
     added_times="$7"
 
@@ -51,7 +72,7 @@ run_time_updater() {
     removed_times=$(json_list_to_bash "$removed_times")
     added_times=$(json_list_to_bash "$added_times")
 
-    echo "Starting tile generation processing for ${path}"
+    echo "Starting cache time updates for ${path}"
 
     # if /dst isn't defined then use /data
     if [[ -d "/dst" ]]; then
@@ -64,11 +85,21 @@ run_time_updater() {
     table_name="${satellite_id}_${instrument}_${data_type}_l1b_${product}"
     echo "Removing times for: ${dbfile}"
     echo "Removed times: ${removed_times}"
-    python3 /work/bin/time_updater.py -r "${dbfile}" "${table_name}" ${removed_times}
+    python3 /work/time_updater.py -r "${dbfile}" "${table_name}" ${removed_times}
     echo "Adding times for: ${dbfile}"
     echo "Added times: ${added_times}"
-    python3 /work/bin/time_updater.py "${dbfile}" "${table_name}" ${added_times}
+    python3 /work/time_updater.py "${dbfile}" "${table_name}" ${added_times}
     echo "Done updating times for: ${dbfile}"
+
+    if [[ ${SEED} -ne 0 ]]; then
+        tileset="${satellite_id}_${instrument}_${data_type}_l1b_${product}"
+        grid="grid_${satellite_id}_${instrument}_${data_type}"
+        for t in ${added_times}; do
+            echo "Seeding cache for ${tileset} @ ${t}"
+            run_mapcache_seed $tileset $grid $t
+            echo "Done seeding"
+        done
+    fi
 }
 export -f run_time_updater
 
@@ -86,14 +117,29 @@ init_time_updater() {
             for sector in ${SECTORS}; do
                 dbfile="${dst_dir}/mapcache_time.sqlite"
                 for product in ${G2G_PRODUCTS}; do
-                    product=${product,,}
                     echo "Initializing database ${dbfile} for product ${product}"
                     set -x
                     current_times=$(wget -O - "http://${WMS_HOST}:${WMS_PORT}/wms_times/${sat_id}/${inst}/${sector}/${product}")
                     table_name="${sat_id}_${inst}_${sector}_l1b_${product}"
-                    python3 /work/bin/time_updater.py --json "${dbfile}" "${table_name}" "${current_times}"
+                    python3 /work/time_updater.py --json "${dbfile}" "${table_name}" "${current_times}"
                     set +x
                 done
+
+                # Preseed the overlays, graticules, and basemap layers
+                if [[ $SEED_OVERLAYS -ne 0 ]]; then
+                    grid="grid_${sat_id}_${inst}_${sector}"
+
+                    echo "Seeding basemap layer"
+                    tileset="${sat_id}_${inst}_basemap"
+                    SEED_ARGS="" run_mapcache_seed $tileset $grid
+                    echo "Seeding borders layer"
+                    tileset="${sat_id}_${inst}_borders"
+                    SEED_ARGS="" run_mapcache_seed $tileset $grid
+                    echo "Seeding graticules layer"
+                    tileset="${sat_id}_${inst}_graticules"
+                    SEED_ARGS="" run_mapcache_seed $tileset $grid
+                    echo "Done seeding ${grid}'s overlay layers"
+                fi
             done
         done
     done
@@ -107,6 +153,9 @@ else
     exit 1
 fi
 
+# Render the MapCache config so we can seed the cache later
+render_config.sh $WMS_HOST $WMS_PORT
+
 # Initialize the database if it doesn't exist already
 echo "Initializing time database..."
 init_time_updater
@@ -115,7 +164,6 @@ if [[ $? -ne 0 ]]; then
     exit 1
 fi
 
-
 echo "Listening to AMQP messages with topic \"$AMQPFIND_TOPIC\""
-python3 amqpfind/amqpfind.py ${AMQPFIND_ARGS} -C "${AMQPFIND_TOPIC}" -j "{satellite_family} {satellite_ID} {instrument} {data_type} {product} \'{removed_times}\' \'{added_times}\'" | xargs -I{} -P3 -n1 bash -c "run_time_updater {}"
+python3 amqpfind/amqpfind.py ${AMQPFIND_ARGS} -C "${AMQPFIND_TOPIC}" -j "{satellite_family} {satellite_ID} {instrument} {data_type} {product} \'{removed_times}\' \'{added_times}\'" | xargs -I{} -P4 -n1 bash -c "run_time_updater {}"