diff --git a/mapcache/Dockerfile b/mapcache/Dockerfile index 4799c8635dcd82bdfb31617fa315440505260511..22fa86abef89f8696c77948b19e665907db9fe90 100644 --- a/mapcache/Dockerfile +++ b/mapcache/Dockerfile @@ -10,21 +10,6 @@ RUN apt-get -y update && \ apt-get -y install gcc g++ cmake sqlite3 libsqlite3-dev pkg-config bash-completion curl subversion && \ apt-get -y clean - -# FIXME: Remove if/when upstream image builds with fcgi -# Install Mapserver (unstable) -#RUN mkdir -p /build_deps && cd /build_deps \ -# && apt-get -y update \ -## && apt-get install -y libprotobuf17 zlib1g-dev libpng-dev libjpeg-turbo8 libfreetype6 libfribidi-dev libharfbuzz-dev libcairo2-dev libfcgi-dev libgeos++-dev postgresql postgis libxml2-dev libgif-dev libjpeg-turbo8-dev libprotobuf-dev protobuf-compiler libprotobuf-c-dev libprotobuf-c1 libprotobuf-dev protobuf-c-compiler \ -# && apt-get install -y zlib1g-dev libpng-dev libjpeg-turbo8 libfreetype6 libfribidi-dev libharfbuzz-dev libcairo2-dev libfcgi-dev libgeos++-dev postgresql postgis libxml2-dev libgif-dev libjpeg-turbo8-dev libprotobuf-dev protobuf-compiler libprotobuf-c-dev libprotobuf-c1 libprotobuf-dev protobuf-c-compiler \ -# && apt-get -y clean \ -# && git clone https://github.com/mapserver/mapserver.git && cd mapserver \ -# && git checkout 0fcc810f0b559c800f950db78a79fa6574799f23 \ -# && mkdir -p build && cd build \ -# && cmake .. -DWITH_PROTOBUFC=OFF -DWITH_POSTGIS=OFF \ -# && make -j$(nproc) \ -# && make install - # apache # http://www.inanzzz.com/index.php/post/rhsb/running-apache-server-as-foreground-on-ubuntu-with-dockerfile # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added @@ -66,16 +51,12 @@ COPY custom-conf /etc/apache2/conf-available/cspp_geo.conf # disable the default which would conflict with our custom RUN a2enconf cspp_geo -# Point apache to the mapserver binary -#RUN ln -s /usr/local/bin/mapserv /usr/lib/cgi-bin/mapserv && \ -# chown ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/* && \ -# chown -h ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/* - RUN pip3 install jinja2 COPY render.py /work/ COPY mapcache_template.xml /work/ -COPY run.sh /work/ +COPY run.sh render_config.sh /work/ COPY html/ /var/www/html/ +ENV PATH /work:$PATH # Add our own custom EPSG codes (HACK) # GOES-16 ABI Full Disk = EPSG:930916 diff --git a/mapcache/render_config.sh b/mapcache/render_config.sh new file mode 100755 index 0000000000000000000000000000000000000000..500b92de2ac6c9199ff5ab7051fcfd82c634dcbe --- /dev/null +++ b/mapcache/render_config.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +if [[ $# -eq 2 ]]; then + export WMS_HOST=$1 + export WMS_PORT=$2 +else + echo "Usage: ./run.sh <wms_hostname> <wms_port>" + exit 1 +fi + +# Load environment variable options to overwrite in the config +export CACHE_DISK_BASE=${CACHE_DISK_BASE:-"/data/tile_cache"} + +# Update the mapcache.xml file with the real host and port name +mc_tmpl="/work/mapcache_template.xml" +mc_xml="/work/mapcache/mapcache.xml" +mkdir -p $(dirname $mc_xml) +python3 /work/render.py $mc_tmpl > $mc_xml + +# Create the tile directory as root +if [[ ! -d ${CACHE_DISK_BASE} ]]; then + mkdir -p ${CACHE_DISK_BASE} && chown www-data:www-data ${CACHE_DISK_BASE} \ + && chmod 775 ${CACHE_DISK_BASE} +fi + diff --git a/mapcache/run.sh b/mapcache/run.sh index 0a6a344cc1255128d3dcc2d3dacc45663a874c18..baf557cbfc3312e3fa19a6680edd6589341e0ca5 100755 --- a/mapcache/run.sh +++ b/mapcache/run.sh @@ -1,26 +1,8 @@ #!/bin/bash -le -if [[ $# -eq 2 ]]; then - export WMS_HOST=$1 - export WMS_PORT=$2 -else - echo "Usage: ./run.sh <wms_hostname> <wms_port>" - exit 1 -fi - -# Load environment variable options to overwrite in the config -export CACHE_DISK_BASE=${CACHE_DISK_BASE:-"/data/tile_cache"} - -# Update the mapcache.xml file with the real host and port name -mc_tmpl="/work/mapcache_template.xml" -mc_xml="/work/mapcache/mapcache.xml" -mkdir -p $(dirname $mc_xml) -python3 /work/render.py $mc_tmpl > $mc_xml - -# Create the tile directory as root -mkdir -p ${CACHE_DISK_BASE} && chown www-data:www-data ${CACHE_DISK_BASE} \ - && chmod 775 ${CACHE_DISK_BASE} - +# Render the template config to a full config +render_config.sh "$@" +# Make sure that the config is reasonable and print out any errors /usr/sbin/apache2ctl configtest - +# Start apache in the foreground /usr/sbin/apache2ctl -DFOREGROUND diff --git a/mapcache_timedb/Dockerfile b/mapcache_timedb/Dockerfile index a76feaa7bc79630fdaafc31236b4fe5c30084413..8b935cf0f60cacf6bc0bf117f996e48e465e5e37 100644 --- a/mapcache_timedb/Dockerfile +++ b/mapcache_timedb/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapcache:latest ENV DEBIAN_FRONTEND=noninteractive ENV TZ=GMT @@ -6,7 +6,7 @@ ENV LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} ENV LANG=C.UTF-8 ENV LC_ALL=C.UTF-8 -RUN apt-get update && apt-get -y upgrade && apt-get install -y \ +RUN apt-get -y update && apt-get -y upgrade && apt-get install -y \ gosu \ pwgen \ tzdata \ @@ -32,10 +32,10 @@ WORKDIR /work # Install amqpfind RUN apt-get update && apt-get install -y unzip && \ - wget http://ssec.wisc.edu/~rayg/pub/amqpfind.zip && \ + wget https://www.ssec.wisc.edu/~rayg/pub/amqpfind.zip && \ unzip amqpfind.zip && \ rm amqpfind.zip && \ rm -rf /var/lib/apt/lists/* -COPY time_updater.py bin/ -COPY run.sh bin/ +COPY time_updater.py /work +COPY updater.sh /work diff --git a/mapcache_timedb/README.md b/mapcache_timedb/README.md index 75634bf89e23f6ff87458fce1f87aad9d06573a2..ce11cc557cd9f314d57b0d7507650203875a36f7 100644 --- a/mapcache_timedb/README.md +++ b/mapcache_timedb/README.md @@ -9,7 +9,7 @@ docker build -t gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapcache: ## Usage ```bash -docker run -d --rm --name cspp-geo-mapcache-timedb --network cspp-geo-rabbit -v cspp-geo-tile-cache:/data gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapcache_timedb:latest ./bin/run.sh cspp-geo-mapserver 80 +docker run -d --rm --name cspp-geo-mapcache-timedb --network cspp-geo-rabbit -v cspp-geo-tile-cache:/data gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapcache_timedb:latest ./bin/updater.sh cspp-geo-mapserver 80 ``` Note the host name and port of the MapServer passed to the `run.sh` at the end. diff --git a/mapcache_timedb/run.sh b/mapcache_timedb/updater.sh similarity index 61% rename from mapcache_timedb/run.sh rename to mapcache_timedb/updater.sh index 9b8e28b4acd5cd47cac0f9f408efd385617fc2dd..12b07e1b6ecc29e3f60953ffe15d54d25b488290 100755 --- a/mapcache_timedb/run.sh +++ b/mapcache_timedb/updater.sh @@ -23,6 +23,9 @@ export G2G_PRODUCTS=${G2G_PRODUCTS:-"C01 C02 C03 C04 C05 C06 C07 C08 C09 C10 C11 export SATELLITES=${SATELLITES:-"g16 g17"} export INSTRUMENTS=${INSTRUMENTS:-"abi"} export SECTORS=${SECTORS:-"radf radc radm1 radm2"} +export SEED=${SEED:-"0"} # 0 (False) or anything else (True) +export SEED_OVERLAYS=${SEED_OVERLAYS:-"1"} # 0 (False) or anything else (True) +export SEED_ARGS=${SEED_ARGS:-""} json_list_to_bash() { blist=$1 @@ -33,6 +36,24 @@ json_list_to_bash() { } export -f json_list_to_bash +run_mapcache_seed() { + CONFIG="/work/mapcache/mapcache.xml" + NPROCS="4" + TILESET=$1 # g16_abi_radf_l1b_true_color + GRID=$2 # grid_g16_abi_radf + # optional (borders/overlays don't have a time dimension) + T=$3 # YYYY-MM-DDTHH:MM:SS (Z will be added, assumes UTC) + if [[ $T == "" ]]; then + TIME="" + else + TIME="-D TIME=${T}Z" # Add Z timezone + fi + set -x + su -l www-data -s /bin/bash -c "export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH; time mapcache_seed -c $CONFIG -t $TILESET -n $NPROCS -g $GRID $TIME $SEED_ARGS" + set +x +} +export -f run_mapcache_seed + run_time_updater() { if [[ $# -ne 7 ]]; then echo "Unexpected number of arguments (expected 7): $#" @@ -43,7 +64,7 @@ run_time_updater() { satellite_id=${2,,} instrument=${3,,} data_type=${4,,} - product=${5,,} + product=${5} removed_times="$6" added_times="$7" @@ -51,7 +72,7 @@ run_time_updater() { removed_times=$(json_list_to_bash "$removed_times") added_times=$(json_list_to_bash "$added_times") - echo "Starting tile generation processing for ${path}" + echo "Starting cache time updates for ${path}" # if /dst isn't defined then use /data if [[ -d "/dst" ]]; then @@ -64,11 +85,21 @@ run_time_updater() { table_name="${satellite_id}_${instrument}_${data_type}_l1b_${product}" echo "Removing times for: ${dbfile}" echo "Removed times: ${removed_times}" - python3 /work/bin/time_updater.py -r "${dbfile}" "${table_name}" ${removed_times} + python3 /work/time_updater.py -r "${dbfile}" "${table_name}" ${removed_times} echo "Adding times for: ${dbfile}" echo "Added times: ${added_times}" - python3 /work/bin/time_updater.py "${dbfile}" "${table_name}" ${added_times} + python3 /work/time_updater.py "${dbfile}" "${table_name}" ${added_times} echo "Done updating times for: ${dbfile}" + + if [[ ${SEED} -ne 0 ]]; then + tileset="${satellite_id}_${instrument}_${data_type}_l1b_${product}" + grid="grid_${satellite_id}_${instrument}_${data_type}" + for t in ${added_times}; do + echo "Seeding cache for ${tileset} @ ${t}" + run_mapcache_seed $tileset $grid $t + echo "Done seeding" + done + fi } export -f run_time_updater @@ -86,14 +117,29 @@ init_time_updater() { for sector in ${SECTORS}; do dbfile="${dst_dir}/mapcache_time.sqlite" for product in ${G2G_PRODUCTS}; do - product=${product,,} echo "Initializing database ${dbfile} for product ${product}" set -x current_times=$(wget -O - "http://${WMS_HOST}:${WMS_PORT}/wms_times/${sat_id}/${inst}/${sector}/${product}") table_name="${sat_id}_${inst}_${sector}_l1b_${product}" - python3 /work/bin/time_updater.py --json "${dbfile}" "${table_name}" "${current_times}" + python3 /work/time_updater.py --json "${dbfile}" "${table_name}" "${current_times}" set +x done + + # Preseed the overlays, graticules, and basemap layers + if [[ $SEED_OVERLAYS -ne 0 ]]; then + grid="grid_${sat_id}_${inst}_${sector}" + + echo "Seeding basemap layer" + tileset="${sat_id}_${inst}_basemap" + SEED_ARGS="" run_mapcache_seed $tileset $grid + echo "Seeding borders layer" + tileset="${sat_id}_${inst}_borders" + SEED_ARGS="" run_mapcache_seed $tileset $grid + echo "Seeding graticules layer" + tileset="${sat_id}_${inst}_graticules" + SEED_ARGS="" run_mapcache_seed $tileset $grid + echo "Done seeding ${grid}'s overlay layers" + fi done done done @@ -107,6 +153,9 @@ else exit 1 fi +# Render the MapCache config so we can seed the cache later +render_config.sh $WMS_HOST $WMS_PORT + # Initialize the database if it doesn't exist already echo "Initializing time database..." init_time_updater @@ -115,7 +164,6 @@ if [[ $? -ne 0 ]]; then exit 1 fi - echo "Listening to AMQP messages with topic \"$AMQPFIND_TOPIC\"" -python3 amqpfind/amqpfind.py ${AMQPFIND_ARGS} -C "${AMQPFIND_TOPIC}" -j "{satellite_family} {satellite_ID} {instrument} {data_type} {product} \'{removed_times}\' \'{added_times}\'" | xargs -I{} -P3 -n1 bash -c "run_time_updater {}" +python3 amqpfind/amqpfind.py ${AMQPFIND_ARGS} -C "${AMQPFIND_TOPIC}" -j "{satellite_family} {satellite_ID} {instrument} {data_type} {product} \'{removed_times}\' \'{added_times}\'" | xargs -I{} -P4 -n1 bash -c "run_time_updater {}"