Skip to content
Snippets Groups Projects
Verified Commit 953ea8f4 authored by David Hoese's avatar David Hoese
Browse files

Add ability to seed cache when mapcache times are added

parent b6f72198
Branches
No related tags found
No related merge requests found
Pipeline #10465 canceled
Stage: build_image
Stage: registry_cleanup
Show whitespace changes
Inline Side-by-side
mapcache/Dockerfile
+ 2
− 21
View file @ 953ea8f4
...@@ -10,21 +10,6 @@ RUN apt-get -y update && \ ...@@ -10,21 +10,6 @@ RUN apt-get -y update && \
apt-get -y install gcc g++ cmake sqlite3 libsqlite3-dev pkg-config bash-completion curl subversion && \ apt-get -y install gcc g++ cmake sqlite3 libsqlite3-dev pkg-config bash-completion curl subversion && \
apt-get -y clean apt-get -y clean
# FIXME: Remove if/when upstream image builds with fcgi
# Install Mapserver (unstable)
#RUN mkdir -p /build_deps && cd /build_deps \
# && apt-get -y update \
## && apt-get install -y libprotobuf17 zlib1g-dev libpng-dev libjpeg-turbo8 libfreetype6 libfribidi-dev libharfbuzz-dev libcairo2-dev libfcgi-dev libgeos++-dev postgresql postgis libxml2-dev libgif-dev libjpeg-turbo8-dev libprotobuf-dev protobuf-compiler libprotobuf-c-dev libprotobuf-c1 libprotobuf-dev protobuf-c-compiler \
# && apt-get install -y zlib1g-dev libpng-dev libjpeg-turbo8 libfreetype6 libfribidi-dev libharfbuzz-dev libcairo2-dev libfcgi-dev libgeos++-dev postgresql postgis libxml2-dev libgif-dev libjpeg-turbo8-dev libprotobuf-dev protobuf-compiler libprotobuf-c-dev libprotobuf-c1 libprotobuf-dev protobuf-c-compiler \
# && apt-get -y clean \
# && git clone https://github.com/mapserver/mapserver.git && cd mapserver \
# && git checkout 0fcc810f0b559c800f950db78a79fa6574799f23 \
# && mkdir -p build && cd build \
# && cmake .. -DWITH_PROTOBUFC=OFF -DWITH_POSTGIS=OFF \
# && make -j$(nproc) \
# && make install
# apache # apache
# http://www.inanzzz.com/index.php/post/rhsb/running-apache-server-as-foreground-on-ubuntu-with-dockerfile # http://www.inanzzz.com/index.php/post/rhsb/running-apache-server-as-foreground-on-ubuntu-with-dockerfile
# add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added
...@@ -66,16 +51,12 @@ COPY custom-conf /etc/apache2/conf-available/cspp_geo.conf ...@@ -66,16 +51,12 @@ COPY custom-conf /etc/apache2/conf-available/cspp_geo.conf
# disable the default which would conflict with our custom # disable the default which would conflict with our custom
RUN a2enconf cspp_geo RUN a2enconf cspp_geo
# Point apache to the mapserver binary
#RUN ln -s /usr/local/bin/mapserv /usr/lib/cgi-bin/mapserv && \
# chown ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/* && \
# chown -h ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/*
RUN pip3 install jinja2 RUN pip3 install jinja2
COPY render.py /work/ COPY render.py /work/
COPY mapcache_template.xml /work/ COPY mapcache_template.xml /work/
COPY run.sh /work/ COPY run.sh render_config.sh /work/
COPY html/ /var/www/html/ COPY html/ /var/www/html/
ENV PATH /work:$PATH
# Add our own custom EPSG codes (HACK) # Add our own custom EPSG codes (HACK)
# GOES-16 ABI Full Disk = EPSG:930916 # GOES-16 ABI Full Disk = EPSG:930916
... ...
......
mapcache/render_config.sh 0 → 100755
+ 25
− 0
View file @ 953ea8f4
#!/usr/bin/env bash
if [[ $# -eq 2 ]]; then
export WMS_HOST=$1
export WMS_PORT=$2
else
echo "Usage: ./run.sh <wms_hostname> <wms_port>"
exit 1
fi
# Load environment variable options to overwrite in the config
export CACHE_DISK_BASE=${CACHE_DISK_BASE:-"/data/tile_cache"}
# Update the mapcache.xml file with the real host and port name
mc_tmpl="/work/mapcache_template.xml"
mc_xml="/work/mapcache/mapcache.xml"
mkdir -p $(dirname $mc_xml)
python3 /work/render.py $mc_tmpl > $mc_xml
# Create the tile directory as root
if [[ ! -d ${CACHE_DISK_BASE} ]]; then
mkdir -p ${CACHE_DISK_BASE} && chown www-data:www-data ${CACHE_DISK_BASE} \
&& chmod 775 ${CACHE_DISK_BASE}
fi
mapcache/run.sh
+ 4
− 22
View file @ 953ea8f4
#!/bin/bash -le #!/bin/bash -le
if [[ $# -eq 2 ]]; then # Render the template config to a full config
export WMS_HOST=$1 render_config.sh "$@"
export WMS_PORT=$2 # Make sure that the config is reasonable and print out any errors
else
echo "Usage: ./run.sh <wms_hostname> <wms_port>"
exit 1
fi
# Load environment variable options to overwrite in the config
export CACHE_DISK_BASE=${CACHE_DISK_BASE:-"/data/tile_cache"}
# Update the mapcache.xml file with the real host and port name
mc_tmpl="/work/mapcache_template.xml"
mc_xml="/work/mapcache/mapcache.xml"
mkdir -p $(dirname $mc_xml)
python3 /work/render.py $mc_tmpl > $mc_xml
# Create the tile directory as root
mkdir -p ${CACHE_DISK_BASE} && chown www-data:www-data ${CACHE_DISK_BASE} \
&& chmod 775 ${CACHE_DISK_BASE}
/usr/sbin/apache2ctl configtest /usr/sbin/apache2ctl configtest
# Start apache in the foreground
/usr/sbin/apache2ctl -DFOREGROUND /usr/sbin/apache2ctl -DFOREGROUND
mapcache_timedb/Dockerfile
+ 5
− 5
View file @ 953ea8f4
FROM ubuntu:18.04 FROM gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapcache:latest
ENV DEBIAN_FRONTEND=noninteractive ENV DEBIAN_FRONTEND=noninteractive
ENV TZ=GMT ENV TZ=GMT
...@@ -6,7 +6,7 @@ ENV LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} ...@@ -6,7 +6,7 @@ ENV LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH}
ENV LANG=C.UTF-8 ENV LANG=C.UTF-8
ENV LC_ALL=C.UTF-8 ENV LC_ALL=C.UTF-8
RUN apt-get update && apt-get -y upgrade && apt-get install -y \ RUN apt-get -y update && apt-get -y upgrade && apt-get install -y \
gosu \ gosu \
pwgen \ pwgen \
tzdata \ tzdata \
...@@ -32,10 +32,10 @@ WORKDIR /work ...@@ -32,10 +32,10 @@ WORKDIR /work
# Install amqpfind # Install amqpfind
RUN apt-get update && apt-get install -y unzip && \ RUN apt-get update && apt-get install -y unzip && \
wget http://ssec.wisc.edu/~rayg/pub/amqpfind.zip && \ wget https://www.ssec.wisc.edu/~rayg/pub/amqpfind.zip && \
unzip amqpfind.zip && \ unzip amqpfind.zip && \
rm amqpfind.zip && \ rm amqpfind.zip && \
rm -rf /var/lib/apt/lists/* rm -rf /var/lib/apt/lists/*
COPY time_updater.py bin/ COPY time_updater.py /work
COPY run.sh bin/ COPY updater.sh /work
mapcache_timedb/README.md
+ 1
− 1
View file @ 953ea8f4
...@@ -9,7 +9,7 @@ docker build -t gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapcache: ...@@ -9,7 +9,7 @@ docker build -t gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapcache:
## Usage ## Usage
```bash ```bash
docker run -d --rm --name cspp-geo-mapcache-timedb --network cspp-geo-rabbit -v cspp-geo-tile-cache:/data gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapcache_timedb:latest ./bin/run.sh cspp-geo-mapserver 80 docker run -d --rm --name cspp-geo-mapcache-timedb --network cspp-geo-rabbit -v cspp-geo-tile-cache:/data gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapcache_timedb:latest ./bin/updater.sh cspp-geo-mapserver 80
``` ```
Note the host name and port of the MapServer passed to the `run.sh` at the end. Note the host name and port of the MapServer passed to the `run.sh` at the end.
... ...
......
mapcache_timedb/run.sh → mapcache_timedb/updater.sh
+ 56
− 8
View file @ 953ea8f4
...@@ -23,6 +23,9 @@ export G2G_PRODUCTS=${G2G_PRODUCTS:-"C01 C02 C03 C04 C05 C06 C07 C08 C09 C10 C11 ...@@ -23,6 +23,9 @@ export G2G_PRODUCTS=${G2G_PRODUCTS:-"C01 C02 C03 C04 C05 C06 C07 C08 C09 C10 C11
export SATELLITES=${SATELLITES:-"g16 g17"} export SATELLITES=${SATELLITES:-"g16 g17"}
export INSTRUMENTS=${INSTRUMENTS:-"abi"} export INSTRUMENTS=${INSTRUMENTS:-"abi"}
export SECTORS=${SECTORS:-"radf radc radm1 radm2"} export SECTORS=${SECTORS:-"radf radc radm1 radm2"}
export SEED=${SEED:-"0"} # 0 (False) or anything else (True)
export SEED_OVERLAYS=${SEED_OVERLAYS:-"1"} # 0 (False) or anything else (True)
export SEED_ARGS=${SEED_ARGS:-""}
json_list_to_bash() { json_list_to_bash() {
blist=$1 blist=$1
...@@ -33,6 +36,24 @@ json_list_to_bash() { ...@@ -33,6 +36,24 @@ json_list_to_bash() {
} }
export -f json_list_to_bash export -f json_list_to_bash
run_mapcache_seed() {
CONFIG="/work/mapcache/mapcache.xml"
NPROCS="4"
TILESET=$1 # g16_abi_radf_l1b_true_color
GRID=$2 # grid_g16_abi_radf
# optional (borders/overlays don't have a time dimension)
T=$3 # YYYY-MM-DDTHH:MM:SS (Z will be added, assumes UTC)
if [[ $T == "" ]]; then
TIME=""
else
TIME="-D TIME=${T}Z" # Add Z timezone
fi
set -x
su -l www-data -s /bin/bash -c "export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH; time mapcache_seed -c $CONFIG -t $TILESET -n $NPROCS -g $GRID $TIME $SEED_ARGS"
set +x
}
export -f run_mapcache_seed
run_time_updater() { run_time_updater() {
if [[ $# -ne 7 ]]; then if [[ $# -ne 7 ]]; then
echo "Unexpected number of arguments (expected 7): $#" echo "Unexpected number of arguments (expected 7): $#"
...@@ -43,7 +64,7 @@ run_time_updater() { ...@@ -43,7 +64,7 @@ run_time_updater() {
satellite_id=${2,,} satellite_id=${2,,}
instrument=${3,,} instrument=${3,,}
data_type=${4,,} data_type=${4,,}
product=${5,,} product=${5}
removed_times="$6" removed_times="$6"
added_times="$7" added_times="$7"
...@@ -51,7 +72,7 @@ run_time_updater() { ...@@ -51,7 +72,7 @@ run_time_updater() {
removed_times=$(json_list_to_bash "$removed_times") removed_times=$(json_list_to_bash "$removed_times")
added_times=$(json_list_to_bash "$added_times") added_times=$(json_list_to_bash "$added_times")
echo "Starting tile generation processing for ${path}" echo "Starting cache time updates for ${path}"
# if /dst isn't defined then use /data # if /dst isn't defined then use /data
if [[ -d "/dst" ]]; then if [[ -d "/dst" ]]; then
...@@ -64,11 +85,21 @@ run_time_updater() { ...@@ -64,11 +85,21 @@ run_time_updater() {
table_name="${satellite_id}_${instrument}_${data_type}_l1b_${product}" table_name="${satellite_id}_${instrument}_${data_type}_l1b_${product}"
echo "Removing times for: ${dbfile}" echo "Removing times for: ${dbfile}"
echo "Removed times: ${removed_times}" echo "Removed times: ${removed_times}"
python3 /work/bin/time_updater.py -r "${dbfile}" "${table_name}" ${removed_times} python3 /work/time_updater.py -r "${dbfile}" "${table_name}" ${removed_times}
echo "Adding times for: ${dbfile}" echo "Adding times for: ${dbfile}"
echo "Added times: ${added_times}" echo "Added times: ${added_times}"
python3 /work/bin/time_updater.py "${dbfile}" "${table_name}" ${added_times} python3 /work/time_updater.py "${dbfile}" "${table_name}" ${added_times}
echo "Done updating times for: ${dbfile}" echo "Done updating times for: ${dbfile}"
if [[ ${SEED} -ne 0 ]]; then
tileset="${satellite_id}_${instrument}_${data_type}_l1b_${product}"
grid="grid_${satellite_id}_${instrument}_${data_type}"
for t in ${added_times}; do
echo "Seeding cache for ${tileset} @ ${t}"
run_mapcache_seed $tileset $grid $t
echo "Done seeding"
done
fi
} }
export -f run_time_updater export -f run_time_updater
...@@ -86,14 +117,29 @@ init_time_updater() { ...@@ -86,14 +117,29 @@ init_time_updater() {
for sector in ${SECTORS}; do for sector in ${SECTORS}; do
dbfile="${dst_dir}/mapcache_time.sqlite" dbfile="${dst_dir}/mapcache_time.sqlite"
for product in ${G2G_PRODUCTS}; do for product in ${G2G_PRODUCTS}; do
product=${product,,}
echo "Initializing database ${dbfile} for product ${product}" echo "Initializing database ${dbfile} for product ${product}"
set -x set -x
current_times=$(wget -O - "http://${WMS_HOST}:${WMS_PORT}/wms_times/${sat_id}/${inst}/${sector}/${product}") current_times=$(wget -O - "http://${WMS_HOST}:${WMS_PORT}/wms_times/${sat_id}/${inst}/${sector}/${product}")
table_name="${sat_id}_${inst}_${sector}_l1b_${product}" table_name="${sat_id}_${inst}_${sector}_l1b_${product}"
python3 /work/bin/time_updater.py --json "${dbfile}" "${table_name}" "${current_times}" python3 /work/time_updater.py --json "${dbfile}" "${table_name}" "${current_times}"
set +x set +x
done done
# Preseed the overlays, graticules, and basemap layers
if [[ $SEED_OVERLAYS -ne 0 ]]; then
grid="grid_${sat_id}_${inst}_${sector}"
echo "Seeding basemap layer"
tileset="${sat_id}_${inst}_basemap"
SEED_ARGS="" run_mapcache_seed $tileset $grid
echo "Seeding borders layer"
tileset="${sat_id}_${inst}_borders"
SEED_ARGS="" run_mapcache_seed $tileset $grid
echo "Seeding graticules layer"
tileset="${sat_id}_${inst}_graticules"
SEED_ARGS="" run_mapcache_seed $tileset $grid
echo "Done seeding ${grid}'s overlay layers"
fi
done done
done done
done done
...@@ -107,6 +153,9 @@ else ...@@ -107,6 +153,9 @@ else
exit 1 exit 1
fi fi
# Render the MapCache config so we can seed the cache later
render_config.sh $WMS_HOST $WMS_PORT
# Initialize the database if it doesn't exist already # Initialize the database if it doesn't exist already
echo "Initializing time database..." echo "Initializing time database..."
init_time_updater init_time_updater
...@@ -115,7 +164,6 @@ if [[ $? -ne 0 ]]; then ...@@ -115,7 +164,6 @@ if [[ $? -ne 0 ]]; then
exit 1 exit 1
fi fi
echo "Listening to AMQP messages with topic \"$AMQPFIND_TOPIC\"" echo "Listening to AMQP messages with topic \"$AMQPFIND_TOPIC\""
python3 amqpfind/amqpfind.py ${AMQPFIND_ARGS} -C "${AMQPFIND_TOPIC}" -j "{satellite_family} {satellite_ID} {instrument} {data_type} {product} \'{removed_times}\' \'{added_times}\'" | xargs -I{} -P3 -n1 bash -c "run_time_updater {}" python3 amqpfind/amqpfind.py ${AMQPFIND_ARGS} -C "${AMQPFIND_TOPIC}" -j "{satellite_family} {satellite_ID} {instrument} {data_type} {product} \'{removed_times}\' \'{added_times}\'" | xargs -I{} -P4 -n1 bash -c "run_time_updater {}"
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment