ci/test_mapserver_image.sh

@@ -8,13 +8,16 @@ error() {
     echo >&2 "ERROR: $@"
 }
 
-if [[ $# -ne 2 ]]; then
-    error "Usage: ./test_mapserver_image.sh <image_url> <image_tag>"
+if [[ $# -eq 1 ]]; then
+    image=$1
+elif [[ $# -eq 2 ]]; then
+    # deprecated: use single URL:tag from now on
+    image="$1:$2"
+else
+    error "Usage: ./test_mapserver_image.sh <image_url:image_tag>"
     exit 1
 fi
 
-image_url=$1
-image_tag=$2
 MINIO_SERVER_NAME="test-minio-server"
 export AWS_ACCESS_KEY_ID="minioadmin"
 export AWS_SECRET_ACCESS_KEY="minioadmin"
@@ -44,7 +47,7 @@ teardown_test() {
     kill_postgres || debug "Could not kill postgres container"
     kill_minio || debug "Could not kill minio container"
     if [ -d $base_tmp_dir ]; then
-        docker run --rm --name tmpremover -v ${base_tmp_dir}:/dst ${image_url}:${image_tag} bash -c "chmod -R 777 /dst"
+        docker run --rm --name tmpremover -v ${base_tmp_dir}:/dst ${image} bash -c "chmod -R 777 /dst"
         rm -rf $base_tmp_dir || debug "Could not remove temporary directory: ${base_tmp_dir}"
     fi
     docker network rm ${NETWORK_NAME} > /dev/null || debug "Could not remove docker network"
@@ -57,12 +60,14 @@ graceful_exit() {
 }
 
 start_test_container() {
-    debug "Starting test docker container (${image_url}:${image_tag})..."
-    docker run --rm -d --network ${NETWORK_NAME} --name test -p 8888:80 $@ ${image_url}:${image_tag}
+    debug "Starting test docker container (${image})..."
+    docker run --rm -d --network ${NETWORK_NAME} --name test -p 8888:80 $@ ${image}
     start_status=$?
     # just wait a bit to let the server start
     sleep 2
     debug "Container started."
+    debug "Installing rasterio into test container for geotiff creation"
+    docker exec -i test python3 -m pip install --break-system-packages --root-user-action ignore rasterio fiona shapely
     return $start_status
 }
 
@@ -225,8 +230,8 @@ create_fake_s3_geotiff() {
   gtiff_fn=$(basename $fn)
   bucket_name=$(basename $(dirname $fn))
 
-  aws s3api create-bucket --endpoint-url "http://localhost:9000" --bucket "${bucket_name}" 1>&2
-  aws s3 cp --endpoint-url "http://localhost:9000" "${fn}" s3://${bucket_name}/ 1>&2
+  aws s3api create-bucket --endpoint-url "http://localhost:9000" --bucket "${bucket_name}" 1>&2 || error "Can't create MinIO S3 bucket"
+  aws s3 cp --endpoint-url "http://localhost:9000" "${fn}" s3://${bucket_name}/ 1>&2 || error "Can't copy geotiff to S3 bucket"
   echo "${bucket_name}/${gtiff_fn}"
 }
 
@@ -261,7 +266,7 @@ check_image_content() {
         cat $img_filename
         return 1
     fi
-    docker exec -i test pip install pillow >/dev/null
+    docker exec -i test python3 -m pip install --break-system-packages --root-user-action ignore pillow >/dev/null
     incontainer_file="/tmp/$(basename $img_filename)"
     docker cp $img_filename test:${incontainer_file}
     docker exec -i test python3 <<EOF

mapserver/Dockerfile

-FROM ubuntu:20.04
+FROM ubuntu:24.04 AS build
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV TZ=GMT
@@ -7,15 +7,13 @@ ENV LANG=C.UTF-8
 ENV LC_ALL=C.UTF-8
 
 RUN apt-get update && apt-get install -y \
-  gosu \
-  pwgen \
   tzdata \
   gcc \
   g++ \
   automake \
   build-essential \
   cmake \
-  sqlite \
+  sqlite3 \
   libsqlite3-dev \
   libxml2-dev \
   libjpeg-dev \
@@ -25,45 +23,33 @@ RUN apt-get update && apt-get install -y \
   libfreetype6 \
   libfreetype6-dev \
   libzstd-dev \
-  libnetcdf-dev \
-  python3-pip \
   pkg-config \
   git \
   wget \
   curl \
-  unzip \
   libcurl4 \
   libcurl4-openssl-dev \
   zlib1g-dev \
   libfribidi-dev \
   libharfbuzz-dev \
-  libcairo2-dev \
   libfcgi-dev \
   libgeos++-dev \
   libpq-dev \
-  postgresql-server-dev-all \
-  libxml2-dev \
   libgif-dev \
-  libprotobuf-dev \
-  protobuf-compiler \
-  libprotobuf-c-dev \
-  libprotobuf-c1 \
-  libprotobuf-dev \
-  protobuf-c-compiler \
   && rm -rf /var/lib/apt/lists/*
 
 # Install libtiff
 RUN mkdir -p /build_deps && cd /build_deps \
-  && wget --no-check-certificate https://download.osgeo.org/libtiff/tiff-4.3.0.tar.gz \
-  && tar -zxf tiff-4.3.0.tar.gz \
-  && cd tiff-4.3.0 \
+  && wget --no-check-certificate http://www.libtiff.org/downloads/tiff-4.6.0t.tar.gz \
+  && tar -zxf tiff-4.6.0t.tar.gz \
+  && cd tiff-4.6.0t \
   && ./configure \
   && make \
   && make install
 
 # Install Proj
 RUN cd /build_deps \
-  && git clone https://github.com/OSGeo/PROJ.git -b 8.2.1 && cd PROJ \
+  && git clone https://github.com/OSGeo/PROJ.git -b 9.4.1 && cd PROJ \
   && mkdir -p build && cd build \
   && cmake .. \
   && make -j$(nproc) \
@@ -71,9 +57,9 @@ RUN cd /build_deps \
 
 # Install libgeotiff
 RUN cd /build_deps \
-  && wget --no-check-certificate https://download.osgeo.org/geotiff/libgeotiff/libgeotiff-1.7.1.tar.gz \
-  && tar -zxf libgeotiff-1.7.1.tar.gz \
-  && cd libgeotiff-1.7.1 \
+  && wget --no-check-certificate https://github.com/OSGeo/libgeotiff/releases/download/1.7.3/libgeotiff-1.7.3.tar.gz \
+  && tar -zxf libgeotiff-1.7.3.tar.gz \
+  && cd libgeotiff-1.7.3 \
   && mkdir -p build && cd build \
   && cmake .. \
   && make \
@@ -81,29 +67,34 @@ RUN cd /build_deps \
 
 # Install GDAL
 RUN cd /build_deps \
-  && git clone --depth 1 --branch v3.5.0alpha1 https://github.com/OSGeo/gdal.git && cd gdal \
-  && ./autogen.sh \
-  && ./configure --with-crypto=no --with-curl=yes --with-netcdf=yes \
-  && make -j$(nproc) \
-  && make install
-
-# Install Cython, Rasterio, and Fiona
-RUN cd /build_deps && pip3 install cython rasterio fiona shapely pyproj
+  && git clone --depth 1 --branch v3.9.1 https://github.com/OSGeo/gdal.git && cd gdal \
+  && mkdir build && cd build \
+  && cmake -DGDAL_USE_CURL=ON -DGDAL_USE_CRYPTOPP=OFF -DGDAL_USE_NETCDF=OFF .. \
+  && cmake --build . \
+  && cmake --build . --target install
 
 # Install Mapserver
 RUN cd /build_deps \
   && git clone https://github.com/mapserver/mapserver.git && cd mapserver \
-  && git checkout rel-7-6-4 \
+  && git checkout rel-8-2-0 \
   && mkdir -p build && cd build \
   && cmake .. -DWITH_GIF=OFF -DWITH_HARFBUZZ=OFF -DWITH_PROTOBUFC=OFF -DWITH_FRIBIDI=OFF -DWITH_POSTGIS=ON -DWITH_GEOS=OFF -DWITH_FCGI=ON -DWITH_CAIRO=OFF \
   && make -j$(nproc) \
   && make install
 
-# Clean up
-RUN cd /tmp && rm -r /build_deps
+FROM ubuntu:24.04 AS proj-update
 
+COPY --from=build /usr/local/share/proj /usr/local/share/proj
 
+RUN apt-get update && apt-get install -y sqlite3
+
+# Add our own custom EPSG codes (HACK)
+# GOES-16 ABI Full Disk = EPSG:930916
+# GOES-17 ABI Full Disk = EPSG:930917
+COPY sql/ /work/sql/
+RUN sqlite3 -bail /usr/local/share/proj/proj.db < /work/sql/goesr_crs.sql
 
+FROM ubuntu:24.04 AS main
 
 WORKDIR /work
 
@@ -118,23 +109,28 @@ ENV APACHE_PID_FILE /var/run/apache2/apache2.pid
 ENV APACHE_SERVER_NAME localhost
 #RUN groupadd -r www-data && useradd -r --create-home -g www-data www-data
 
-# install httpd runtime dependencies
+# install httpd and other runtime dependencies
 # https://httpd.apache.org/docs/2.4/install.html#requirements
-RUN apt-get update && apt-get -y install apache2 libapache2-mod-fcgid libapache2-mod-php php-common php-cli php-fpm php && \
+RUN apt-get update && apt-get -y install apache2 libapache2-mod-fcgid curl unzip python3-pip && \
     ls /etc/apache2/mods-available && \
     ls /etc/apache2/mods-enabled && \
     a2enmod actions proxy_fcgi setenvif cgi fcgid rewrite && \
-    a2enconf php7.4-fpm serve-cgi-bin && \
+    a2enconf serve-cgi-bin && \
     apt-get -y clean && \
     rm -rf /var/lib/apt/lists/*
 
-RUN curl -LOk https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_countries.zip \
-  && curl -LOk https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_1_states_provinces_lines.zip \
-  && curl -LOk https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_lakes.zip \
-  && curl -LOk https://www2.census.gov/geo/tiger/GENZ2018/shp/cb_2018_us_county_500k.zip \
-  && curl -LOk https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_graticules_1.zip \
-  && curl -LOk https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_graticules_5.zip \
-  && curl -LOk https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_graticules_10.zip \
+COPY --from=build /usr/local/bin /usr/local/bin
+COPY --from=proj-update /usr/local/share/proj /usr/local/share/proj
+COPY --from=build /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu
+COPY --from=build /usr/local/lib/libgdal* /usr/local/lib/libproj* /usr/local/lib/libtiff* /usr/local/lib/libgeotiff* /usr/local/lib/libmapserver* /usr/local/lib/
+
+RUN curl -LO -e https://www.naturalearthdata.com https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_countries.zip \
+  && curl -LO -e https://www.naturalearthdata.com https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_1_states_provinces_lines.zip \
+  && curl -LO -e https://www.naturalearthdata.com https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_lakes.zip \
+  && curl -LO -e https://www.naturalearthdata.com https://www2.census.gov/geo/tiger/GENZ2018/shp/cb_2018_us_county_500k.zip \
+  && curl -LO -e https://www.naturalearthdata.com https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_graticules_1.zip \
+  && curl -LO -e https://www.naturalearthdata.com https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_graticules_5.zip \
+  && curl -LO -e https://www.naturalearthdata.com https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_graticules_10.zip \
   && mkdir -p /work/shapefiles \
   && unzip -d /work/shapefiles ne_10m_admin_0_countries.zip \
   && unzip -d /work/shapefiles ne_10m_admin_1_states_provinces_lines.zip \
@@ -168,19 +164,14 @@ RUN ln -s /usr/local/bin/mapserv /usr/lib/cgi-bin/mapserv && \
     chown ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/* && \
     chown -h ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/*
 
-RUN pip3 install jinja2 psycopg2 && rm -r /root/.cache/pip
+RUN python3 -m pip install --break-system-packages --root-user-action ignore jinja2 && rm -r /root/.cache/pip
 
 COPY render.py run.sh abi_l1b_template.map /work/
+COPY mapserver.conf /usr/local/etc/mapserver.conf
 COPY mapfiles/ /work/mapfiles/
 COPY html/ /var/www/html/
 COPY default_settings.yaml /work/geosphere_settings.yaml
 
-# Add our own custom EPSG codes (HACK)
-# GOES-16 ABI Full Disk = EPSG:930916
-# GOES-17 ABI Full Disk = EPSG:930917
-COPY sql/ /work/sql/
-RUN sqlite3 -bail /usr/local/share/proj/proj.db < /work/sql/goesr_crs.sql
-
 # Check the config before we finish
 RUN apache2ctl configtest
 

mapserver/mapserver.conf

+CONFIG
+    ENV
+        MS_MAP_PATTERN "^/work/mapfiles"
+    END
+
+    MAPS
+    END
+
+    PLUGINS
+    END
+END

mapserver/site-conf

@@ -32,6 +32,7 @@
         # helper function to lowercase product name
         RewriteMap lc int:tolower
 
+        SetEnv MAPSERVER_CONFIG_FILE /usr/local/etc/mapserver.conf
         # /wms/g16/abi/radf/l1b?...
         # /wms/g16/abi/radf/borders?...
         # FCGI version of the URL

mapserver/tiledb_geospatial_Dockerfile

-# Based on
-
-FROM ubuntu:18.04
-LABEL maintainer="support@tiledb.io"
-
-ENV DEBIAN_FRONTEND=noninteractive
-ENV TZ=GMT
-ENV LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH}
-ENV LANG=C.UTF-8
-ENV LC_ALL=C.UTF-8
-
-RUN apt-get update && apt-get install -y \
-  gosu \
-  pwgen \
-  tzdata \
-  gcc \
-  g++ \
-  build-essential \
-  cmake \
-  sqlite \
-  libsqlite3-dev \
-  libxml2-dev \
-  libjpeg-dev \
-  libpng-dev \
-  libfreetype6-dev \
-  libzstd-dev \
-  python3-pip \
-  git \
-  wget \
-  && rm -rf /var/lib/apt/lists/*
-
-# Install tiledb using 1.7.2 release
-RUN mkdir -p /build_deps && cd /build_deps \
-  && git clone https://github.com/TileDB-Inc/TileDB.git -b 1.7.2 && cd TileDB \
-  && mkdir -p build && cd build \
-  && cmake -DTILEDB_VERBOSE=ON -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. \
-  && make -j$(nproc) \
-  && make -C tiledb install
-
-# Install curl after building tiledb
-RUN apt-get update && apt-get install -y \
-  libcurl4 \
-  libcurl4-openssl-dev \
-  && rm -rf /var/lib/apt/lists/*
-
-# Install OpenJPEG
-RUN cd /build_deps \
-  && git clone https://github.com/uclouvain/openjpeg.git -b v2.2.0 && cd openjpeg \
-  && mkdir -p build && cd build \
-  && cmake .. \
-  && make -j$(nproc) \
-  && make install
-
-# Install libtiff
-RUN cd /build_deps \
-  && wget --no-check-certificate https://download.osgeo.org/libtiff/tiff-4.1.0.tar.gz \
-  && tar -zxf tiff-4.1.0.tar.gz \
-  && cd tiff-4.1.0 \
-  && ./configure \
-  && make \
-  && make install
-
-# Install Proj
-RUN cd /build_deps \
-  && git clone https://github.com/OSGeo/PROJ.git -b 6.2.1 && cd PROJ \
-  && mkdir -p build && cd build \
-  && cmake .. \
-  && make -j$(nproc) \
-  && make install
-
-# Install libgeotiff
-RUN cd /build_deps \
-  && wget --no-check-certificate https://download.osgeo.org/geotiff/libgeotiff/libgeotiff-1.5.1.tar.gz \
-  && tar -zxf libgeotiff-1.5.1.tar.gz \
-  && cd libgeotiff-1.5.1 \
-  && mkdir -p build && cd build \
-  && cmake .. \
-  && make \
-  && make install
-
-# Install GDAL
-RUN cd /build_deps \
-  && git clone https://github.com/OSGeo/gdal.git && cd gdal/gdal \
-  && git checkout c99a871a7bdedc751c503bb8cf508d9016510fe0 \
-  && ./configure --with-crypto=no --with-curl=no \
-  && make -j$(nproc) \
-  && make install
-
-## Install TileDB-Py
-RUN cd /build_deps \
-  && pip3 install numpy \
-  && git clone https://github.com/TileDB-Inc/TileDB-Py.git -b 0.5.3 \
-  && cd TileDB-Py && python3 setup.py install
-
-## Install XArray
-RUN cd /build_deps && pip3 install xarray
-
-## Install Dask
-RUN cd /build_deps \
-  && pip3 install toolz && pip3 install dask_image \
-  && git clone https://github.com/dask/dask.git && cd dask \
-  && git checkout 807f3225cf840f28ce7cf89b88fea63d473889e7 \
-  && python3 setup.py install \
-  && pip3 install dask distributed --upgrade \
-  && pip3 install dask-image
-
-# Install Rasterio
-RUN cd /build_deps && pip3 install cython
-RUN cd /build_deps \
-  && git clone https://github.com/mapbox/rasterio.git -b 1.1.0 && cd rasterio \
-  && python3 setup.py install
-
-# Install Fiona
-RUN cd /build_deps \
-  && git clone https://github.com/Toblerity/Fiona.git && cd Fiona \
-  && python3 setup.py install
-
-# Install TileDB-SAR
-RUN cd /build_deps \
-  && git clone https://github.com/TileDB-Inc/TileDB-SAR.git && cd TileDB-SAR \
-  && git checkout 888059a15d87ae95fff6dc01c8bd4343ee4eaee1 \
-  && python3 setup.py install
-
-# Install Mapserver
-RUN cd /build_deps \
-  && git clone https://github.com/mapserver/mapserver.git && cd mapserver \
-  && git checkout 0fcc810f0b559c800f950db78a79fa6574799f23 \
-  && mkdir -p build && cd build \
-  && cmake .. -DWITH_GIF=OFF -DWITH_HARFBUZZ=OFF -DWITH_PROTOBUFC=OFF -DWITH_FRIBIDI=OFF -DWITH_POSTGIS=OFF -DWITH_GEOS=OFF -DWITH_FCGI=OFF -DWITH_CAIRO=OFF \
-  && make \
-  && make install
-
-# Install LasZIP
-RUN cd /build_deps \
-  && wget https://github.com/LASzip/LASzip/releases/download/3.4.1/laszip-src-3.4.1.tar.gz \
-  && tar -zxf laszip-src-3.4.1.tar.gz \
-  && cd laszip-src-3.4.1 \
-  && mkdir -p build && cd build \
-  && cmake .. \
-  && make \
-  && make install
-
-# Install PDAL
-RUN cd /build_deps \
-  && git clone https://github.com/PDAL/PDAL.git -b 2.0.1 && cd PDAL \
-  && mkdir -p build && cd build \
-  && cmake .. \
-  && make \
-  && make install
-
-# Install PDAL Python
-RUN pip3 install packaging \
-  && git clone https://github.com/PDAL/python pdalextension \
-  && cd pdalextension \
-  && python3 setup.py build \
-  && python3 setup.py install
-
-# Clean up
-RUN cd /tmp && rm -r /build_deps