diff --git a/ci/test_mapserver_image.sh b/ci/test_mapserver_image.sh index 0398313022a1e39d6264c7946c4abb4bd23c01a9..f781a9c9b1e92a2b9cbe862fa4b6b29964556d7c 100755 --- a/ci/test_mapserver_image.sh +++ b/ci/test_mapserver_image.sh @@ -8,13 +8,16 @@ error() { echo >&2 "ERROR: $@" } -if [[ $# -ne 2 ]]; then - error "Usage: ./test_mapserver_image.sh <image_url> <image_tag>" +if [[ $# -eq 1 ]]; then + image=$1 +elif [[ $# -eq 2 ]]; then + # deprecated: use single URL:tag from now on + image="$1:$2" +else + error "Usage: ./test_mapserver_image.sh <image_url:image_tag>" exit 1 fi -image_url=$1 -image_tag=$2 MINIO_SERVER_NAME="test-minio-server" export AWS_ACCESS_KEY_ID="minioadmin" export AWS_SECRET_ACCESS_KEY="minioadmin" @@ -44,7 +47,7 @@ teardown_test() { kill_postgres || debug "Could not kill postgres container" kill_minio || debug "Could not kill minio container" if [ -d $base_tmp_dir ]; then - docker run --rm --name tmpremover -v ${base_tmp_dir}:/dst ${image_url}:${image_tag} bash -c "chmod -R 777 /dst" + docker run --rm --name tmpremover -v ${base_tmp_dir}:/dst ${image} bash -c "chmod -R 777 /dst" rm -rf $base_tmp_dir || debug "Could not remove temporary directory: ${base_tmp_dir}" fi docker network rm ${NETWORK_NAME} > /dev/null || debug "Could not remove docker network" @@ -57,12 +60,14 @@ graceful_exit() { } start_test_container() { - debug "Starting test docker container (${image_url}:${image_tag})..." - docker run --rm -d --network ${NETWORK_NAME} --name test -p 8888:80 $@ ${image_url}:${image_tag} + debug "Starting test docker container (${image})..." + docker run --rm -d --network ${NETWORK_NAME} --name test -p 8888:80 $@ ${image} start_status=$? # just wait a bit to let the server start sleep 2 debug "Container started." + debug "Installing rasterio into test container for geotiff creation" + docker exec -i test python3 -m pip install --break-system-packages --root-user-action ignore rasterio fiona shapely return $start_status } @@ -225,8 +230,8 @@ create_fake_s3_geotiff() { gtiff_fn=$(basename $fn) bucket_name=$(basename $(dirname $fn)) - aws s3api create-bucket --endpoint-url "http://localhost:9000" --bucket "${bucket_name}" 1>&2 - aws s3 cp --endpoint-url "http://localhost:9000" "${fn}" s3://${bucket_name}/ 1>&2 + aws s3api create-bucket --endpoint-url "http://localhost:9000" --bucket "${bucket_name}" 1>&2 || error "Can't create MinIO S3 bucket" + aws s3 cp --endpoint-url "http://localhost:9000" "${fn}" s3://${bucket_name}/ 1>&2 || error "Can't copy geotiff to S3 bucket" echo "${bucket_name}/${gtiff_fn}" } @@ -261,7 +266,7 @@ check_image_content() { cat $img_filename return 1 fi - docker exec -i test pip install pillow >/dev/null + docker exec -i test python3 -m pip install --break-system-packages --root-user-action ignore pillow >/dev/null incontainer_file="/tmp/$(basename $img_filename)" docker cp $img_filename test:${incontainer_file} docker exec -i test python3 <<EOF diff --git a/mapserver/Dockerfile b/mapserver/Dockerfile index 84abb7f5e8376479a25fcfbdcec1cbebfd162d9c..9f281fd47ca931c3bcbc22de9634aae506d11747 100644 --- a/mapserver/Dockerfile +++ b/mapserver/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:24.04 AS build ENV DEBIAN_FRONTEND=noninteractive ENV TZ=GMT @@ -7,15 +7,15 @@ ENV LANG=C.UTF-8 ENV LC_ALL=C.UTF-8 RUN apt-get update && apt-get install -y \ - gosu \ - pwgen \ +# gosu \ +# pwgen \ tzdata \ gcc \ g++ \ automake \ build-essential \ cmake \ - sqlite \ + sqlite3 \ libsqlite3-dev \ libxml2-dev \ libjpeg-dev \ @@ -25,45 +25,34 @@ RUN apt-get update && apt-get install -y \ libfreetype6 \ libfreetype6-dev \ libzstd-dev \ - libnetcdf-dev \ - python3-pip \ pkg-config \ git \ wget \ curl \ - unzip \ libcurl4 \ libcurl4-openssl-dev \ zlib1g-dev \ libfribidi-dev \ libharfbuzz-dev \ - libcairo2-dev \ libfcgi-dev \ libgeos++-dev \ libpq-dev \ - postgresql-server-dev-all \ - libxml2-dev \ +# postgresql-server-dev-all \ libgif-dev \ - libprotobuf-dev \ - protobuf-compiler \ - libprotobuf-c-dev \ - libprotobuf-c1 \ - libprotobuf-dev \ - protobuf-c-compiler \ && rm -rf /var/lib/apt/lists/* # Install libtiff RUN mkdir -p /build_deps && cd /build_deps \ - && wget --no-check-certificate https://download.osgeo.org/libtiff/tiff-4.3.0.tar.gz \ - && tar -zxf tiff-4.3.0.tar.gz \ - && cd tiff-4.3.0 \ + && wget --no-check-certificate http://www.libtiff.org/downloads/tiff-4.6.0t.tar.gz \ + && tar -zxf tiff-4.6.0t.tar.gz \ + && cd tiff-4.6.0t \ && ./configure \ && make \ && make install # Install Proj RUN cd /build_deps \ - && git clone https://github.com/OSGeo/PROJ.git -b 8.2.1 && cd PROJ \ + && git clone https://github.com/OSGeo/PROJ.git -b 9.4.1 && cd PROJ \ && mkdir -p build && cd build \ && cmake .. \ && make -j$(nproc) \ @@ -71,9 +60,9 @@ RUN cd /build_deps \ # Install libgeotiff RUN cd /build_deps \ - && wget --no-check-certificate https://download.osgeo.org/geotiff/libgeotiff/libgeotiff-1.7.1.tar.gz \ - && tar -zxf libgeotiff-1.7.1.tar.gz \ - && cd libgeotiff-1.7.1 \ + && wget --no-check-certificate https://github.com/OSGeo/libgeotiff/releases/download/1.7.3/libgeotiff-1.7.3.tar.gz \ + && tar -zxf libgeotiff-1.7.3.tar.gz \ + && cd libgeotiff-1.7.3 \ && mkdir -p build && cd build \ && cmake .. \ && make \ @@ -81,29 +70,34 @@ RUN cd /build_deps \ # Install GDAL RUN cd /build_deps \ - && git clone --depth 1 --branch v3.5.0alpha1 https://github.com/OSGeo/gdal.git && cd gdal \ - && ./autogen.sh \ - && ./configure --with-crypto=no --with-curl=yes --with-netcdf=yes \ - && make -j$(nproc) \ - && make install - -# Install Cython, Rasterio, and Fiona -RUN cd /build_deps && pip3 install cython rasterio fiona shapely pyproj + && git clone --depth 1 --branch v3.9.1 https://github.com/OSGeo/gdal.git && cd gdal \ + && mkdir build && cd build \ + && cmake -DGDAL_USE_CURL=ON -DGDAL_USE_CRYPTOPP=OFF -DGDAL_USE_NETCDF=OFF .. \ + && cmake --build . \ + && cmake --build . --target install # Install Mapserver RUN cd /build_deps \ && git clone https://github.com/mapserver/mapserver.git && cd mapserver \ - && git checkout rel-7-6-4 \ + && git checkout rel-8-2-0 \ && mkdir -p build && cd build \ && cmake .. -DWITH_GIF=OFF -DWITH_HARFBUZZ=OFF -DWITH_PROTOBUFC=OFF -DWITH_FRIBIDI=OFF -DWITH_POSTGIS=ON -DWITH_GEOS=OFF -DWITH_FCGI=ON -DWITH_CAIRO=OFF \ && make -j$(nproc) \ && make install -# Clean up -RUN cd /tmp && rm -r /build_deps +FROM ubuntu:24.04 AS proj-update +COPY --from=build /usr/local/share/proj /usr/local/share/proj +RUN apt-get update && apt-get install -y sqlite3 + +# Add our own custom EPSG codes (HACK) +# GOES-16 ABI Full Disk = EPSG:930916 +# GOES-17 ABI Full Disk = EPSG:930917 +COPY sql/ /work/sql/ +RUN sqlite3 -bail /usr/local/share/proj/proj.db < /work/sql/goesr_crs.sql +FROM ubuntu:24.04 AS main WORKDIR /work @@ -118,23 +112,28 @@ ENV APACHE_PID_FILE /var/run/apache2/apache2.pid ENV APACHE_SERVER_NAME localhost #RUN groupadd -r www-data && useradd -r --create-home -g www-data www-data -# install httpd runtime dependencies +# install httpd and other runtime dependencies # https://httpd.apache.org/docs/2.4/install.html#requirements -RUN apt-get update && apt-get -y install apache2 libapache2-mod-fcgid libapache2-mod-php php-common php-cli php-fpm php && \ +RUN apt-get update && apt-get -y install apache2 libapache2-mod-fcgid curl unzip python3-pip && \ ls /etc/apache2/mods-available && \ ls /etc/apache2/mods-enabled && \ a2enmod actions proxy_fcgi setenvif cgi fcgid rewrite && \ - a2enconf php7.4-fpm serve-cgi-bin && \ + a2enconf serve-cgi-bin && \ apt-get -y clean && \ rm -rf /var/lib/apt/lists/* -RUN curl -LOk https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_countries.zip \ - && curl -LOk https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_1_states_provinces_lines.zip \ - && curl -LOk https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_lakes.zip \ - && curl -LOk https://www2.census.gov/geo/tiger/GENZ2018/shp/cb_2018_us_county_500k.zip \ - && curl -LOk https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_graticules_1.zip \ - && curl -LOk https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_graticules_5.zip \ - && curl -LOk https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_graticules_10.zip \ +COPY --from=build /usr/local/bin /usr/local/bin +COPY --from=proj-update /usr/local/share/proj /usr/local/share/proj +COPY --from=build /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu +COPY --from=build /usr/local/lib/libgdal* /usr/local/lib/libproj* /usr/local/lib/libtiff* /usr/local/lib/libgeotiff* /usr/local/lib/libmapserver* /usr/local/lib/ + +RUN curl -LO -e https://www.naturalearthdata.com https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_countries.zip \ + && curl -LO -e https://www.naturalearthdata.com https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_1_states_provinces_lines.zip \ + && curl -LO -e https://www.naturalearthdata.com https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_lakes.zip \ + && curl -LO -e https://www.naturalearthdata.com https://www2.census.gov/geo/tiger/GENZ2018/shp/cb_2018_us_county_500k.zip \ + && curl -LO -e https://www.naturalearthdata.com https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_graticules_1.zip \ + && curl -LO -e https://www.naturalearthdata.com https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_graticules_5.zip \ + && curl -LO -e https://www.naturalearthdata.com https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_graticules_10.zip \ && mkdir -p /work/shapefiles \ && unzip -d /work/shapefiles ne_10m_admin_0_countries.zip \ && unzip -d /work/shapefiles ne_10m_admin_1_states_provinces_lines.zip \ @@ -168,19 +167,14 @@ RUN ln -s /usr/local/bin/mapserv /usr/lib/cgi-bin/mapserv && \ chown ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/* && \ chown -h ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/* -RUN pip3 install jinja2 psycopg2 && rm -r /root/.cache/pip +RUN python3 -m pip install --break-system-packages --root-user-action ignore jinja2 && rm -r /root/.cache/pip COPY render.py run.sh abi_l1b_template.map /work/ +COPY mapserver.conf /usr/local/etc/mapserver.conf COPY mapfiles/ /work/mapfiles/ COPY html/ /var/www/html/ COPY default_settings.yaml /work/geosphere_settings.yaml -# Add our own custom EPSG codes (HACK) -# GOES-16 ABI Full Disk = EPSG:930916 -# GOES-17 ABI Full Disk = EPSG:930917 -COPY sql/ /work/sql/ -RUN sqlite3 -bail /usr/local/share/proj/proj.db < /work/sql/goesr_crs.sql - # Check the config before we finish RUN apache2ctl configtest diff --git a/mapserver/mapserver.conf b/mapserver/mapserver.conf new file mode 100644 index 0000000000000000000000000000000000000000..1099f02586739619102330bebc05d453aac734ed --- /dev/null +++ b/mapserver/mapserver.conf @@ -0,0 +1,11 @@ +CONFIG + ENV + MS_MAP_PATTERN "^/work/mapfiles" + END + + MAPS + END + + PLUGINS + END +END diff --git a/mapserver/site-conf b/mapserver/site-conf index c2b4c307a85fc335de6dd82d50057ad91fd44131..ebb8693d4a55dfd9c5502001f844d999e75dc420 100644 --- a/mapserver/site-conf +++ b/mapserver/site-conf @@ -32,6 +32,7 @@ # helper function to lowercase product name RewriteMap lc int:tolower + SetEnv MAPSERVER_CONFIG_FILE /usr/local/etc/mapserver.conf # /wms/g16/abi/radf/l1b?... # /wms/g16/abi/radf/borders?... # FCGI version of the URL diff --git a/mapserver/tiledb_geospatial_Dockerfile b/mapserver/tiledb_geospatial_Dockerfile deleted file mode 100644 index f80985ec2029f49535e75bbf5a7d205e5528b913..0000000000000000000000000000000000000000 --- a/mapserver/tiledb_geospatial_Dockerfile +++ /dev/null @@ -1,159 +0,0 @@ -# Based on - -FROM ubuntu:18.04 -LABEL maintainer="support@tiledb.io" - -ENV DEBIAN_FRONTEND=noninteractive -ENV TZ=GMT -ENV LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} -ENV LANG=C.UTF-8 -ENV LC_ALL=C.UTF-8 - -RUN apt-get update && apt-get install -y \ - gosu \ - pwgen \ - tzdata \ - gcc \ - g++ \ - build-essential \ - cmake \ - sqlite \ - libsqlite3-dev \ - libxml2-dev \ - libjpeg-dev \ - libpng-dev \ - libfreetype6-dev \ - libzstd-dev \ - python3-pip \ - git \ - wget \ - && rm -rf /var/lib/apt/lists/* - -# Install tiledb using 1.7.2 release -RUN mkdir -p /build_deps && cd /build_deps \ - && git clone https://github.com/TileDB-Inc/TileDB.git -b 1.7.2 && cd TileDB \ - && mkdir -p build && cd build \ - && cmake -DTILEDB_VERBOSE=ON -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. \ - && make -j$(nproc) \ - && make -C tiledb install - -# Install curl after building tiledb -RUN apt-get update && apt-get install -y \ - libcurl4 \ - libcurl4-openssl-dev \ - && rm -rf /var/lib/apt/lists/* - -# Install OpenJPEG -RUN cd /build_deps \ - && git clone https://github.com/uclouvain/openjpeg.git -b v2.2.0 && cd openjpeg \ - && mkdir -p build && cd build \ - && cmake .. \ - && make -j$(nproc) \ - && make install - -# Install libtiff -RUN cd /build_deps \ - && wget --no-check-certificate https://download.osgeo.org/libtiff/tiff-4.1.0.tar.gz \ - && tar -zxf tiff-4.1.0.tar.gz \ - && cd tiff-4.1.0 \ - && ./configure \ - && make \ - && make install - -# Install Proj -RUN cd /build_deps \ - && git clone https://github.com/OSGeo/PROJ.git -b 6.2.1 && cd PROJ \ - && mkdir -p build && cd build \ - && cmake .. \ - && make -j$(nproc) \ - && make install - -# Install libgeotiff -RUN cd /build_deps \ - && wget --no-check-certificate https://download.osgeo.org/geotiff/libgeotiff/libgeotiff-1.5.1.tar.gz \ - && tar -zxf libgeotiff-1.5.1.tar.gz \ - && cd libgeotiff-1.5.1 \ - && mkdir -p build && cd build \ - && cmake .. \ - && make \ - && make install - -# Install GDAL -RUN cd /build_deps \ - && git clone https://github.com/OSGeo/gdal.git && cd gdal/gdal \ - && git checkout c99a871a7bdedc751c503bb8cf508d9016510fe0 \ - && ./configure --with-crypto=no --with-curl=no \ - && make -j$(nproc) \ - && make install - -## Install TileDB-Py -RUN cd /build_deps \ - && pip3 install numpy \ - && git clone https://github.com/TileDB-Inc/TileDB-Py.git -b 0.5.3 \ - && cd TileDB-Py && python3 setup.py install - -## Install XArray -RUN cd /build_deps && pip3 install xarray - -## Install Dask -RUN cd /build_deps \ - && pip3 install toolz && pip3 install dask_image \ - && git clone https://github.com/dask/dask.git && cd dask \ - && git checkout 807f3225cf840f28ce7cf89b88fea63d473889e7 \ - && python3 setup.py install \ - && pip3 install dask distributed --upgrade \ - && pip3 install dask-image - -# Install Rasterio -RUN cd /build_deps && pip3 install cython -RUN cd /build_deps \ - && git clone https://github.com/mapbox/rasterio.git -b 1.1.0 && cd rasterio \ - && python3 setup.py install - -# Install Fiona -RUN cd /build_deps \ - && git clone https://github.com/Toblerity/Fiona.git && cd Fiona \ - && python3 setup.py install - -# Install TileDB-SAR -RUN cd /build_deps \ - && git clone https://github.com/TileDB-Inc/TileDB-SAR.git && cd TileDB-SAR \ - && git checkout 888059a15d87ae95fff6dc01c8bd4343ee4eaee1 \ - && python3 setup.py install - -# Install Mapserver -RUN cd /build_deps \ - && git clone https://github.com/mapserver/mapserver.git && cd mapserver \ - && git checkout 0fcc810f0b559c800f950db78a79fa6574799f23 \ - && mkdir -p build && cd build \ - && cmake .. -DWITH_GIF=OFF -DWITH_HARFBUZZ=OFF -DWITH_PROTOBUFC=OFF -DWITH_FRIBIDI=OFF -DWITH_POSTGIS=OFF -DWITH_GEOS=OFF -DWITH_FCGI=OFF -DWITH_CAIRO=OFF \ - && make \ - && make install - -# Install LasZIP -RUN cd /build_deps \ - && wget https://github.com/LASzip/LASzip/releases/download/3.4.1/laszip-src-3.4.1.tar.gz \ - && tar -zxf laszip-src-3.4.1.tar.gz \ - && cd laszip-src-3.4.1 \ - && mkdir -p build && cd build \ - && cmake .. \ - && make \ - && make install - -# Install PDAL -RUN cd /build_deps \ - && git clone https://github.com/PDAL/PDAL.git -b 2.0.1 && cd PDAL \ - && mkdir -p build && cd build \ - && cmake .. \ - && make \ - && make install - -# Install PDAL Python -RUN pip3 install packaging \ - && git clone https://github.com/PDAL/python pdalextension \ - && cd pdalextension \ - && python3 setup.py build \ - && python3 setup.py install - -# Clean up -RUN cd /tmp && rm -r /build_deps