Skip to content
Snippets Groups Projects
Commit 67d07dd3 authored by David Hoese's avatar David Hoese
Browse files

Merge branch 'feature-tile-gen' into 'master' 

Add tile generation container

See merge request cspp_geo/cspp-geo-web-viewer!7
parents db1233a3 c8d36d55
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 794 additions and 293 deletions
mapserver/Dockerfile
+ 38
− 65
View file @ 67d07dd3
FROM ubuntu:eoan FROM tiledb/tiledb-geospatial:latest
WORKDIR /work
# FIXME: Remove once added to parent image
RUN pip3 install shapely
RUN apt-get -y update && \ RUN apt-get -y update && \
apt-get -y upgrade && \ apt-get -y upgrade && \
apt-get -y install gcc g++ cmake sqlite3 libsqlite3-dev pkg-config bash-completion curl subversion && \ apt-get -y install gcc g++ cmake sqlite3 libsqlite3-dev pkg-config bash-completion curl subversion && \
apt-get -y clean apt-get -y clean
# proj 6.1.1
RUN mkdir -p /build/proj && \
cd /build/proj && \
curl -O https://download.osgeo.org/proj/proj-6.1.1.tar.gz && \
tar xf proj-6.1.1.tar.gz && \
cd proj-6.1.1 && \
./configure && \
make -j $(nproc) && \
make install && \
rm -rf /build/proj
# mkdir build && \
# cd build && \
# cmake .. && \
# cmake --build . && \
# gdal 3.0.1 # FIXME: Remove if/when upstream image builds with fcgi
RUN mkdir -p /build/gdal && \ # Install Mapserver (unstable)
cd /build/gdal && \ RUN mkdir -p /build_deps && cd /build_deps \
curl -O http://download.osgeo.org/gdal/3.0.1/gdal-3.0.1.tar.gz && \ && apt-get -y update \
tar xf gdal-3.0.1.tar.gz && \ # && apt-get install -y libprotobuf17 zlib1g-dev libpng-dev libjpeg-turbo8 libfreetype6 libfribidi-dev libharfbuzz-dev libcairo2-dev libfcgi-dev libgeos++-dev postgresql postgis libxml2-dev libgif-dev libjpeg-turbo8-dev libprotobuf-dev protobuf-compiler libprotobuf-c-dev libprotobuf-c1 libprotobuf-dev protobuf-c-compiler \
cd gdal-3.0.1 && \ && apt-get install -y zlib1g-dev libpng-dev libjpeg-turbo8 libfreetype6 libfribidi-dev libharfbuzz-dev libcairo2-dev libfcgi-dev libgeos++-dev postgresql postgis libxml2-dev libgif-dev libjpeg-turbo8-dev libprotobuf-dev protobuf-compiler libprotobuf-c-dev libprotobuf-c1 libprotobuf-dev protobuf-c-compiler \
./configure && \ && apt-get -y clean \
make -j$(nproc) && \ && git clone https://github.com/mapserver/mapserver.git && cd mapserver \
make install && \ && git checkout 0fcc810f0b559c800f950db78a79fa6574799f23 \
rm -rf /build/gdal && mkdir -p build && cd build \
&& cmake .. -DWITH_PROTOBUFC=OFF -DWITH_POSTGIS=OFF \
# mapserver && make -j$(nproc) \
RUN mkdir -p /build/mapserver && \ && make install
apt-get -y update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y libprotobuf17 zlib1g-dev libpng-dev libjpeg-turbo8 libfreetype6 libfribidi-dev libharfbuzz-dev libcairo2-dev libfcgi-dev libgeos++-dev postgresql postgis libxml2-dev libgif-dev libjpeg-turbo8-dev libprotobuf-dev protobuf-compiler libprotobuf-c-dev libprotobuf-c1 libprotobuf-dev protobuf-c-compiler && \
apt-get -y clean && \
curl -O http://download.osgeo.org/mapserver/mapserver-7.4.2.tar.gz && \
tar xf mapserver-7.4.2.tar.gz && \
cd mapserver-7.4.2 && \
mkdir build && \
cd build && \
cmake .. -DWITH_POSTGIS=0 && \
make -j$(nproc) && \
make install
# postgres
#service postgresql start
# postgis
# pgbouncer
# apache # apache
# http://www.inanzzz.com/index.php/post/rhsb/running-apache-server-as-foreground-on-ubuntu-with-dockerfile # http://www.inanzzz.com/index.php/post/rhsb/running-apache-server-as-foreground-on-ubuntu-with-dockerfile
...@@ -66,35 +38,36 @@ ENV APACHE_SERVER_NAME localhost ...@@ -66,35 +38,36 @@ ENV APACHE_SERVER_NAME localhost
# install httpd runtime dependencies # install httpd runtime dependencies
# https://httpd.apache.org/docs/2.4/install.html#requirements # https://httpd.apache.org/docs/2.4/install.html#requirements
RUN apt-get -y install apache2 libapache2-mod-fcgid && \ RUN cd /build_deps && apt-get -y install apache2 libapache2-mod-fcgid && \
ls /etc/apache2/mods-available && \ ls /etc/apache2/mods-available && \
ls /etc/apache2/mods-enabled && \ ls /etc/apache2/mods-enabled && \
apt-get -y install libapache2-mod-php7.3 php7.3-common php7.3-cli php7.3-fpm php7.3 && \ apt-get -y install libapache2-mod-php7.2 php7.2-common php7.2-cli php7.2-fpm php7.2 && \
a2enmod actions proxy_fcgi setenvif cgi fcgid && \ a2enmod actions proxy_fcgi setenvif cgi fcgid rewrite && \
a2enconf php7.3-fpm serve-cgi-bin && \ a2enconf php7.2-fpm serve-cgi-bin && \
apt-get -y clean && \ apt-get -y clean && \
rm -rf /var/lib/apt/lists/* rm -rf /var/lib/apt/lists/*
COPY apache-conf /etc/apache2/apache2.conf COPY site-conf /etc/apache2/sites-available/cspp_geo.conf
# disable the default which would conflict with our custom
RUN a2ensite cspp_geo && a2dissite 000-default
COPY cgi-bin/* /usr/lib/cgi-bin/
# Point apache to the mapserver binary # Point apache to the mapserver binary
RUN ln -s /usr/local/bin/mapserv /usr/lib/cgi-bin/mapserv && \ RUN ln -s /usr/local/bin/mapserv /usr/lib/cgi-bin/mapserv && \
chown ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/mapserv && \ chown ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/* && \
chown -h ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/mapserv chown -h ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/*
COPY mapfiles/ /work/mapfiles/
COPY html/ /var/www/html/
# Add our own custom EPSG codes (HACK)
# GOES-16 ABI Full Disk = EPSG:930916
# GOES-17 ABI Full Disk = EPSG:930917
COPY sql/ /work/sql/
RUN sqlite3 -init /work/sql/goesr_crs.sql /usr/local/share/proj/proj.db
# https://httpd.apache.org/docs/2.4/stopping.html#gracefulstop # https://httpd.apache.org/docs/2.4/stopping.html#gracefulstop
STOPSIGNAL WINCH STOPSIGNAL WINCH
EXPOSE 80 EXPOSE 80
CMD ["/usr/sbin/apache2ctl", "-DFOREGROUND"] CMD ["/usr/sbin/apache2ctl", "-DFOREGROUND"]
##############################################################
# php
# python? (for shp2csv type stuff)
mapserver/README.md
+ 15
− 1
View file @ 67d07dd3
...@@ -3,9 +3,23 @@ ...@@ -3,9 +3,23 @@
## Usage ## Usage
```bash ```bash
docker run -p 8888:80 -d gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapserver:latest docker run -p 8888:80 -d --rm --name cspp-geo-mapserver -v cspp-geo-abi-l1b-geotiffs:/data gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapserver:latest
``` ```
Then the main mapserv CGI script can be accessed with: Then the main mapserv CGI script can be accessed with:
http://localhost:8888/cgi-bin/mapserv http://localhost:8888/cgi-bin/mapserv
## Special Notes
This image has had its installation of the PROJ library modified to include
new non-standard EPSG codes to support the projections of certain satellite
instruments. These additions are part of the Dockerfile build process and
include the following definitions:
1. EPSG:930916 - The GOES-16 ABI Fixed Grid geostationary coordinate reference
system. The numbers are meant to represent "GEOG16" while also attempting
to avoid conflicts with future additions to the official EPSG database.
2. EPSG:930917 - The GOES-17 ABI Fixed Grid geostationary coordinate reference
system. The numbers are meant to represent "GEOG17" while also attempting
to avoid conflicts with future additions to the official EPSG database.
mapserver/apache-conf deleted 100644 → 0
+ 0
− 227
View file @ db1233a3
# This is the main Apache server configuration file. It contains the
# configuration directives that give the server its instructions.
# See http://httpd.apache.org/docs/2.4/ for detailed information about
# the directives and /usr/share/doc/apache2/README.Debian about Debian specific
# hints.
#
#
# Summary of how the Apache 2 configuration works in Debian:
# The Apache 2 web server configuration in Debian is quite different to
# upstream's suggested way to configure the web server. This is because Debian's
# default Apache2 installation attempts to make adding and removing modules,
# virtual hosts, and extra configuration directives as flexible as possible, in
# order to make automating the changes and administering the server as easy as
# possible.
# It is split into several files forming the configuration hierarchy outlined
# below, all located in the /etc/apache2/ directory:
#
# /etc/apache2/
# |-- apache2.conf
# | `-- ports.conf
# |-- mods-enabled
# | |-- *.load
# | `-- *.conf
# |-- conf-enabled
# | `-- *.conf
# `-- sites-enabled
# `-- *.conf
#
#
# * apache2.conf is the main configuration file (this file). It puts the pieces
# together by including all remaining configuration files when starting up the
# web server.
#
# * ports.conf is always included from the main configuration file. It is
# supposed to determine listening ports for incoming connections which can be
# customized anytime.
#
# * Configuration files in the mods-enabled/, conf-enabled/ and sites-enabled/
# directories contain particular configuration snippets which manage modules,
# global configuration fragments, or virtual host configurations,
# respectively.
#
# They are activated by symlinking available configuration files from their
# respective *-available/ counterparts. These should be managed by using our
# helpers a2enmod/a2dismod, a2ensite/a2dissite and a2enconf/a2disconf. See
# their respective man pages for detailed information.
#
# * The binary is called apache2. Due to the use of environment variables, in
# the default configuration, apache2 needs to be started/stopped with
# /etc/init.d/apache2 or apache2ctl. Calling /usr/bin/apache2 directly will not
# work with the default configuration.
# Global configuration
#
#
# ServerRoot: The top of the directory tree under which the server's
# configuration, error, and log files are kept.
#
# NOTE! If you intend to place this on an NFS (or otherwise network)
# mounted filesystem then please read the Mutex documentation (available
# at <URL:http://httpd.apache.org/docs/2.4/mod/core.html#mutex>);
# you will save yourself a lot of trouble.
#
# Do NOT add a slash at the end of the directory path.
#
#ServerRoot "/etc/apache2"
#
# The accept serialization lock file MUST BE STORED ON A LOCAL DISK.
#
#Mutex file:${APACHE_LOCK_DIR} default
#
# The directory where shm and other runtime files will be stored.
#
DefaultRuntimeDir ${APACHE_RUN_DIR}
#
# PidFile: The file in which the server should record its process
# identification number when it starts.
# This needs to be set in /etc/apache2/envvars
#
PidFile ${APACHE_PID_FILE}
#
# Timeout: The number of seconds before receives and sends time out.
#
Timeout 300
#
# KeepAlive: Whether or not to allow persistent connections (more than
# one request per connection). Set to "Off" to deactivate.
#
KeepAlive On
#
# MaxKeepAliveRequests: The maximum number of requests to allow
# during a persistent connection. Set to 0 to allow an unlimited amount.
# We recommend you leave this number high, for maximum performance.
#
MaxKeepAliveRequests 100
#
# KeepAliveTimeout: Number of seconds to wait for the next request from the
# same client on the same connection.
#
KeepAliveTimeout 5
# These need to be set in /etc/apache2/envvars
User ${APACHE_RUN_USER}
Group ${APACHE_RUN_GROUP}
#
# HostnameLookups: Log the names of clients or just their IP addresses
# e.g., www.apache.org (on) or 204.62.129.132 (off).
# The default is off because it'd be overall better for the net if people
# had to knowingly turn this feature on, since enabling it means that
# each client request will result in AT LEAST one lookup request to the
# nameserver.
#
HostnameLookups Off
# ErrorLog: The location of the error log file.
# If you do not specify an ErrorLog directive within a <VirtualHost>
# container, error messages relating to that virtual host will be
# logged here. If you *do* define an error logfile for a <VirtualHost>
# container, that host's errors will be logged there and not here.
#
ErrorLog ${APACHE_LOG_DIR}/error.log
#
# LogLevel: Control the severity of messages logged to the error_log.
# Available values: trace8, ..., trace1, debug, info, notice, warn,
# error, crit, alert, emerg.
# It is also possible to configure the log level for particular modules, e.g.
# "LogLevel info ssl:warn"
#
LogLevel warn
# Include module configuration:
IncludeOptional mods-enabled/*.load
IncludeOptional mods-enabled/*.conf
# Include list of ports to listen on
Include ports.conf
# Sets the default security model of the Apache2 HTTPD server. It does
# not allow access to the root filesystem outside of /usr/share and /var/www.
# The former is used by web applications packaged in Debian,
# the latter may be used for local directories served by the web server. If
# your system is serving content from a sub-directory in /srv you must allow
# access here, or in any related virtual host.
<Directory />
Options FollowSymLinks
AllowOverride None
Require all denied
</Directory>
<Directory /usr/share>
AllowOverride None
Require all granted
</Directory>
<Directory /var/www/>
Options Indexes FollowSymLinks
AllowOverride None
Require all granted
</Directory>
#<Directory /srv/>
# Options Indexes FollowSymLinks
# AllowOverride None
# Require all granted
#</Directory>
# AccessFileName: The name of the file to look for in each directory
# for additional configuration directives. See also the AllowOverride
# directive.
#
AccessFileName .htaccess
#
# The following lines prevent .htaccess and .htpasswd files from being
# viewed by Web clients.
#
<FilesMatch "^\.ht">
Require all denied
</FilesMatch>
#
# The following directives define some format nicknames for use with
# a CustomLog directive.
#
# These deviate from the Common Log Format definitions in that they use %O
# (the actual bytes sent including headers) instead of %b (the size of the
# requested file), because the latter makes it impossible to detect partial
# requests.
#
# Note that the use of %{X-Forwarded-For}i instead of %h is not recommended.
# Use mod_remoteip instead.
#
LogFormat "%v:%p %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" vhost_combined
LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" combined
LogFormat "%h %l %u %t \"%r\" %>s %O" common
LogFormat "%{Referer}i -> %U" referer
LogFormat "%{User-agent}i" agent
# Include of directories ignores editors' and dpkg's backup files,
# see README.Debian for details.
# Include generic snippets of statements
IncludeOptional conf-enabled/*.conf
# Include the virtual host configurations:
IncludeOptional sites-enabled/*.conf
# vim: syntax=apache ts=4 sw=4 sts=4 sr noet
\ No newline at end of file
mapserver/cgi-bin/layer_times.py 0 → 100755
+ 14
− 0
View file @ 67d07dd3
#!/usr/bin/env python3
import cgi
import json
import fiona
form = cgi.FieldStorage()
layer = form['layer'].value
with fiona.open(layer, 'r') as shp_file:
times = [x['properties']['time'] for x in shp_file]
print("Content-Type: application/json")
print("Access-Control-Allow-Origin: *")
print() # blank line, end of headers
print(json.dumps(times))
mapserver/html/index.html 0 → 100644
+ 21
− 0
View file @ 67d07dd3
<html>
<head></head>
<body>
<h1>CSPP Geo Geo2Grid Tile Server</h1>
This tile server for the CSPP Geo Geo2Grid project.
<br/>
<br/>
To access tiles using a standard WMS protocol:
<br/>
<br/>
http://localhost/wms/goes16/abi/fldk/l1b?...
<br/>
<br/>
Where "fldk" stands for Full Disk and "..." represents all remaining WMS
parameters. Other sectors available are "m01", "m02", and "conus".
<br/>
<br/>
</body>
</html>
\ No newline at end of file
mapserver/mapfiles/g16_abi_radf_l1b.map 0 → 100644
+ 109
− 0
View file @ 67d07dd3
MAP
IMAGETYPE PNG
SIZE 256 256
EXTENT -180 -90 180 90
PROJECTION
"init=epsg:930916"
END
WEB
METADATA
"wms_title" "Weather data"
"wms_onlineresource" "https://myhost/mapserv"
"wms_enable_request" "*"
END
END
##############
# True Color #
##############
LAYER
NAME "true_color_index"
TYPE TILEINDEX
DATA "/data/tiles/g16/abi/radf/true_color/true_color"
END
LAYER
NAME "true_color"
TYPE RASTER
TILEITEM "location"
TILEINDEX "true_color_index"
# Comment below to default to transparency
# OFFSITE 0 0 0
PROJECTION
"init=epsg:930916"
END
METADATA
"wms_title" "GOES-16 ABI True Color"
"wms_extent" "-180 -90 180 90"
"wms_timeextent" "2017-01-01/2020-12-31"
"wms_timeformat" "YYYY-MM-DDTHH:MM:SS"
"wms_timeitem" "time" # time is a metadata item
"wms_timedefault" "2019-12-12T19:20:18"
"wms_enable_request" "*"
END
FILTER (`[time]` = `2019-12-12T19:20:18`)
END # goes raster layer ends here
######################
# True Color Lat/Lon #
######################
LAYER
NAME "true_color_index_ll"
TYPE TILEINDEX
DATA "/data/tiles/g16/abi/radf_ll/true_color_ll/true_color_ll"
END
LAYER
NAME "true_color_ll"
TYPE RASTER
TILEITEM "location"
TILEINDEX "true_color_index_ll"
# Comment below to default to transparency
# OFFSITE 0 0 0
PROJECTION
"init=epsg:4326"
END
METADATA
"wms_title" "GOES-16 ABI True Color"
"wms_extent" "-180 -90 180 90"
"wms_timeextent" "2017-01-01/2020-12-31"
"wms_timeformat" "YYYY-MM-DDTHH:MM:SS"
"wms_timeitem" "time" # time is a metadata item
"wms_timedefault" "2019-12-12T19:20:18"
"wms_enable_request" "*"
END
FILTER (`[time]` = `2019-12-12T19:20:18`)
END # goes raster layer ends here
###################
# True Color Fake #
###################
LAYER
NAME "true_color_test"
TYPE RASTER
TILEITEM "location"
TILEINDEX "true_color_index"
# Comment below to default to transparency
# OFFSITE 0 0 0
PROJECTION
"init=epsg:930916"
END
METADATA
"wms_title" "GOES-16 ABI True Color"
"wms_extent" "-180 -90 180 90"
"wms_timeextent" "2017-01-01/2020-12-31"
"wms_timeformat" "YYYY-MM-DDTHH:MM:SS"
"wms_timeitem" "time" # time is a metadata item
"wms_timedefault" "2019-12-12T19:20:18"
"wms_enable_request" "*"
END
FILTER (`[time]` = `2019-12-12T19:20:18`)
END # goes raster layer ends here
END # end of map file
mapserver/site-conf 0 → 100644
+ 43
− 0
View file @ 67d07dd3
<VirtualHost *:80>
# The ServerName directive sets the request scheme, hostname and port that
# the server uses to identify itself. This is used when creating
# redirection URLs. In the context of virtual hosts, the ServerName
# specifies what hostname must appear in the request's Host: header to
# match this virtual host. For the default virtual host (this file) this
# value is not decisive as it is used as a last resort host regardless.
# However, you must set it for any further virtual host explicitly.
#ServerName www.example.com
ServerAdmin david.hoese@ssec.wisc.edu
DocumentRoot /var/www/html
# Available loglevels: trace8, ..., trace1, debug, info, notice, warn,
# error, crit, alert, emerg.
# It is also possible to configure the loglevel for particular
# modules, e.g.
#LogLevel info ssl:warn
ErrorLog ${APACHE_LOG_DIR}/error.log
CustomLog ${APACHE_LOG_DIR}/access.log combined
# For most configuration files from conf-available/, which are
# enabled or disabled at a global level, it is possible to
# include a line for only one particular virtual host. For example the
# following line enables the CGI configuration for this host only
# after it has been globally disabled with "a2disconf".
#Include conf-available/serve-cgi-bin.conf
# Custom CSPP Geo
# LoadModule rewrite_module modules/mod_rewrite.so
RewriteEngine on
# /wms/goes16/abi/fldk/
RewriteRule "^/wms/([^/]+)/([^/]+)/([^/]+)/l1b?(.*)" "/cgi-bin/mapserv?map=/work/mapfiles/$1_$2_$3_l1b.map&$4" [PT,QSA]
# /wms_times/g16/abi/radf/true_color
RewriteRule "^/wms_times/([^/]+)/([^/]+)/([^/]+)/([^/]+)" "/cgi-bin/layer_times.py?layer=/data/tiles/$1/$2/$3/$4/$4.shp" [PT,QSA]
# FIXME: We need to include the sector
# /data/goes/grb/goes16/2020/2020_01_21_021/abi/L1b/RadF/GOES-16_ABI_RadF_C01_20200121_000016_GOES-East.tif
# "/data/tiles/g16/abi/radf/true_color/true_color"
LogLevel alert rewrite:trace6
</VirtualHost>
# vim: syntax=apache ts=4 sw=4 sts=4 sr noet
\ No newline at end of file
mapserver/sql/goesr_crs.sql 0 → 100644
+ 2
− 0
View file @ 67d07dd3
INSERT INTO projected_crs (auth_name, code, name, description, scope, coordinate_system_auth_name, coordinate_system_code, geodetic_crs_auth_name, geodetic_crs_code, conversion_auth_name, conversion_code, area_of_use_auth_name, area_of_use_code, text_definition, deprecated) VALUES ('EPSG', '930917', 'GOES-17 ABI Fixed Grid', 'GOES-17 (GOES-WEST) ABI Fixed Grid in the Geostationary projection', null, null, null, 'EPSG', '4269', null, null, null, null, 'PROJCRS["GOES-17 ABI Fixed Grid",BASEGEOGCRS["GOES-17 ABI Fixed Grid",DATUM["North American Datum 1983",ELLIPSOID["GRS 1980",6378137,298.257222101,LENGTHUNIT["metre",1]],ID["EPSG",6269]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8901]]],CONVERSION["unknown",METHOD["Geostationary Satellite (Sweep X)"],PARAMETER["Longitude of natural origin",-137,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8802]],PARAMETER["Satellite Height",35786023,LENGTHUNIT["metre",1,ID["EPSG",9001]]],PARAMETER["False easting",0,LENGTHUNIT["metre",1],ID["EPSG",8806]],PARAMETER["False northing",0,LENGTHUNIT["metre",1],ID["EPSG",8807]]],CS[Cartesian,2],AXIS["(E)",east,ORDER[1],LENGTHUNIT["metre",1,ID["EPSG",9001]]],AXIS["(N)",north,ORDER[2],LENGTHUNIT["metre",1,ID["EPSG",9001]]]]', 0);
INSERT INTO projected_crs (auth_name, code, name, description, scope, coordinate_system_auth_name, coordinate_system_code, geodetic_crs_auth_name, geodetic_crs_code, conversion_auth_name, conversion_code, area_of_use_auth_name, area_of_use_code, text_definition, deprecated) VALUES ('EPSG', '930916', 'GOES-16 ABI Fixed Grid', 'GOES-16 (GOES-WEST) ABI Fixed Grid in the Geostationary projection', null, null, null, 'EPSG', '4269', null, null, null, null, 'PROJCRS["GOES-16 ABI Fixed Grid",BASEGEOGCRS["GOES-16 ABI Fixed Grid",DATUM["North American Datum 1983",ELLIPSOID["GRS 1980",6378137,298.257222101,LENGTHUNIT["metre",1]],ID["EPSG",6269]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8901]]],CONVERSION["unknown",METHOD["Geostationary Satellite (Sweep X)"],PARAMETER["Longitude of natural origin",-75,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8802]],PARAMETER["Satellite Height",35786023,LENGTHUNIT["metre",1,ID["EPSG",9001]]],PARAMETER["False easting",0,LENGTHUNIT["metre",1],ID["EPSG",8806]],PARAMETER["False northing",0,LENGTHUNIT["metre",1],ID["EPSG",8807]]],CS[Cartesian,2],AXIS["(E)",east,ORDER[1],LENGTHUNIT["metre",1,ID["EPSG",9001]]],AXIS["(N)",north,ORDER[2],LENGTHUNIT["metre",1,ID["EPSG",9001]]]]', 0);
\ No newline at end of file
mapserver/tiledb_geospatial_Dockerfile 0 → 100644
+ 159
− 0
View file @ 67d07dd3
# Based on
FROM ubuntu:18.04
LABEL maintainer="support@tiledb.io"
ENV DEBIAN_FRONTEND=noninteractive
ENV TZ=GMT
ENV LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH}
ENV LANG=C.UTF-8
ENV LC_ALL=C.UTF-8
RUN apt-get update && apt-get install -y \
gosu \
pwgen \
tzdata \
gcc \
g++ \
build-essential \
cmake \
sqlite \
libsqlite3-dev \
libxml2-dev \
libjpeg-dev \
libpng-dev \
libfreetype6-dev \
libzstd-dev \
python3-pip \
git \
wget \
&& rm -rf /var/lib/apt/lists/*
# Install tiledb using 1.7.2 release
RUN mkdir -p /build_deps && cd /build_deps \
&& git clone https://github.com/TileDB-Inc/TileDB.git -b 1.7.2 && cd TileDB \
&& mkdir -p build && cd build \
&& cmake -DTILEDB_VERBOSE=ON -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. \
&& make -j$(nproc) \
&& make -C tiledb install
# Install curl after building tiledb
RUN apt-get update && apt-get install -y \
libcurl4 \
libcurl4-openssl-dev \
&& rm -rf /var/lib/apt/lists/*
# Install OpenJPEG
RUN cd /build_deps \
&& git clone https://github.com/uclouvain/openjpeg.git -b v2.2.0 && cd openjpeg \
&& mkdir -p build && cd build \
&& cmake .. \
&& make -j$(nproc) \
&& make install
# Install libtiff
RUN cd /build_deps \
&& wget --no-check-certificate https://download.osgeo.org/libtiff/tiff-4.1.0.tar.gz \
&& tar -zxf tiff-4.1.0.tar.gz \
&& cd tiff-4.1.0 \
&& ./configure \
&& make \
&& make install
# Install Proj
RUN cd /build_deps \
&& git clone https://github.com/OSGeo/PROJ.git -b 6.2.1 && cd PROJ \
&& mkdir -p build && cd build \
&& cmake .. \
&& make -j$(nproc) \
&& make install
# Install libgeotiff
RUN cd /build_deps \
&& wget --no-check-certificate https://download.osgeo.org/geotiff/libgeotiff/libgeotiff-1.5.1.tar.gz \
&& tar -zxf libgeotiff-1.5.1.tar.gz \
&& cd libgeotiff-1.5.1 \
&& mkdir -p build && cd build \
&& cmake .. \
&& make \
&& make install
# Install GDAL
RUN cd /build_deps \
&& git clone https://github.com/OSGeo/gdal.git && cd gdal/gdal \
&& git checkout c99a871a7bdedc751c503bb8cf508d9016510fe0 \
&& ./configure --with-crypto=no --with-curl=no \
&& make -j$(nproc) \
&& make install
## Install TileDB-Py
RUN cd /build_deps \
&& pip3 install numpy \
&& git clone https://github.com/TileDB-Inc/TileDB-Py.git -b 0.5.3 \
&& cd TileDB-Py && python3 setup.py install
## Install XArray
RUN cd /build_deps && pip3 install xarray
## Install Dask
RUN cd /build_deps \
&& pip3 install toolz && pip3 install dask_image \
&& git clone https://github.com/dask/dask.git && cd dask \
&& git checkout 807f3225cf840f28ce7cf89b88fea63d473889e7 \
&& python3 setup.py install \
&& pip3 install dask distributed --upgrade \
&& pip3 install dask-image
# Install Rasterio
RUN cd /build_deps && pip3 install cython
RUN cd /build_deps \
&& git clone https://github.com/mapbox/rasterio.git -b 1.1.0 && cd rasterio \
&& python3 setup.py install
# Install Fiona
RUN cd /build_deps \
&& git clone https://github.com/Toblerity/Fiona.git && cd Fiona \
&& python3 setup.py install
# Install TileDB-SAR
RUN cd /build_deps \
&& git clone https://github.com/TileDB-Inc/TileDB-SAR.git && cd TileDB-SAR \
&& git checkout 888059a15d87ae95fff6dc01c8bd4343ee4eaee1 \
&& python3 setup.py install
# Install Mapserver
RUN cd /build_deps \
&& git clone https://github.com/mapserver/mapserver.git && cd mapserver \
&& git checkout 0fcc810f0b559c800f950db78a79fa6574799f23 \
&& mkdir -p build && cd build \
&& cmake .. -DWITH_GIF=OFF -DWITH_HARFBUZZ=OFF -DWITH_PROTOBUFC=OFF -DWITH_FRIBIDI=OFF -DWITH_POSTGIS=OFF -DWITH_GEOS=OFF -DWITH_FCGI=OFF -DWITH_CAIRO=OFF \
&& make \
&& make install
# Install LasZIP
RUN cd /build_deps \
&& wget https://github.com/LASzip/LASzip/releases/download/3.4.1/laszip-src-3.4.1.tar.gz \
&& tar -zxf laszip-src-3.4.1.tar.gz \
&& cd laszip-src-3.4.1 \
&& mkdir -p build && cd build \
&& cmake .. \
&& make \
&& make install
# Install PDAL
RUN cd /build_deps \
&& git clone https://github.com/PDAL/PDAL.git -b 2.0.1 && cd PDAL \
&& mkdir -p build && cd build \
&& cmake .. \
&& make \
&& make install
# Install PDAL Python
RUN pip3 install packaging \
&& git clone https://github.com/PDAL/python pdalextension \
&& cd pdalextension \
&& python3 setup.py build \
&& python3 setup.py install
# Clean up
RUN cd /tmp && rm -r /build_deps
tile_gen/Dockerfile 0 → 100644
+ 16
− 0
View file @ 67d07dd3
FROM tiledb/tiledb-geospatial:latest
WORKDIR /work
# TODO may need the unzip command to be installed if not already
RUN apt-get update && apt-get install -y unzip && \
wget http://ssec.wisc.edu/~rayg/pub/amqpfind.zip && \
unzip amqpfind.zip && \
rm amqpfind.zip && \
rm -rf /var/lib/apt/lists/*
# FIXME: Remove once added to parent image
RUN pip3 install shapely
COPY tile_index.py .
COPY generate_tiles.py .
COPY run.sh .
tile_gen/README.md 0 → 100644
+ 31
− 0
View file @ 67d07dd3
# Tile Generation
## Build
```bash
docker build -t gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/tile_gen:latest tile_gen/
docker push gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/tile_gen:latest
```
## Usage
```bash
docker run -d --rm --network cspp-geo-rabbit --cpus 2 --name cspp-geo-tilegen-g16-radf -e AMQPFIND_TOPIC="data.goes.g16.abi.radf.l1b.geotiff.complete" -v cspp-geo-abi-l1b-geotiffs:/data gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/tile_gen:latest ./run.sh
```
To run the version of tile generation that remaps to EPSG:4326:
```bash
docker run -d --rm --network cspp-geo-rabbit --cpus 6 --name cspp-geo-tilegen-g16-radf-ll -e AMQPFIND_TOPIC="data.goes.g16.abi.radf.l1b.geotiff.complete" -e G2G_PRODUCTS="true_color" -e TILE_ARGS="--remap --shape-file {product}_ll.shp" -v cspp-geo-abi-l1b-geotiffs:/data gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/tile_gen:latest ./run.sh
```
NOTE: For full disk data, remapping all products can take a really long
time and fall behind. The above command limits processing to "true_color"
only. To attempt to process all products remove the below portion of the
command.
```
-e G2G_PRODUCTS="true_color"
```
And adding more CPUs.
\ No newline at end of file
tile_gen/generate_tiles.py 0 → 100644
+ 105
− 0
View file @ 67d07dd3
#!/usr/bin/env python3
import os
import sys
import warnings
import logging
import subprocess
import shutil
from glob import glob
import tile_index
LOG = logging.getLogger(__name__)
def group_files(products, input_files):
"""Group input geotiff files by product."""
groups = {}
for prod in products:
prods_files = [f for f in input_files if prod in f]
if prods_files:
groups[prod] = prods_files
if len(groups) != len(products):
warnings.warn("Not all product geotiffs were provided.")
return groups
def remap_to_lonlat(itif, otif):
"""Remap a single geotiff by calling gdalwarp."""
try:
subprocess.run(['gdalwarp', '-multi', '-wo', 'NUM_THREADS=ALL_CPUS', '-t_srs', 'EPSG:4326', itif, otif], check=True)
except subprocess.CalledProcessError:
LOG.error("Could not remap geotiff %s -> %s" % (itif, otif))
return None
return otif
def remap_tifs(input_tifs, out_dir, remap_suffix):
"""Remap all input geotiffs to EPSG:4326."""
for itif in input_tifs:
ifn = os.path.basename(itif)
otif = os.path.join(out_dir, ifn.replace('.tif', remap_suffix))
otif = remap_to_lonlat(itif, otif)
if otif is not None:
yield otif
def link_or_copy(input_tifs, out_dir):
"""Hardlink input tifs to output directory."""
for prod_file in input_tifs:
out_file = os.path.join(out_dir, os.path.basename(prod_file))
try:
os.link(prod_file, out_file)
except OSError:
# on different mounts probably?
shutil.copy2(prod_file, out_file)
yield out_file
def main():
import argparse
parser = argparse.ArgumentParser(description="Take input geotiffs and generate mapserver compatible tiles.")
parser.add_argument('--remap', action='store_true',
help="Remap input geotiffs to EPSG:4326")
parser.add_argument('--remap-suffix', default='_ll.tif',
help="Replace 'tif' with provided suffix when geotiffs are remapped.")
parser.add_argument('-p', '--products', nargs="*",
help="Product names to group together in each "
"'layer'. Product name must be in the filename.")
parser.add_argument('--shape-file', default='{product}.shp',
help="Shapefile filename pattern to use and placed "
"in the output directory. "
"(default: '{product}.shp')")
parser.add_argument('out_dir',
help="Output path to save tile information to (ex. '/data/tiles/{product}')")
parser.add_argument('input_files', nargs="+",
help="Input geotiffs to generate tiles for (separate from product lists with '--')")
args = parser.parse_args()
groups = group_files(args.products, args.input_files)
for prod, prod_files in groups.items():
out_dir = args.out_dir.format(product=prod)
os.makedirs(out_dir, exist_ok=True)
shp_fn = args.shape_file.format(product=prod)
shp_pathname = os.path.join(out_dir, shp_fn)
if args.remap:
# remap if needed
prod_files = list(remap_tifs(prod_files, out_dir, args.remap_suffix))
else:
# hardlink if needed
prod_files = list(link_or_copy(prod_files, out_dir))
# get all products in the current directory
ext = os.path.splitext(prod_files[-1])[-1]
all_prod_files = sorted(glob(os.path.join(out_dir, '*' + prod + '*' + ext)))
# create shape file
LOG.info("Rebuilding shapefile index with:\n\t{}".format(", ".join(all_prod_files)))
tile_index.index(all_prod_files, shp_pathname)
if __name__ == "__main__":
sys.exit(main())
\ No newline at end of file
tile_gen/run.sh 0 → 100755
+ 81
− 0
View file @ 67d07dd3
#!/bin/bash -le
# Usage: run.sh
# Environment variables used for configuration:
# AMQPFIND_ARGS: Arguments to pass to amqpfind when listening for new input
# events. Should not include the "-C" topic flag (see AMQPFIND_TOPIC).
# Default: "-H cspp-geo-rabbit -X satellite -u guest -p guest"
# AMQPFIND_TOPIC: Topic to use for incoming data events.
# Default: "data.goes.*.abi.*.l1b.netcdf.complete"
# The first asterisk (3rd element) can limit processing to a particular
# satellite (ex. `g16`). The second asterisk (5th element) can be used
# to limit to a particular sector (choices: radf, radc, radm1, radm2)
# AMQPSEND_ARGS: Arguments to pass to amqpsend when sending out new data
# events. Default: "-H cspp-geo-rabbit -X satellite -u guest -p guest"
# Verify that the data mount is available
test -d "/data"
export AMQPFIND_ARGS=${AMQPFIND_ARGS:-"-H cspp-geo-rabbit -X satellite -u guest -p guest"}
export AMQPSEND_ARGS=${AMQPSEND_ARGS:-"-H cspp-geo-rabbit -X satellite -u guest -p guest"}
export AMQPFIND_TOPIC=${AMQPFIND_TOPIC:-'data.goes.*.abi.*.l1b.geotiff.complete'}
export G2G_PRODUCTS=${G2G_PRODUCTS:-"C01 C02 C03 C04 C05 C06 C07 C08 C09 C10 C11 C12 C13 C14 C15 C16 true_color"}
export TILE_ARGS=${TILE_ARGS:-""}
export TMPDIR=${TMPDIR:-"/dst/tmp"}
run_tile_gen() {
if [[ $# -ne 5 ]]; then
echo "Unexpected number of arguments (expected 5): $#"
return 1
fi
satellite_family=${1,,}
satellite_id=${2,,}
instrument=${3,,}
data_type=${4,,}
path="$5"
echo "Starting tile generation processing for ${path}"
# convert path from a relative path to an absolute path
path="/data/${path}"
# if /dst isn't defined then use /data
if [[ -d "/dst" ]]; then
dst_dir="/dst"
else
dst_dir="/data"
fi
# update shapefile in a temporary directory
# and resample geotiff if necessary
# FUTURE: TileDB will be updated in-place
# generate_tiles.py will make a temporary directory
out_dir="${dst_dir}/tiles/${satellite_id}/${instrument}/${data_type}"
# if we are going to remap, separate the geotiffs and shapefile from unremapped
if [[ ${TILE_ARGS} == *"--remap"* ]]; then
echo "Adding '_ll' prefix to tile output directory because of remapping"
out_dir="${out_dir}_ll"
mkdir -p ${out_dir}
# add string formatting portion to separate add 'product' sub-directory
out_dir="${out_dir}/{product}_ll"
else
mkdir -p ${out_dir}
# add string formatting portion to separate add 'product' sub-directory
out_dir="${out_dir}/{product}"
fi
echo "Generating tiles in directory: ${out_dir}"
python3 generate_tiles.py ${TILE_ARGS} -p ${G2G_PRODUCTS} -- ${out_dir} ${path}
# OUT/<product>/<product>.shp
glob_pattern="${out_dir}/*/*.shp"
# Remove the /data prefix
glob_pattern="${glob_pattern/${dst_dir}\//}"
amqpsend_topic="data.${satellite_family}.${satellite_id}.${instrument}.${data_type}.l1b.tiledb.complete"
# json_info="{path: ${glob_pattern}, satellite_family: ${satellite_family}, satellite_ID: ${satellite_id}, instrument: ${instrument}, data_type: ${data_type}}"
json_info="{\"path\": \"${glob_pattern}\", \"satellite_family\": \"${satellite_family}\", \"satellite_ID\": \"${satellite_id}\", \"instrument\": \"${instrument}\", \"data_type\": \"${data_type}\"}"
echo -e "[[\"$amqpsend_topic\", $json_info]]" | python3 /work/amqpfind/amqpsend.py ${AMQPSEND_ARGS}
echo "Done generating tiles for ${path}"
}
export -f run_tile_gen
echo "Listening to AMQP messages with topic \"$AMQPFIND_TOPIC\""
python3 amqpfind/amqpfind.py ${AMQPFIND_ARGS} -C "${AMQPFIND_TOPIC}" -j "{satellite_family} {satellite_ID} {instrument} {data_type} \'{path}\'" | xargs -I{} -P3 -n1 bash -c "run_tile_gen {}"
tile_gen/tile_index.py 0 → 100644
+ 160
− 0
View file @ 67d07dd3
import sys
import fiona
from fiona.crs import from_epsg
import rasterio
from shapely.geometry import mapping, box
import argparse
import datetime
import glob
import logging
import os
import re
import shutil
# Remap geostationary to EPSG 4326
# gdalwarp -t_srs EPSG:4326 in.tif out.tif
# Note if using docker tiledb-geospatial image then requires shapely - `pip3 install shapely`
# export CPL_DEBUG=ON - GDAL
# export MS_DEBUGLEVEL=6 - MAPSERVER
# python3 tile_index.py -dir data
# sample mapserver queries
# mapserv -nh "QUERY_STRING=map=goes.map&request=GetCapabilities&service=WMS&version=1.1.1"
# mapserv -nh "QUERY_STRING=map=goes.map&request=GetMap&service=WMS&version=1.1.1&layers=goes_abi&srs=EPSG:4326&bbox=-180,-90,180,90&format=image/jpeg&WIDTH=1000&HEIGHT=1000&TIME=2019-12-12T19:10:18" > out.jpg
# TODO add overview example to mapserver
logger = logging.getLogger(__name__)
temporal_schema = {
'geometry': 'Polygon',
'properties': {
'location': 'str',
'time': 'str:19'
}
}
possible_time_regex = (
(re.compile(r'\d{4}\d{2}\d{2}_\d{2}\d{2}\d{2}'), '%Y%m%d_%H%M%S'),
(re.compile(r'\d{4}\d{2}\d{2}T\d{2}\d{2}\d{2}'), '%Y%m%dT%H%M%S'),
)
def get_file_time(fn):
for regex, time_fmt in possible_time_regex:
matches = regex.findall(fn)
if matches:
return datetime.datetime.strptime(matches[-1], time_fmt)
else:
raise ValueError("Unknown filename scheme, can't determine file time.")
def index(input_files, output_shapefile):
"""Create shapefile for location and times of provided geotiffs or tileDB arrays.
Note: All layers to be included in the shapefile must be provided all at
once. Repeated calls to this function will overwrite existing
shapefile information.
"""
import tempfile
out_dir, shp_fn = os.path.split(output_shapefile)
tmp_dir = tempfile.mkdtemp("_tile_index")
tmp_shapefile = os.path.join(tmp_dir, shp_fn)
with fiona.open(tmp_shapefile, 'w', driver='ESRI Shapefile',
schema=temporal_schema) as output:
for f in input_files:
try:
dt = get_file_time(f)
except ValueError:
logger.error(f"Can't time for file {f}")
continue
logger.info(f"Indexing {f} {dt.isoformat()}")
with rasterio.open(f) as src:
g = box(*src.bounds)
output.write(
{
'geometry': mapping(g),
'properties': {
'location': f,
'time': dt.isoformat()
}
})
# move the shapefile contents to the final destination
for fn in os.listdir(tmp_dir):
shutil.move(os.path.join(tmp_dir, fn), os.path.join(out_dir, fn))
# we don't need the temporary directory anymore
shutil.rmtree(tmp_dir, ignore_errors=True)
# def index(src_dir, output):
# files = glob.glob(os.path.join(src_dir, '*.tif'))
# folders = [os.path.join(src_dir, o) for o in os.listdir(src_dir) if os.path.isdir(os.path.join(src_dir, o))]
#
# with fiona.open(output, 'w', driver='ESRI Shapefile',
# schema=temporal_schema) as output:
# # simple toggle between indexing tiff files or tiledb arrays
# if len(files) > 0:
# it = files
# else:
# it = folders
#
# for f in it:
# parts = f.split('_')
# tstamp = parts[5] + parts[6]
# dt = datetime.datetime.strptime(tstamp, '%Y%m%d%H%M%S')
# logger.info(f"Indexing {f} {dt.isoformat()}")
# with rasterio.open(f) as src:
# g = box(*src.bounds)
#
# output.write(
# {
# 'geometry': mapping(g),
# 'properties': {
# 'location': f,
# 'time': dt.isoformat()
# }
# })
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-d', '--directory',
help='Single directory to search for .tif files or TileDB array directories.')
parser.add_argument('-o', '--output', default='img_index.shp')
parser.add_argument('input_files', nargs='*',
help='TileDB directories or GeoTIFF files to ingest.')
args = parser.parse_args()
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s"
)
if args.input_files:
logger.info("Using input files...")
it = args.input_files
else:
src_dir = args.directory
files = glob.glob(os.path.join(src_dir, '*.tif'))
folders = [os.path.join(src_dir, o) for o in os.listdir(src_dir) if os.path.isdir(os.path.join(src_dir, o))]
# simple toggle between indexing tiff files or tiledb arrays
if len(files) > 0:
it = files
else:
it = folders
if not it:
raise ValueError("No valid inputs provided.")
if args.directory:
logger.info('Indexer starting')
index(it, args.output)
if __name__ == "__main__":
sys.exit(main())
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment