diff --git a/mapserver/Dockerfile b/mapserver/Dockerfile
index 77ecdd8ee29840873e21552b1be0f0291c830fc4..4982922f612c61545f4c07e9763f387df8934e28 100644
--- a/mapserver/Dockerfile
+++ b/mapserver/Dockerfile
@@ -1,57 +1,29 @@
-FROM ubuntu:eoan
+FROM tiledb/tiledb-geospatial:latest
+
+WORKDIR /work
+
+# FIXME: Remove once added to parent image
+RUN pip3 install shapely
RUN apt-get -y update && \
apt-get -y upgrade && \
apt-get -y install gcc g++ cmake sqlite3 libsqlite3-dev pkg-config bash-completion curl subversion && \
apt-get -y clean
-# proj 6.1.1
-RUN mkdir -p /build/proj && \
- cd /build/proj && \
- curl -O https://download.osgeo.org/proj/proj-6.1.1.tar.gz && \
- tar xf proj-6.1.1.tar.gz && \
- cd proj-6.1.1 && \
- ./configure && \
- make -j $(nproc) && \
- make install && \
- rm -rf /build/proj
-
-# mkdir build && \
-# cd build && \
-# cmake .. && \
-# cmake --build . && \
-# gdal 3.0.1
-RUN mkdir -p /build/gdal && \
- cd /build/gdal && \
- curl -O http://download.osgeo.org/gdal/3.0.1/gdal-3.0.1.tar.gz && \
- tar xf gdal-3.0.1.tar.gz && \
- cd gdal-3.0.1 && \
- ./configure && \
- make -j$(nproc) && \
- make install && \
- rm -rf /build/gdal
-
-# mapserver
-RUN mkdir -p /build/mapserver && \
- apt-get -y update && \
- DEBIAN_FRONTEND=noninteractive apt-get install -y libprotobuf17 zlib1g-dev libpng-dev libjpeg-turbo8 libfreetype6 libfribidi-dev libharfbuzz-dev libcairo2-dev libfcgi-dev libgeos++-dev postgresql postgis libxml2-dev libgif-dev libjpeg-turbo8-dev libprotobuf-dev protobuf-compiler libprotobuf-c-dev libprotobuf-c1 libprotobuf-dev protobuf-c-compiler && \
- apt-get -y clean && \
- curl -O http://download.osgeo.org/mapserver/mapserver-7.4.2.tar.gz && \
- tar xf mapserver-7.4.2.tar.gz && \
- cd mapserver-7.4.2 && \
- mkdir build && \
- cd build && \
- cmake .. -DWITH_POSTGIS=0 && \
- make -j$(nproc) && \
- make install
-
-# postgres
-#service postgresql start
-
-# postgis
-
-# pgbouncer
+# FIXME: Remove if/when upstream image builds with fcgi
+# Install Mapserver (unstable)
+RUN mkdir -p /build_deps && cd /build_deps \
+ && apt-get -y update \
+# && apt-get install -y libprotobuf17 zlib1g-dev libpng-dev libjpeg-turbo8 libfreetype6 libfribidi-dev libharfbuzz-dev libcairo2-dev libfcgi-dev libgeos++-dev postgresql postgis libxml2-dev libgif-dev libjpeg-turbo8-dev libprotobuf-dev protobuf-compiler libprotobuf-c-dev libprotobuf-c1 libprotobuf-dev protobuf-c-compiler \
+ && apt-get install -y zlib1g-dev libpng-dev libjpeg-turbo8 libfreetype6 libfribidi-dev libharfbuzz-dev libcairo2-dev libfcgi-dev libgeos++-dev postgresql postgis libxml2-dev libgif-dev libjpeg-turbo8-dev libprotobuf-dev protobuf-compiler libprotobuf-c-dev libprotobuf-c1 libprotobuf-dev protobuf-c-compiler \
+ && apt-get -y clean \
+ && git clone https://github.com/mapserver/mapserver.git && cd mapserver \
+ && git checkout 0fcc810f0b559c800f950db78a79fa6574799f23 \
+ && mkdir -p build && cd build \
+ && cmake .. -DWITH_PROTOBUFC=OFF -DWITH_POSTGIS=OFF \
+ && make -j$(nproc) \
+ && make install
# apache
# http://www.inanzzz.com/index.php/post/rhsb/running-apache-server-as-foreground-on-ubuntu-with-dockerfile
@@ -66,35 +38,36 @@ ENV APACHE_SERVER_NAME localhost
# install httpd runtime dependencies
# https://httpd.apache.org/docs/2.4/install.html#requirements
-RUN apt-get -y install apache2 libapache2-mod-fcgid && \
+RUN cd /build_deps && apt-get -y install apache2 libapache2-mod-fcgid && \
ls /etc/apache2/mods-available && \
ls /etc/apache2/mods-enabled && \
- apt-get -y install libapache2-mod-php7.3 php7.3-common php7.3-cli php7.3-fpm php7.3 && \
- a2enmod actions proxy_fcgi setenvif cgi fcgid && \
- a2enconf php7.3-fpm serve-cgi-bin && \
+ apt-get -y install libapache2-mod-php7.2 php7.2-common php7.2-cli php7.2-fpm php7.2 && \
+ a2enmod actions proxy_fcgi setenvif cgi fcgid rewrite && \
+ a2enconf php7.2-fpm serve-cgi-bin && \
apt-get -y clean && \
rm -rf /var/lib/apt/lists/*
-COPY apache-conf /etc/apache2/apache2.conf
+COPY site-conf /etc/apache2/sites-available/cspp_geo.conf
+# disable the default which would conflict with our custom
+RUN a2ensite cspp_geo && a2dissite 000-default
+COPY cgi-bin/* /usr/lib/cgi-bin/
# Point apache to the mapserver binary
RUN ln -s /usr/local/bin/mapserv /usr/lib/cgi-bin/mapserv && \
- chown ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/mapserv && \
- chown -h ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/mapserv
+ chown ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/* && \
+ chown -h ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/*
+
+COPY mapfiles/ /work/mapfiles/
+COPY html/ /var/www/html/
+
+# Add our own custom EPSG codes (HACK)
+# GOES-16 ABI Full Disk = EPSG:930916
+# GOES-17 ABI Full Disk = EPSG:930917
+COPY sql/ /work/sql/
+RUN sqlite3 -init /work/sql/goesr_crs.sql /usr/local/share/proj/proj.db
# https://httpd.apache.org/docs/2.4/stopping.html#gracefulstop
STOPSIGNAL WINCH
EXPOSE 80
CMD ["/usr/sbin/apache2ctl", "-DFOREGROUND"]
-
-##############################################################
-
-
-
-
-
-# php
-
-# python? (for shp2csv type stuff)
-
diff --git a/mapserver/README.md b/mapserver/README.md
index 3bc0a906305e80f48d8e60e5f7539f7861413308..7361c7f4a16ccdd6fa4854c5ed4fcdf4007097d9 100644
--- a/mapserver/README.md
+++ b/mapserver/README.md
@@ -3,9 +3,23 @@
## Usage
```bash
-docker run -p 8888:80 -d gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapserver:latest
+docker run -p 8888:80 -d --rm --name cspp-geo-mapserver -v cspp-geo-abi-l1b-geotiffs:/data gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapserver:latest
```
Then the main mapserv CGI script can be accessed with:
http://localhost:8888/cgi-bin/mapserv
+
+## Special Notes
+
+This image has had its installation of the PROJ library modified to include
+new non-standard EPSG codes to support the projections of certain satellite
+instruments. These additions are part of the Dockerfile build process and
+include the following definitions:
+
+1. EPSG:930916 - The GOES-16 ABI Fixed Grid geostationary coordinate reference
+ system. The numbers are meant to represent "GEOG16" while also attempting
+ to avoid conflicts with future additions to the official EPSG database.
+2. EPSG:930917 - The GOES-17 ABI Fixed Grid geostationary coordinate reference
+ system. The numbers are meant to represent "GEOG17" while also attempting
+ to avoid conflicts with future additions to the official EPSG database.
diff --git a/mapserver/apache-conf b/mapserver/apache-conf
deleted file mode 100644
index 2560da5bb9090684b7a263dd90d2ce668cb72c4d..0000000000000000000000000000000000000000
--- a/mapserver/apache-conf
+++ /dev/null
@@ -1,227 +0,0 @@
-# This is the main Apache server configuration file. It contains the
-# configuration directives that give the server its instructions.
-# See http://httpd.apache.org/docs/2.4/ for detailed information about
-# the directives and /usr/share/doc/apache2/README.Debian about Debian specific
-# hints.
-#
-#
-# Summary of how the Apache 2 configuration works in Debian:
-# The Apache 2 web server configuration in Debian is quite different to
-# upstream's suggested way to configure the web server. This is because Debian's
-# default Apache2 installation attempts to make adding and removing modules,
-# virtual hosts, and extra configuration directives as flexible as possible, in
-# order to make automating the changes and administering the server as easy as
-# possible.
-
-# It is split into several files forming the configuration hierarchy outlined
-# below, all located in the /etc/apache2/ directory:
-#
-# /etc/apache2/
-# |-- apache2.conf
-# | `-- ports.conf
-# |-- mods-enabled
-# | |-- *.load
-# | `-- *.conf
-# |-- conf-enabled
-# | `-- *.conf
-# `-- sites-enabled
-# `-- *.conf
-#
-#
-# * apache2.conf is the main configuration file (this file). It puts the pieces
-# together by including all remaining configuration files when starting up the
-# web server.
-#
-# * ports.conf is always included from the main configuration file. It is
-# supposed to determine listening ports for incoming connections which can be
-# customized anytime.
-#
-# * Configuration files in the mods-enabled/, conf-enabled/ and sites-enabled/
-# directories contain particular configuration snippets which manage modules,
-# global configuration fragments, or virtual host configurations,
-# respectively.
-#
-# They are activated by symlinking available configuration files from their
-# respective *-available/ counterparts. These should be managed by using our
-# helpers a2enmod/a2dismod, a2ensite/a2dissite and a2enconf/a2disconf. See
-# their respective man pages for detailed information.
-#
-# * The binary is called apache2. Due to the use of environment variables, in
-# the default configuration, apache2 needs to be started/stopped with
-# /etc/init.d/apache2 or apache2ctl. Calling /usr/bin/apache2 directly will not
-# work with the default configuration.
-
-
-# Global configuration
-#
-
-#
-# ServerRoot: The top of the directory tree under which the server's
-# configuration, error, and log files are kept.
-#
-# NOTE! If you intend to place this on an NFS (or otherwise network)
-# mounted filesystem then please read the Mutex documentation (available
-# at <URL:http://httpd.apache.org/docs/2.4/mod/core.html#mutex>);
-# you will save yourself a lot of trouble.
-#
-# Do NOT add a slash at the end of the directory path.
-#
-#ServerRoot "/etc/apache2"
-
-#
-# The accept serialization lock file MUST BE STORED ON A LOCAL DISK.
-#
-#Mutex file:${APACHE_LOCK_DIR} default
-
-#
-# The directory where shm and other runtime files will be stored.
-#
-
-DefaultRuntimeDir ${APACHE_RUN_DIR}
-
-#
-# PidFile: The file in which the server should record its process
-# identification number when it starts.
-# This needs to be set in /etc/apache2/envvars
-#
-PidFile ${APACHE_PID_FILE}
-
-#
-# Timeout: The number of seconds before receives and sends time out.
-#
-Timeout 300
-
-#
-# KeepAlive: Whether or not to allow persistent connections (more than
-# one request per connection). Set to "Off" to deactivate.
-#
-KeepAlive On
-
-#
-# MaxKeepAliveRequests: The maximum number of requests to allow
-# during a persistent connection. Set to 0 to allow an unlimited amount.
-# We recommend you leave this number high, for maximum performance.
-#
-MaxKeepAliveRequests 100
-
-#
-# KeepAliveTimeout: Number of seconds to wait for the next request from the
-# same client on the same connection.
-#
-KeepAliveTimeout 5
-
-
-# These need to be set in /etc/apache2/envvars
-User ${APACHE_RUN_USER}
-Group ${APACHE_RUN_GROUP}
-
-#
-# HostnameLookups: Log the names of clients or just their IP addresses
-# e.g., www.apache.org (on) or 204.62.129.132 (off).
-# The default is off because it'd be overall better for the net if people
-# had to knowingly turn this feature on, since enabling it means that
-# each client request will result in AT LEAST one lookup request to the
-# nameserver.
-#
-HostnameLookups Off
-
-# ErrorLog: The location of the error log file.
-# If you do not specify an ErrorLog directive within a <VirtualHost>
-# container, error messages relating to that virtual host will be
-# logged here. If you *do* define an error logfile for a <VirtualHost>
-# container, that host's errors will be logged there and not here.
-#
-ErrorLog ${APACHE_LOG_DIR}/error.log
-
-#
-# LogLevel: Control the severity of messages logged to the error_log.
-# Available values: trace8, ..., trace1, debug, info, notice, warn,
-# error, crit, alert, emerg.
-# It is also possible to configure the log level for particular modules, e.g.
-# "LogLevel info ssl:warn"
-#
-LogLevel warn
-
-# Include module configuration:
-IncludeOptional mods-enabled/*.load
-IncludeOptional mods-enabled/*.conf
-
-# Include list of ports to listen on
-Include ports.conf
-
-
-# Sets the default security model of the Apache2 HTTPD server. It does
-# not allow access to the root filesystem outside of /usr/share and /var/www.
-# The former is used by web applications packaged in Debian,
-# the latter may be used for local directories served by the web server. If
-# your system is serving content from a sub-directory in /srv you must allow
-# access here, or in any related virtual host.
-<Directory />
- Options FollowSymLinks
- AllowOverride None
- Require all denied
-</Directory>
-
-<Directory /usr/share>
- AllowOverride None
- Require all granted
-</Directory>
-
-<Directory /var/www/>
- Options Indexes FollowSymLinks
- AllowOverride None
- Require all granted
-</Directory>
-
-#<Directory /srv/>
-# Options Indexes FollowSymLinks
-# AllowOverride None
-# Require all granted
-#</Directory>
-
-
-
-
-# AccessFileName: The name of the file to look for in each directory
-# for additional configuration directives. See also the AllowOverride
-# directive.
-#
-AccessFileName .htaccess
-
-#
-# The following lines prevent .htaccess and .htpasswd files from being
-# viewed by Web clients.
-#
-<FilesMatch "^\.ht">
- Require all denied
-</FilesMatch>
-
-
-#
-# The following directives define some format nicknames for use with
-# a CustomLog directive.
-#
-# These deviate from the Common Log Format definitions in that they use %O
-# (the actual bytes sent including headers) instead of %b (the size of the
-# requested file), because the latter makes it impossible to detect partial
-# requests.
-#
-# Note that the use of %{X-Forwarded-For}i instead of %h is not recommended.
-# Use mod_remoteip instead.
-#
-LogFormat "%v:%p %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" vhost_combined
-LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" combined
-LogFormat "%h %l %u %t \"%r\" %>s %O" common
-LogFormat "%{Referer}i -> %U" referer
-LogFormat "%{User-agent}i" agent
-
-# Include of directories ignores editors' and dpkg's backup files,
-# see README.Debian for details.
-
-# Include generic snippets of statements
-IncludeOptional conf-enabled/*.conf
-
-# Include the virtual host configurations:
-IncludeOptional sites-enabled/*.conf
-
-# vim: syntax=apache ts=4 sw=4 sts=4 sr noet
\ No newline at end of file
diff --git a/mapserver/cgi-bin/layer_times.py b/mapserver/cgi-bin/layer_times.py
new file mode 100755
index 0000000000000000000000000000000000000000..8666f0bf65e2d060afb56c9d6a89c8556f090887
--- /dev/null
+++ b/mapserver/cgi-bin/layer_times.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+import cgi
+import json
+import fiona
+
+form = cgi.FieldStorage()
+layer = form['layer'].value
+with fiona.open(layer, 'r') as shp_file:
+ times = [x['properties']['time'] for x in shp_file]
+
+print("Content-Type: application/json")
+print("Access-Control-Allow-Origin: *")
+print() # blank line, end of headers
+print(json.dumps(times))
diff --git a/mapserver/html/index.html b/mapserver/html/index.html
new file mode 100644
index 0000000000000000000000000000000000000000..e6ed5388e3338d35984e82cbf86ed938e093f8c6
--- /dev/null
+++ b/mapserver/html/index.html
@@ -0,0 +1,21 @@
+<html>
+<head></head>
+<body>
+<h1>CSPP Geo Geo2Grid Tile Server</h1>
+
+This tile server for the CSPP Geo Geo2Grid project.
+<br/>
+<br/>
+To access tiles using a standard WMS protocol:
+<br/>
+<br/>
+http://localhost/wms/goes16/abi/fldk/l1b?...
+<br/>
+<br/>
+Where "fldk" stands for Full Disk and "..." represents all remaining WMS
+parameters. Other sectors available are "m01", "m02", and "conus".
+<br/>
+<br/>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/mapserver/mapfiles/g16_abi_radf_l1b.map b/mapserver/mapfiles/g16_abi_radf_l1b.map
new file mode 100644
index 0000000000000000000000000000000000000000..d989dc4c112317afecca7c6e324e3ec384f5784b
--- /dev/null
+++ b/mapserver/mapfiles/g16_abi_radf_l1b.map
@@ -0,0 +1,109 @@
+MAP
+ IMAGETYPE PNG
+ SIZE 256 256
+ EXTENT -180 -90 180 90
+
+ PROJECTION
+ "init=epsg:930916"
+ END
+
+ WEB
+ METADATA
+ "wms_title" "Weather data"
+ "wms_onlineresource" "https://myhost/mapserv"
+ "wms_enable_request" "*"
+ END
+ END
+
+
+ ##############
+ # True Color #
+ ##############
+
+ LAYER
+ NAME "true_color_index"
+ TYPE TILEINDEX
+ DATA "/data/tiles/g16/abi/radf/true_color/true_color"
+ END
+
+ LAYER
+ NAME "true_color"
+ TYPE RASTER
+ TILEITEM "location"
+ TILEINDEX "true_color_index"
+ # Comment below to default to transparency
+ # OFFSITE 0 0 0
+ PROJECTION
+ "init=epsg:930916"
+ END
+ METADATA
+ "wms_title" "GOES-16 ABI True Color"
+ "wms_extent" "-180 -90 180 90"
+ "wms_timeextent" "2017-01-01/2020-12-31"
+ "wms_timeformat" "YYYY-MM-DDTHH:MM:SS"
+ "wms_timeitem" "time" # time is a metadata item
+ "wms_timedefault" "2019-12-12T19:20:18"
+ "wms_enable_request" "*"
+ END
+ FILTER (`[time]` = `2019-12-12T19:20:18`)
+ END # goes raster layer ends here
+
+ ######################
+ # True Color Lat/Lon #
+ ######################
+
+ LAYER
+ NAME "true_color_index_ll"
+ TYPE TILEINDEX
+ DATA "/data/tiles/g16/abi/radf_ll/true_color_ll/true_color_ll"
+ END
+
+ LAYER
+ NAME "true_color_ll"
+ TYPE RASTER
+ TILEITEM "location"
+ TILEINDEX "true_color_index_ll"
+ # Comment below to default to transparency
+ # OFFSITE 0 0 0
+ PROJECTION
+ "init=epsg:4326"
+ END
+ METADATA
+ "wms_title" "GOES-16 ABI True Color"
+ "wms_extent" "-180 -90 180 90"
+ "wms_timeextent" "2017-01-01/2020-12-31"
+ "wms_timeformat" "YYYY-MM-DDTHH:MM:SS"
+ "wms_timeitem" "time" # time is a metadata item
+ "wms_timedefault" "2019-12-12T19:20:18"
+ "wms_enable_request" "*"
+ END
+ FILTER (`[time]` = `2019-12-12T19:20:18`)
+ END # goes raster layer ends here
+
+ ###################
+ # True Color Fake #
+ ###################
+
+ LAYER
+ NAME "true_color_test"
+ TYPE RASTER
+ TILEITEM "location"
+ TILEINDEX "true_color_index"
+ # Comment below to default to transparency
+ # OFFSITE 0 0 0
+ PROJECTION
+ "init=epsg:930916"
+ END
+ METADATA
+ "wms_title" "GOES-16 ABI True Color"
+ "wms_extent" "-180 -90 180 90"
+ "wms_timeextent" "2017-01-01/2020-12-31"
+ "wms_timeformat" "YYYY-MM-DDTHH:MM:SS"
+ "wms_timeitem" "time" # time is a metadata item
+ "wms_timedefault" "2019-12-12T19:20:18"
+ "wms_enable_request" "*"
+ END
+ FILTER (`[time]` = `2019-12-12T19:20:18`)
+ END # goes raster layer ends here
+
+END # end of map file
diff --git a/mapserver/site-conf b/mapserver/site-conf
new file mode 100644
index 0000000000000000000000000000000000000000..0c812d2e18e49146c1dfb537f702802b4f65e8f5
--- /dev/null
+++ b/mapserver/site-conf
@@ -0,0 +1,43 @@
+<VirtualHost *:80>
+ # The ServerName directive sets the request scheme, hostname and port that
+ # the server uses to identify itself. This is used when creating
+ # redirection URLs. In the context of virtual hosts, the ServerName
+ # specifies what hostname must appear in the request's Host: header to
+ # match this virtual host. For the default virtual host (this file) this
+ # value is not decisive as it is used as a last resort host regardless.
+ # However, you must set it for any further virtual host explicitly.
+ #ServerName www.example.com
+
+ ServerAdmin david.hoese@ssec.wisc.edu
+ DocumentRoot /var/www/html
+
+ # Available loglevels: trace8, ..., trace1, debug, info, notice, warn,
+ # error, crit, alert, emerg.
+ # It is also possible to configure the loglevel for particular
+ # modules, e.g.
+ #LogLevel info ssl:warn
+
+ ErrorLog ${APACHE_LOG_DIR}/error.log
+ CustomLog ${APACHE_LOG_DIR}/access.log combined
+
+ # For most configuration files from conf-available/, which are
+ # enabled or disabled at a global level, it is possible to
+ # include a line for only one particular virtual host. For example the
+ # following line enables the CGI configuration for this host only
+ # after it has been globally disabled with "a2disconf".
+ #Include conf-available/serve-cgi-bin.conf
+
+ # Custom CSPP Geo
+ # LoadModule rewrite_module modules/mod_rewrite.so
+ RewriteEngine on
+ # /wms/goes16/abi/fldk/
+ RewriteRule "^/wms/([^/]+)/([^/]+)/([^/]+)/l1b?(.*)" "/cgi-bin/mapserv?map=/work/mapfiles/$1_$2_$3_l1b.map&$4" [PT,QSA]
+ # /wms_times/g16/abi/radf/true_color
+ RewriteRule "^/wms_times/([^/]+)/([^/]+)/([^/]+)/([^/]+)" "/cgi-bin/layer_times.py?layer=/data/tiles/$1/$2/$3/$4/$4.shp" [PT,QSA]
+ # FIXME: We need to include the sector
+ # /data/goes/grb/goes16/2020/2020_01_21_021/abi/L1b/RadF/GOES-16_ABI_RadF_C01_20200121_000016_GOES-East.tif
+ # "/data/tiles/g16/abi/radf/true_color/true_color"
+ LogLevel alert rewrite:trace6
+</VirtualHost>
+
+# vim: syntax=apache ts=4 sw=4 sts=4 sr noet
\ No newline at end of file
diff --git a/mapserver/sql/goesr_crs.sql b/mapserver/sql/goesr_crs.sql
new file mode 100644
index 0000000000000000000000000000000000000000..0cbf495a70ae4aa8f38bd1da6fc4c3a9cd036e3e
--- /dev/null
+++ b/mapserver/sql/goesr_crs.sql
@@ -0,0 +1,2 @@
+INSERT INTO projected_crs (auth_name, code, name, description, scope, coordinate_system_auth_name, coordinate_system_code, geodetic_crs_auth_name, geodetic_crs_code, conversion_auth_name, conversion_code, area_of_use_auth_name, area_of_use_code, text_definition, deprecated) VALUES ('EPSG', '930917', 'GOES-17 ABI Fixed Grid', 'GOES-17 (GOES-WEST) ABI Fixed Grid in the Geostationary projection', null, null, null, 'EPSG', '4269', null, null, null, null, 'PROJCRS["GOES-17 ABI Fixed Grid",BASEGEOGCRS["GOES-17 ABI Fixed Grid",DATUM["North American Datum 1983",ELLIPSOID["GRS 1980",6378137,298.257222101,LENGTHUNIT["metre",1]],ID["EPSG",6269]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8901]]],CONVERSION["unknown",METHOD["Geostationary Satellite (Sweep X)"],PARAMETER["Longitude of natural origin",-137,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8802]],PARAMETER["Satellite Height",35786023,LENGTHUNIT["metre",1,ID["EPSG",9001]]],PARAMETER["False easting",0,LENGTHUNIT["metre",1],ID["EPSG",8806]],PARAMETER["False northing",0,LENGTHUNIT["metre",1],ID["EPSG",8807]]],CS[Cartesian,2],AXIS["(E)",east,ORDER[1],LENGTHUNIT["metre",1,ID["EPSG",9001]]],AXIS["(N)",north,ORDER[2],LENGTHUNIT["metre",1,ID["EPSG",9001]]]]', 0);
+INSERT INTO projected_crs (auth_name, code, name, description, scope, coordinate_system_auth_name, coordinate_system_code, geodetic_crs_auth_name, geodetic_crs_code, conversion_auth_name, conversion_code, area_of_use_auth_name, area_of_use_code, text_definition, deprecated) VALUES ('EPSG', '930916', 'GOES-16 ABI Fixed Grid', 'GOES-16 (GOES-WEST) ABI Fixed Grid in the Geostationary projection', null, null, null, 'EPSG', '4269', null, null, null, null, 'PROJCRS["GOES-16 ABI Fixed Grid",BASEGEOGCRS["GOES-16 ABI Fixed Grid",DATUM["North American Datum 1983",ELLIPSOID["GRS 1980",6378137,298.257222101,LENGTHUNIT["metre",1]],ID["EPSG",6269]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8901]]],CONVERSION["unknown",METHOD["Geostationary Satellite (Sweep X)"],PARAMETER["Longitude of natural origin",-75,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8802]],PARAMETER["Satellite Height",35786023,LENGTHUNIT["metre",1,ID["EPSG",9001]]],PARAMETER["False easting",0,LENGTHUNIT["metre",1],ID["EPSG",8806]],PARAMETER["False northing",0,LENGTHUNIT["metre",1],ID["EPSG",8807]]],CS[Cartesian,2],AXIS["(E)",east,ORDER[1],LENGTHUNIT["metre",1,ID["EPSG",9001]]],AXIS["(N)",north,ORDER[2],LENGTHUNIT["metre",1,ID["EPSG",9001]]]]', 0);
\ No newline at end of file
diff --git a/mapserver/tiledb_geospatial_Dockerfile b/mapserver/tiledb_geospatial_Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..f80985ec2029f49535e75bbf5a7d205e5528b913
--- /dev/null
+++ b/mapserver/tiledb_geospatial_Dockerfile
@@ -0,0 +1,159 @@
+# Based on
+
+FROM ubuntu:18.04
+LABEL maintainer="support@tiledb.io"
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TZ=GMT
+ENV LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH}
+ENV LANG=C.UTF-8
+ENV LC_ALL=C.UTF-8
+
+RUN apt-get update && apt-get install -y \
+ gosu \
+ pwgen \
+ tzdata \
+ gcc \
+ g++ \
+ build-essential \
+ cmake \
+ sqlite \
+ libsqlite3-dev \
+ libxml2-dev \
+ libjpeg-dev \
+ libpng-dev \
+ libfreetype6-dev \
+ libzstd-dev \
+ python3-pip \
+ git \
+ wget \
+ && rm -rf /var/lib/apt/lists/*
+
+# Install tiledb using 1.7.2 release
+RUN mkdir -p /build_deps && cd /build_deps \
+ && git clone https://github.com/TileDB-Inc/TileDB.git -b 1.7.2 && cd TileDB \
+ && mkdir -p build && cd build \
+ && cmake -DTILEDB_VERBOSE=ON -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. \
+ && make -j$(nproc) \
+ && make -C tiledb install
+
+# Install curl after building tiledb
+RUN apt-get update && apt-get install -y \
+ libcurl4 \
+ libcurl4-openssl-dev \
+ && rm -rf /var/lib/apt/lists/*
+
+# Install OpenJPEG
+RUN cd /build_deps \
+ && git clone https://github.com/uclouvain/openjpeg.git -b v2.2.0 && cd openjpeg \
+ && mkdir -p build && cd build \
+ && cmake .. \
+ && make -j$(nproc) \
+ && make install
+
+# Install libtiff
+RUN cd /build_deps \
+ && wget --no-check-certificate https://download.osgeo.org/libtiff/tiff-4.1.0.tar.gz \
+ && tar -zxf tiff-4.1.0.tar.gz \
+ && cd tiff-4.1.0 \
+ && ./configure \
+ && make \
+ && make install
+
+# Install Proj
+RUN cd /build_deps \
+ && git clone https://github.com/OSGeo/PROJ.git -b 6.2.1 && cd PROJ \
+ && mkdir -p build && cd build \
+ && cmake .. \
+ && make -j$(nproc) \
+ && make install
+
+# Install libgeotiff
+RUN cd /build_deps \
+ && wget --no-check-certificate https://download.osgeo.org/geotiff/libgeotiff/libgeotiff-1.5.1.tar.gz \
+ && tar -zxf libgeotiff-1.5.1.tar.gz \
+ && cd libgeotiff-1.5.1 \
+ && mkdir -p build && cd build \
+ && cmake .. \
+ && make \
+ && make install
+
+# Install GDAL
+RUN cd /build_deps \
+ && git clone https://github.com/OSGeo/gdal.git && cd gdal/gdal \
+ && git checkout c99a871a7bdedc751c503bb8cf508d9016510fe0 \
+ && ./configure --with-crypto=no --with-curl=no \
+ && make -j$(nproc) \
+ && make install
+
+## Install TileDB-Py
+RUN cd /build_deps \
+ && pip3 install numpy \
+ && git clone https://github.com/TileDB-Inc/TileDB-Py.git -b 0.5.3 \
+ && cd TileDB-Py && python3 setup.py install
+
+## Install XArray
+RUN cd /build_deps && pip3 install xarray
+
+## Install Dask
+RUN cd /build_deps \
+ && pip3 install toolz && pip3 install dask_image \
+ && git clone https://github.com/dask/dask.git && cd dask \
+ && git checkout 807f3225cf840f28ce7cf89b88fea63d473889e7 \
+ && python3 setup.py install \
+ && pip3 install dask distributed --upgrade \
+ && pip3 install dask-image
+
+# Install Rasterio
+RUN cd /build_deps && pip3 install cython
+RUN cd /build_deps \
+ && git clone https://github.com/mapbox/rasterio.git -b 1.1.0 && cd rasterio \
+ && python3 setup.py install
+
+# Install Fiona
+RUN cd /build_deps \
+ && git clone https://github.com/Toblerity/Fiona.git && cd Fiona \
+ && python3 setup.py install
+
+# Install TileDB-SAR
+RUN cd /build_deps \
+ && git clone https://github.com/TileDB-Inc/TileDB-SAR.git && cd TileDB-SAR \
+ && git checkout 888059a15d87ae95fff6dc01c8bd4343ee4eaee1 \
+ && python3 setup.py install
+
+# Install Mapserver
+RUN cd /build_deps \
+ && git clone https://github.com/mapserver/mapserver.git && cd mapserver \
+ && git checkout 0fcc810f0b559c800f950db78a79fa6574799f23 \
+ && mkdir -p build && cd build \
+ && cmake .. -DWITH_GIF=OFF -DWITH_HARFBUZZ=OFF -DWITH_PROTOBUFC=OFF -DWITH_FRIBIDI=OFF -DWITH_POSTGIS=OFF -DWITH_GEOS=OFF -DWITH_FCGI=OFF -DWITH_CAIRO=OFF \
+ && make \
+ && make install
+
+# Install LasZIP
+RUN cd /build_deps \
+ && wget https://github.com/LASzip/LASzip/releases/download/3.4.1/laszip-src-3.4.1.tar.gz \
+ && tar -zxf laszip-src-3.4.1.tar.gz \
+ && cd laszip-src-3.4.1 \
+ && mkdir -p build && cd build \
+ && cmake .. \
+ && make \
+ && make install
+
+# Install PDAL
+RUN cd /build_deps \
+ && git clone https://github.com/PDAL/PDAL.git -b 2.0.1 && cd PDAL \
+ && mkdir -p build && cd build \
+ && cmake .. \
+ && make \
+ && make install
+
+# Install PDAL Python
+RUN pip3 install packaging \
+ && git clone https://github.com/PDAL/python pdalextension \
+ && cd pdalextension \
+ && python3 setup.py build \
+ && python3 setup.py install
+
+# Clean up
+RUN cd /tmp && rm -r /build_deps
diff --git a/tile_gen/Dockerfile b/tile_gen/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..125540c43684370b59e725755a438fbf2982bb40
--- /dev/null
+++ b/tile_gen/Dockerfile
@@ -0,0 +1,16 @@
+FROM tiledb/tiledb-geospatial:latest
+
+WORKDIR /work
+
+# TODO may need the unzip command to be installed if not already
+RUN apt-get update && apt-get install -y unzip && \
+ wget http://ssec.wisc.edu/~rayg/pub/amqpfind.zip && \
+ unzip amqpfind.zip && \
+ rm amqpfind.zip && \
+ rm -rf /var/lib/apt/lists/*
+
+# FIXME: Remove once added to parent image
+RUN pip3 install shapely
+COPY tile_index.py .
+COPY generate_tiles.py .
+COPY run.sh .
diff --git a/tile_gen/README.md b/tile_gen/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..ac6fa4d140449483cfe400b8ab3699813fab4f5f
--- /dev/null
+++ b/tile_gen/README.md
@@ -0,0 +1,31 @@
+# Tile Generation
+
+## Build
+
+```bash
+docker build -t gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/tile_gen:latest tile_gen/
+docker push gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/tile_gen:latest
+```
+
+## Usage
+
+```bash
+docker run -d --rm --network cspp-geo-rabbit --cpus 2 --name cspp-geo-tilegen-g16-radf -e AMQPFIND_TOPIC="data.goes.g16.abi.radf.l1b.geotiff.complete" -v cspp-geo-abi-l1b-geotiffs:/data gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/tile_gen:latest ./run.sh
+```
+
+To run the version of tile generation that remaps to EPSG:4326:
+
+```bash
+docker run -d --rm --network cspp-geo-rabbit --cpus 6 --name cspp-geo-tilegen-g16-radf-ll -e AMQPFIND_TOPIC="data.goes.g16.abi.radf.l1b.geotiff.complete" -e G2G_PRODUCTS="true_color" -e TILE_ARGS="--remap --shape-file {product}_ll.shp" -v cspp-geo-abi-l1b-geotiffs:/data gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/tile_gen:latest ./run.sh
+```
+
+NOTE: For full disk data, remapping all products can take a really long
+time and fall behind. The above command limits processing to "true_color"
+only. To attempt to process all products remove the below portion of the
+command.
+
+```
+-e G2G_PRODUCTS="true_color"
+```
+
+And adding more CPUs.
\ No newline at end of file
diff --git a/tile_gen/generate_tiles.py b/tile_gen/generate_tiles.py
new file mode 100644
index 0000000000000000000000000000000000000000..2312f6b1c800bf047d9eab36a0b3a8306e004793
--- /dev/null
+++ b/tile_gen/generate_tiles.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import warnings
+import logging
+import subprocess
+import shutil
+from glob import glob
+
+import tile_index
+
+LOG = logging.getLogger(__name__)
+
+
+def group_files(products, input_files):
+ """Group input geotiff files by product."""
+ groups = {}
+ for prod in products:
+ prods_files = [f for f in input_files if prod in f]
+ if prods_files:
+ groups[prod] = prods_files
+ if len(groups) != len(products):
+ warnings.warn("Not all product geotiffs were provided.")
+ return groups
+
+
+def remap_to_lonlat(itif, otif):
+ """Remap a single geotiff by calling gdalwarp."""
+ try:
+ subprocess.run(['gdalwarp', '-multi', '-wo', 'NUM_THREADS=ALL_CPUS', '-t_srs', 'EPSG:4326', itif, otif], check=True)
+ except subprocess.CalledProcessError:
+ LOG.error("Could not remap geotiff %s -> %s" % (itif, otif))
+ return None
+ return otif
+
+
+def remap_tifs(input_tifs, out_dir, remap_suffix):
+ """Remap all input geotiffs to EPSG:4326."""
+ for itif in input_tifs:
+ ifn = os.path.basename(itif)
+ otif = os.path.join(out_dir, ifn.replace('.tif', remap_suffix))
+ otif = remap_to_lonlat(itif, otif)
+ if otif is not None:
+ yield otif
+
+
+def link_or_copy(input_tifs, out_dir):
+ """Hardlink input tifs to output directory."""
+ for prod_file in input_tifs:
+ out_file = os.path.join(out_dir, os.path.basename(prod_file))
+ try:
+ os.link(prod_file, out_file)
+ except OSError:
+ # on different mounts probably?
+ shutil.copy2(prod_file, out_file)
+ yield out_file
+
+
+def main():
+ import argparse
+ parser = argparse.ArgumentParser(description="Take input geotiffs and generate mapserver compatible tiles.")
+ parser.add_argument('--remap', action='store_true',
+ help="Remap input geotiffs to EPSG:4326")
+ parser.add_argument('--remap-suffix', default='_ll.tif',
+ help="Replace 'tif' with provided suffix when geotiffs are remapped.")
+ parser.add_argument('-p', '--products', nargs="*",
+ help="Product names to group together in each "
+ "'layer'. Product name must be in the filename.")
+ parser.add_argument('--shape-file', default='{product}.shp',
+ help="Shapefile filename pattern to use and placed "
+ "in the output directory. "
+ "(default: '{product}.shp')")
+ parser.add_argument('out_dir',
+ help="Output path to save tile information to (ex. '/data/tiles/{product}')")
+ parser.add_argument('input_files', nargs="+",
+ help="Input geotiffs to generate tiles for (separate from product lists with '--')")
+ args = parser.parse_args()
+
+ groups = group_files(args.products, args.input_files)
+ for prod, prod_files in groups.items():
+ out_dir = args.out_dir.format(product=prod)
+ os.makedirs(out_dir, exist_ok=True)
+ shp_fn = args.shape_file.format(product=prod)
+ shp_pathname = os.path.join(out_dir, shp_fn)
+
+ if args.remap:
+ # remap if needed
+ prod_files = list(remap_tifs(prod_files, out_dir, args.remap_suffix))
+ else:
+ # hardlink if needed
+ prod_files = list(link_or_copy(prod_files, out_dir))
+
+ # get all products in the current directory
+ ext = os.path.splitext(prod_files[-1])[-1]
+ all_prod_files = sorted(glob(os.path.join(out_dir, '*' + prod + '*' + ext)))
+
+ # create shape file
+ LOG.info("Rebuilding shapefile index with:\n\t{}".format(", ".join(all_prod_files)))
+ tile_index.index(all_prod_files, shp_pathname)
+
+
+
+if __name__ == "__main__":
+ sys.exit(main())
\ No newline at end of file
diff --git a/tile_gen/run.sh b/tile_gen/run.sh
new file mode 100755
index 0000000000000000000000000000000000000000..449169bbe594746a7cd09bca1f245ccb7579816b
--- /dev/null
+++ b/tile_gen/run.sh
@@ -0,0 +1,81 @@
+#!/bin/bash -le
+# Usage: run.sh
+# Environment variables used for configuration:
+# AMQPFIND_ARGS: Arguments to pass to amqpfind when listening for new input
+# events. Should not include the "-C" topic flag (see AMQPFIND_TOPIC).
+# Default: "-H cspp-geo-rabbit -X satellite -u guest -p guest"
+# AMQPFIND_TOPIC: Topic to use for incoming data events.
+# Default: "data.goes.*.abi.*.l1b.netcdf.complete"
+# The first asterisk (3rd element) can limit processing to a particular
+# satellite (ex. `g16`). The second asterisk (5th element) can be used
+# to limit to a particular sector (choices: radf, radc, radm1, radm2)
+# AMQPSEND_ARGS: Arguments to pass to amqpsend when sending out new data
+# events. Default: "-H cspp-geo-rabbit -X satellite -u guest -p guest"
+
+# Verify that the data mount is available
+test -d "/data"
+
+export AMQPFIND_ARGS=${AMQPFIND_ARGS:-"-H cspp-geo-rabbit -X satellite -u guest -p guest"}
+export AMQPSEND_ARGS=${AMQPSEND_ARGS:-"-H cspp-geo-rabbit -X satellite -u guest -p guest"}
+export AMQPFIND_TOPIC=${AMQPFIND_TOPIC:-'data.goes.*.abi.*.l1b.geotiff.complete'}
+export G2G_PRODUCTS=${G2G_PRODUCTS:-"C01 C02 C03 C04 C05 C06 C07 C08 C09 C10 C11 C12 C13 C14 C15 C16 true_color"}
+export TILE_ARGS=${TILE_ARGS:-""}
+export TMPDIR=${TMPDIR:-"/dst/tmp"}
+
+run_tile_gen() {
+ if [[ $# -ne 5 ]]; then
+ echo "Unexpected number of arguments (expected 5): $#"
+ return 1
+ fi
+
+ satellite_family=${1,,}
+ satellite_id=${2,,}
+ instrument=${3,,}
+ data_type=${4,,}
+ path="$5"
+ echo "Starting tile generation processing for ${path}"
+ # convert path from a relative path to an absolute path
+ path="/data/${path}"
+
+ # if /dst isn't defined then use /data
+ if [[ -d "/dst" ]]; then
+ dst_dir="/dst"
+ else
+ dst_dir="/data"
+ fi
+
+ # update shapefile in a temporary directory
+ # and resample geotiff if necessary
+ # FUTURE: TileDB will be updated in-place
+ # generate_tiles.py will make a temporary directory
+ out_dir="${dst_dir}/tiles/${satellite_id}/${instrument}/${data_type}"
+ # if we are going to remap, separate the geotiffs and shapefile from unremapped
+ if [[ ${TILE_ARGS} == *"--remap"* ]]; then
+ echo "Adding '_ll' prefix to tile output directory because of remapping"
+ out_dir="${out_dir}_ll"
+ mkdir -p ${out_dir}
+ # add string formatting portion to separate add 'product' sub-directory
+ out_dir="${out_dir}/{product}_ll"
+ else
+ mkdir -p ${out_dir}
+ # add string formatting portion to separate add 'product' sub-directory
+ out_dir="${out_dir}/{product}"
+ fi
+ echo "Generating tiles in directory: ${out_dir}"
+ python3 generate_tiles.py ${TILE_ARGS} -p ${G2G_PRODUCTS} -- ${out_dir} ${path}
+ # OUT/<product>/<product>.shp
+ glob_pattern="${out_dir}/*/*.shp"
+ # Remove the /data prefix
+ glob_pattern="${glob_pattern/${dst_dir}\//}"
+
+ amqpsend_topic="data.${satellite_family}.${satellite_id}.${instrument}.${data_type}.l1b.tiledb.complete"
+# json_info="{path: ${glob_pattern}, satellite_family: ${satellite_family}, satellite_ID: ${satellite_id}, instrument: ${instrument}, data_type: ${data_type}}"
+ json_info="{\"path\": \"${glob_pattern}\", \"satellite_family\": \"${satellite_family}\", \"satellite_ID\": \"${satellite_id}\", \"instrument\": \"${instrument}\", \"data_type\": \"${data_type}\"}"
+ echo -e "[[\"$amqpsend_topic\", $json_info]]" | python3 /work/amqpfind/amqpsend.py ${AMQPSEND_ARGS}
+ echo "Done generating tiles for ${path}"
+}
+
+export -f run_tile_gen
+echo "Listening to AMQP messages with topic \"$AMQPFIND_TOPIC\""
+python3 amqpfind/amqpfind.py ${AMQPFIND_ARGS} -C "${AMQPFIND_TOPIC}" -j "{satellite_family} {satellite_ID} {instrument} {data_type} \'{path}\'" | xargs -I{} -P3 -n1 bash -c "run_tile_gen {}"
+
diff --git a/tile_gen/tile_index.py b/tile_gen/tile_index.py
new file mode 100644
index 0000000000000000000000000000000000000000..6ff1769ef093322586201601cdc4432279c55e6f
--- /dev/null
+++ b/tile_gen/tile_index.py
@@ -0,0 +1,160 @@
+import sys
+import fiona
+from fiona.crs import from_epsg
+import rasterio
+from shapely.geometry import mapping, box
+
+import argparse
+import datetime
+import glob
+import logging
+import os
+import re
+import shutil
+
+# Remap geostationary to EPSG 4326
+# gdalwarp -t_srs EPSG:4326 in.tif out.tif
+
+
+# Note if using docker tiledb-geospatial image then requires shapely - `pip3 install shapely`
+# export CPL_DEBUG=ON - GDAL
+# export MS_DEBUGLEVEL=6 - MAPSERVER
+# python3 tile_index.py -dir data
+# sample mapserver queries
+# mapserv -nh "QUERY_STRING=map=goes.map&request=GetCapabilities&service=WMS&version=1.1.1"
+# mapserv -nh "QUERY_STRING=map=goes.map&request=GetMap&service=WMS&version=1.1.1&layers=goes_abi&srs=EPSG:4326&bbox=-180,-90,180,90&format=image/jpeg&WIDTH=1000&HEIGHT=1000&TIME=2019-12-12T19:10:18" > out.jpg
+# TODO add overview example to mapserver
+
+logger = logging.getLogger(__name__)
+
+temporal_schema = {
+ 'geometry': 'Polygon',
+ 'properties': {
+ 'location': 'str',
+ 'time': 'str:19'
+ }
+}
+
+possible_time_regex = (
+ (re.compile(r'\d{4}\d{2}\d{2}_\d{2}\d{2}\d{2}'), '%Y%m%d_%H%M%S'),
+ (re.compile(r'\d{4}\d{2}\d{2}T\d{2}\d{2}\d{2}'), '%Y%m%dT%H%M%S'),
+)
+
+
+def get_file_time(fn):
+ for regex, time_fmt in possible_time_regex:
+ matches = regex.findall(fn)
+ if matches:
+ return datetime.datetime.strptime(matches[-1], time_fmt)
+ else:
+ raise ValueError("Unknown filename scheme, can't determine file time.")
+
+
+def index(input_files, output_shapefile):
+ """Create shapefile for location and times of provided geotiffs or tileDB arrays.
+
+ Note: All layers to be included in the shapefile must be provided all at
+ once. Repeated calls to this function will overwrite existing
+ shapefile information.
+
+ """
+ import tempfile
+ out_dir, shp_fn = os.path.split(output_shapefile)
+ tmp_dir = tempfile.mkdtemp("_tile_index")
+ tmp_shapefile = os.path.join(tmp_dir, shp_fn)
+ with fiona.open(tmp_shapefile, 'w', driver='ESRI Shapefile',
+ schema=temporal_schema) as output:
+ for f in input_files:
+ try:
+ dt = get_file_time(f)
+ except ValueError:
+ logger.error(f"Can't time for file {f}")
+ continue
+
+ logger.info(f"Indexing {f} {dt.isoformat()}")
+ with rasterio.open(f) as src:
+ g = box(*src.bounds)
+
+ output.write(
+ {
+ 'geometry': mapping(g),
+ 'properties': {
+ 'location': f,
+ 'time': dt.isoformat()
+ }
+ })
+
+ # move the shapefile contents to the final destination
+ for fn in os.listdir(tmp_dir):
+ shutil.move(os.path.join(tmp_dir, fn), os.path.join(out_dir, fn))
+ # we don't need the temporary directory anymore
+ shutil.rmtree(tmp_dir, ignore_errors=True)
+
+
+# def index(src_dir, output):
+# files = glob.glob(os.path.join(src_dir, '*.tif'))
+# folders = [os.path.join(src_dir, o) for o in os.listdir(src_dir) if os.path.isdir(os.path.join(src_dir, o))]
+#
+# with fiona.open(output, 'w', driver='ESRI Shapefile',
+# schema=temporal_schema) as output:
+# # simple toggle between indexing tiff files or tiledb arrays
+# if len(files) > 0:
+# it = files
+# else:
+# it = folders
+#
+# for f in it:
+# parts = f.split('_')
+# tstamp = parts[5] + parts[6]
+# dt = datetime.datetime.strptime(tstamp, '%Y%m%d%H%M%S')
+# logger.info(f"Indexing {f} {dt.isoformat()}")
+# with rasterio.open(f) as src:
+# g = box(*src.bounds)
+#
+# output.write(
+# {
+# 'geometry': mapping(g),
+# 'properties': {
+# 'location': f,
+# 'time': dt.isoformat()
+# }
+# })
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-d', '--directory',
+ help='Single directory to search for .tif files or TileDB array directories.')
+ parser.add_argument('-o', '--output', default='img_index.shp')
+ parser.add_argument('input_files', nargs='*',
+ help='TileDB directories or GeoTIFF files to ingest.')
+ args = parser.parse_args()
+
+ logging.basicConfig(
+ level=logging.INFO,
+ format="%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s"
+ )
+
+ if args.input_files:
+ logger.info("Using input files...")
+ it = args.input_files
+ else:
+ src_dir = args.directory
+ files = glob.glob(os.path.join(src_dir, '*.tif'))
+ folders = [os.path.join(src_dir, o) for o in os.listdir(src_dir) if os.path.isdir(os.path.join(src_dir, o))]
+ # simple toggle between indexing tiff files or tiledb arrays
+ if len(files) > 0:
+ it = files
+ else:
+ it = folders
+
+ if not it:
+ raise ValueError("No valid inputs provided.")
+
+ if args.directory:
+ logger.info('Indexer starting')
+ index(it, args.output)
+
+
+if __name__ == "__main__":
+ sys.exit(main())