From f58454727a7d86c09af9b788f10d3db2cd0093bc Mon Sep 17 00:00:00 2001
From: David Hoese <david.hoese@ssec.wisc.edu>
Date: Mon, 27 Jul 2020 13:59:17 -0500
Subject: [PATCH] Remove content that is now in geosphere-mapserver and
geosphere-mapcache repositories
---
mapcache/Dockerfile | 75 -------
mapcache/README.md | 41 ----
mapcache/custom-conf | 1 -
mapcache/html/index.html | 25 ---
mapcache/mapcache_template.xml | 281 ------------------------
mapcache/render.py | 130 -----------
mapcache/render_config.sh | 29 ---
mapcache/run.sh | 8 -
mapcache/site-conf | 44 ----
mapcache/sql/goesr_crs.sql | 2 -
mapcache_timedb/Dockerfile | 19 --
mapcache_timedb/README.md | 39 ----
mapcache_timedb/time_updater.py | 71 ------
mapcache_timedb/updater.sh | 207 -----------------
mapserver/Dockerfile | 108 ---------
mapserver/README.md | 44 ----
mapserver/abi_l1b_template.map | 57 -----
mapserver/cgi-bin/layer_times.py | 18 --
mapserver/html/index.html | 21 --
mapserver/mapfiles/basemap.map | 130 -----------
mapserver/mapfiles/borders.map | 145 ------------
mapserver/mapfiles/generic_metadata.map | 3 -
mapserver/mapfiles/graticules.map | 159 --------------
mapserver/mapfiles/output_formats.map | 87 --------
mapserver/render.py | 82 -------
mapserver/run.sh | 14 --
mapserver/site-conf | 56 -----
mapserver/sql/goesr_crs.sql | 2 -
mapserver/tiledb_geospatial_Dockerfile | 159 --------------
tile_gen/Dockerfile | 15 --
tile_gen/README.md | 31 ---
tile_gen/generate_tiles.py | 131 -----------
tile_gen/run.sh | 91 --------
tile_gen/tile_index.py | 177 ---------------
34 files changed, 2502 deletions(-)
delete mode 100644 mapcache/Dockerfile
delete mode 100644 mapcache/README.md
delete mode 100644 mapcache/custom-conf
delete mode 100644 mapcache/html/index.html
delete mode 100644 mapcache/mapcache_template.xml
delete mode 100644 mapcache/render.py
delete mode 100755 mapcache/render_config.sh
delete mode 100755 mapcache/run.sh
delete mode 100644 mapcache/site-conf
delete mode 100644 mapcache/sql/goesr_crs.sql
delete mode 100644 mapcache_timedb/Dockerfile
delete mode 100644 mapcache_timedb/README.md
delete mode 100644 mapcache_timedb/time_updater.py
delete mode 100755 mapcache_timedb/updater.sh
delete mode 100644 mapserver/Dockerfile
delete mode 100644 mapserver/README.md
delete mode 100644 mapserver/abi_l1b_template.map
delete mode 100755 mapserver/cgi-bin/layer_times.py
delete mode 100644 mapserver/html/index.html
delete mode 100644 mapserver/mapfiles/basemap.map
delete mode 100644 mapserver/mapfiles/borders.map
delete mode 100644 mapserver/mapfiles/generic_metadata.map
delete mode 100644 mapserver/mapfiles/graticules.map
delete mode 100644 mapserver/mapfiles/output_formats.map
delete mode 100644 mapserver/render.py
delete mode 100755 mapserver/run.sh
delete mode 100644 mapserver/site-conf
delete mode 100644 mapserver/sql/goesr_crs.sql
delete mode 100644 mapserver/tiledb_geospatial_Dockerfile
delete mode 100644 tile_gen/Dockerfile
delete mode 100644 tile_gen/README.md
delete mode 100644 tile_gen/generate_tiles.py
delete mode 100755 tile_gen/run.sh
delete mode 100644 tile_gen/tile_index.py
diff --git a/mapcache/Dockerfile b/mapcache/Dockerfile
deleted file mode 100644
index 22fa86a..0000000
--- a/mapcache/Dockerfile
+++ /dev/null
@@ -1,75 +0,0 @@
-FROM tiledb/tiledb-geospatial:latest
-
-WORKDIR /work
-
-# FIXME: Remove once added to parent image
-RUN pip3 install shapely
-
-RUN apt-get -y update && \
- apt-get -y upgrade && \
- apt-get -y install gcc g++ cmake sqlite3 libsqlite3-dev pkg-config bash-completion curl subversion && \
- apt-get -y clean
-
-# apache
-# http://www.inanzzz.com/index.php/post/rhsb/running-apache-server-as-foreground-on-ubuntu-with-dockerfile
-# add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added
-ENV APACHE_RUN_USER www-data
-ENV APACHE_RUN_GROUP www-data
-ENV APACHE_LOCK_DIR /var/lock/apache2
-ENV APACHE_LOG_DIR /var/log/apache2
-ENV APACHE_PID_FILE /var/run/apache2/apache2.pid
-ENV APACHE_SERVER_NAME localhost
-#RUN groupadd -r www-data && useradd -r --create-home -g www-data www-data
-
-# install httpd runtime dependencies
-# https://httpd.apache.org/docs/2.4/install.html#requirements
-RUN mkdir -p /build_deps && cd /build_deps && apt-get -y update && \
- apt-get -y install apache2 apache2-dev && \
- a2enmod actions setenvif rewrite headers && \
- apt-get -y clean && \
- rm -rf /var/lib/apt/lists/*
-
-# Install MapCache (unstable)
-RUN mkdir -p /build_deps && cd /build_deps \
- && apt-get -y update \
-# && apt-get install -y libprotobuf17 zlib1g-dev libpng-dev libjpeg-turbo8 libfreetype6 libfribidi-dev libharfbuzz-dev libcairo2-dev libfcgi-dev libgeos++-dev postgresql postgis libxml2-dev libgif-dev libjpeg-turbo8-dev libprotobuf-dev protobuf-compiler libprotobuf-c-dev libprotobuf-c1 libprotobuf-dev protobuf-c-compiler \
- && apt-get install -y zlib1g-dev libpng-dev libjpeg-turbo8 libfreetype6 libfribidi-dev libharfbuzz-dev libcairo2-dev libfcgi-dev libgeos++-dev postgresql postgis libxml2-dev libgif-dev libjpeg-turbo8-dev libprotobuf-dev protobuf-compiler libprotobuf-c-dev libprotobuf-c1 libprotobuf-dev protobuf-c-compiler \
- && apt-get -y clean \
- && git clone https://github.com/mapserver/mapcache.git && cd mapcache \
- && git checkout c467a3bb444cab69c52bc78b91b9bae3a9415f2f \
- && mkdir -p build && cd build \
- && cmake .. \
-# && cmake -DCMAKE_BUILD_TYPE="Debug" .. \
- && make -j$(nproc) \
- && make install
-
-COPY site-conf /etc/apache2/sites-available/cspp_geo.conf
-# disable the default which would conflict with our custom
-RUN a2ensite cspp_geo && a2dissite 000-default
-
-COPY custom-conf /etc/apache2/conf-available/cspp_geo.conf
-# disable the default which would conflict with our custom
-RUN a2enconf cspp_geo
-
-RUN pip3 install jinja2
-COPY render.py /work/
-COPY mapcache_template.xml /work/
-COPY run.sh render_config.sh /work/
-COPY html/ /var/www/html/
-ENV PATH /work:$PATH
-
-# Add our own custom EPSG codes (HACK)
-# GOES-16 ABI Full Disk = EPSG:930916
-# GOES-17 ABI Full Disk = EPSG:930917
-COPY sql/ /work/sql/
-RUN sqlite3 -init /work/sql/goesr_crs.sql /usr/local/share/proj/proj.db
-
-# Check the config before we finish
-# mapcache.xml is now a template until the container starts
-#RUN apache2ctl configtest
-
-# https://httpd.apache.org/docs/2.4/stopping.html#gracefulstop
-STOPSIGNAL WINCH
-
-EXPOSE 80
-#CMD ["/usr/sbin/apache2ctl", "-DFOREGROUND"]
diff --git a/mapcache/README.md b/mapcache/README.md
deleted file mode 100644
index 07d930a..0000000
--- a/mapcache/README.md
+++ /dev/null
@@ -1,41 +0,0 @@
-# MapServer
-
-## Build
-
-```bash
-docker build -t gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapcache:latest mapcache/
-```
-
-## Usage
-
-```bash
-docker run -p 8889:80 -d --rm --name cspp-geo-mapcache1 --network cspp-geo-rabbit -v cspp-geo-tile-cache:/data gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapcache:latest ./run.sh cspp-geo-mapserver 80
-```
-
-Note the host name and port of the MapServer passed to the `run.sh` at the end.
-The the main mapcache CGI script can be accessed with:
-
-```bash
-http://localhost:8889/mapcache/wmts
-```
-
-An example usage of this including some WMS/WMTS URL parameters:
-
-```bash
-http://localhost:8889/mapcache/wmts?TIME=2020-01-01T12:00:00Z&SERVICE=WMTS&REQUEST=GetTile&VERSION=1.0.0&LAYER=g16_abi_radf_l1b_true_color&STYLE=default&TILEMATRIXSET=grid_g16_abi_radf&TILEMATRIX=1&TILEROW=1&TILECOL=1&FORMAT=image/png
-```
-
-
-## Special Notes
-
-This image has had its installation of the PROJ library modified to include
-new non-standard EPSG codes to support the projections of certain satellite
-instruments. These additions are part of the Dockerfile build process and
-include the following definitions:
-
-1. EPSG:930916 - The GOES-16 ABI Fixed Grid geostationary coordinate reference
- system. The numbers are meant to represent "GEOG16" while also attempting
- to avoid conflicts with future additions to the official EPSG database.
-2. EPSG:930917 - The GOES-17 ABI Fixed Grid geostationary coordinate reference
- system. The numbers are meant to represent "GEOG17" while also attempting
- to avoid conflicts with future additions to the official EPSG database.
diff --git a/mapcache/custom-conf b/mapcache/custom-conf
deleted file mode 100644
index a922059..0000000
--- a/mapcache/custom-conf
+++ /dev/null
@@ -1 +0,0 @@
-LoadModule mapcache_module /usr/lib/apache2/modules/mod_mapcache.so
diff --git a/mapcache/html/index.html b/mapcache/html/index.html
deleted file mode 100644
index a6b66a0..0000000
--- a/mapcache/html/index.html
+++ /dev/null
@@ -1,25 +0,0 @@
-<html>
-<head></head>
-<body>
-<h1>CSPP Geo Geo2Grid Tile Cache Server</h1>
-
-This server acts as a tile cache for the CSPP Geo Geo2Grid project.
-<br/>
-<br/>
-To access tiles using a standard WMTS protocol:
-<br/>
-<br/>
-http://presto.ssec.wisc.edu:8889/mapcache/wmts/?TIME=2020-03-23T17%3A00%3A21Z&SERVICE=WMTS&REQUEST=GetTile&VERSION=1.0.0&LAYER=g16_abi_radf_l1b_true_color&STYLE=default&TILEMATRIXSET=grid_g16_abi_radf&TILEMATRIX=1&TILEROW=1&TILECOL=1&FORMAT=image%2Fpng
-<br/>
-<br/>
-Where "radf" corresponds to Full Disk and "..." represents all remaining WMS
-parameters. Other sectors available are "radm01", "radm02", and "radc".
-<br>
-Note the full URL is typically not needed by WMTS clients as they will fill in
-the blanks or defaults. You can see an example "demo" client provided by
-MapCache here: http://presto.ssec.wisc.edu:8889/mapcache/demo/wmts
-<br/>
-<br/>
-
-</body>
-</html>
\ No newline at end of file
diff --git a/mapcache/mapcache_template.xml b/mapcache/mapcache_template.xml
deleted file mode 100644
index 88f3f16..0000000
--- a/mapcache/mapcache_template.xml
+++ /dev/null
@@ -1,281 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<mapcache>
- <metadata>
- <title>CSPP Geo Web Viewer Tile Cache</title>
- <abstract>This is a tile cache instance for the CSPP Geo Geo2Grid Tile Server</abstract>
- <!-- For GetCapabilities proxying -->
-<!-- <url>http://presto:8888/wms</url>-->
- </metadata>
-
- {% for inst in instruments %}
- {% for plat in instruments[inst] %}
- <!-- Overlay resolutions-->
- <grid name="grid_{{ plat }}_{{ inst }}_radf_overlays">
- <metadata>
- <title>{{ PLATFORM_POS_NAMES[plat] }} ABI Full Disk Grid</title>
- </metadata>
- <srs>{{ PLATFORM_EPSGS[plat] }}</srs>
- <size>256 256</size>
- <extent>-5434894.885 -5434894.885 5434894.885 5434894.885</extent>
- <units>m</units>
- <resolutions>{{ overlay_resolutions }}</resolutions>
- <origin>top-left</origin>
- </grid>
- {% for res in grid_resolutions[inst] %}
- <grid name="grid_{{ plat }}_{{ inst }}_radf_{{ grid_resolutions[inst][res][0] }}">
- <metadata>
- <title>{{ PLATFORM_POS_NAMES[plat] }} ABI Full Disk Grid at {{ grid_resolutions[inst][res][0] }}</title>
- </metadata>
- <srs>{{ PLATFORM_EPSGS[plat] }}</srs>
- <size>256 256</size>
- <extent>-5434894.885 -5434894.885 5434894.885 5434894.885</extent>
- <units>m</units>
- <resolutions>{{ grid_resolutions[inst][res][1] }}</resolutions>
- <origin>top-left</origin>
- </grid>
- {% endfor %}
- {% endfor %}
- {% endfor %}
-
- <cache name="disk" type="disk">
- <base>{{ cache_disk_base }}</base>
- <symlink_blank/>
- </cache>
-
- <format name="png_basic" type="PNG">
- </format>
-
- <!-- source
-
- the service to query for obtaining images if they are not in the cache
- for now, type can only be "wms"
- -->
- {% for inst in instruments %}
- {% for plat in instruments[inst] %}
- {% for sector in sectors %}
- {% for product in products %}
- <source name="{{ plat }}_{{ inst }}_{{ sector }}_l1b_{{ product }}" type="wms">
- <getmap>
- <params>
- <FORMAT>{{ IMAGE_FORMATS[product] }}</FORMAT>
- <LAYERS>{{ product }}</LAYERS>
- </params>
- </getmap>
- <http>
- <url>http://{{ wms_host }}:{{ wms_port }}/wms/{{ plat }}/abi/{{ sector }}/l1b</url>
- <connection_timeout>10</connection_timeout>
- <timeout>15</timeout>
- </http>
- </source>
- {% endfor %}
- {% endfor %}
-<!-- Basemap -->
- <source name="{{ plat }}_{{ inst }}_basemap" type="wms">
- <getmap>
- <params>
- <FORMAT>png256</FORMAT>
- <LAYERS>bw_map</LAYERS>
- </params>
- </getmap>
- <http>
- <url>http://{{ wms_host }}:{{ wms_port }}/wms/basemap</url>
- <connection_timeout>10</connection_timeout>
- <timeout>15</timeout>
- </http>
- </source>
- <!-- Overlays -->
- <source name="{{ plat }}_{{ inst }}_borders" type="wms">
- <getmap>
- <params>
- <FORMAT>rgba</FORMAT>
- <LAYERS>bw_map</LAYERS>
- </params>
- </getmap>
- <http>
- <url>http://{{ wms_host }}:{{ wms_port }}/wms/borders</url>
- <connection_timeout>10</connection_timeout>
- <timeout>15</timeout>
- </http>
- </source>
- <!-- Graticules -->
- <source name="{{ plat }}_{{ inst }}_graticules" type="wms">
- <getmap>
- <params>
- <FORMAT>rgba</FORMAT>
- <LAYERS>latlongrid</LAYERS>
- </params>
- </getmap>
- <http>
- <url>http://{{ wms_host }}:{{ wms_port }}/wms/graticules</url>
- <connection_timeout>10</connection_timeout>
- <timeout>15</timeout>
- </http>
- </source>
- {% endfor %}
- {% endfor %}
-
- <!-- tileset
- a tileset regroups a <source>, <grid> and <cache>.
- the "name" parameter is mandatory, and represents the key by which this tileset
- will be requested by clients (e.g. in the WMS LAYERS parameter)
- -->
-<!-- TODO: Make restricted_extents for lower resolutions channels and for CONUS and Meso sectors...maybe not mesos -->
-<!-- TODO: Might be able to make a tileset for each resolution/sector combination. No I can't because the mapserver would return a single image for all layers specified -->
- {% for inst in instruments %}
- {% for plat in instruments[inst] %}
- {% for sector in sectors %}
- {% for product in products %}
- <tileset name="{{ plat }}_{{ inst }}_{{ sector }}_l1b_{{ product }}">
- <source>{{ plat }}_{{ inst }}_{{ sector }}_l1b_{{ product }}</source>
- <cache>disk</cache>
-
- <!-- grid: the "name" attribute of a preconfigured <grid>
- you can also use the following notation to limit the area that will be cached and served to clients:
- <grid restricted_extent="-10 40 10 50">WGS84</grid>
- this way is better than using a grid with a limited extent, as in this way the tiles that are already
- cached are not invalidated should you want to modify the restricted extent in the future. When using
- the restricted_extent attribute, you should give the corresponding information to the client that will
- be using the service.
- you can also limit the zoom levels that are cached/accessible by using the minzoom, maxzoom attributes.
-
-
- NOTE: when adding a <grid> element, you *MUST* make sure that the source you have selected is able to
- return images in the grid's srs.
- -->
-<!-- <grid restricted_extent="-10 40 10 50" minzoom="6" maxzoom="8">WGS84</grid>-->
- <grid>grid_{{ plat }}_{{ inst }}_radf_{{ channel_resolutions[inst][product][0] }}</grid>
-
- <!-- metadata
- optional metadata tags used for responding to GetCapabilities request.
- you can put anything in here, although only the title, abstract and
- keywords tags are currently used to populate the GetCapabilities
- document.
- -->
- <metadata>
- <title>CSPP Geo Geo2Grid Web Viewer - {{ PLATFORM_LONG_NAMES[plat] }} ABI</title>
- <abstract>{{ PLATFORM_LONG_NAMES[plat] }} ABI image tiles generated by CSPP Geo Geo2Grid</abstract>
- <keywords>
- <keyword>goesr</keyword>
- <keyword>abi</keyword>
- <keyword>cspp</keyword>
- <keyword>geo2grid</keyword>
- <keyword>ssec</keyword>
- </keywords>
- </metadata>
- <format>png_basic</format>
- <auto_expire>{{ image_autoexpire }}</auto_expire>
-
- <dimensions>
- <dimension type="time" name="time" default="DEFAULT">
-<!-- TODO: https://github.com/mapserver/mapcache/wiki/MapCache-Performance-Analysis-A-Case-Study#requesting-second-level-dimension-entries-at-tile-or-at-map-level-->
- <dbfile>/data/mapcache_time.sqlite</dbfile>
- <validate_query>
- SELECT strftime("%Y-%m-%dT%H:%M:%SZ",ts) FROM {{ plat }}_{{ inst }}_{{ sector }}_l1b_{{ product }}
- WHERE ts >= datetime(:start_timestamp,"unixepoch")
- AND ts <= datetime(:end_timestamp,"unixepoch")
- ORDER BY ts DESC
- </validate_query>
- <list_query>SELECT strftime("%Y-%m-%dT%H:%M:%SZ", ts) FROM {{ plat }}_{{ inst }}_{{ sector }}_l1b_{{ product }}</list_query>
- </dimension>
- <!-- coming in a future version: support for ISO8601 date/time dimensions -->
- </dimensions>
- </tileset>
- {% endfor %}
- {% endfor %}
-
- <tileset name="{{ plat }}_{{ inst }}_basemap">
- <source>{{ plat }}_{{ inst }}_basemap</source>
- <cache>disk</cache>
- <grid>grid_{{ plat }}_{{ inst }}_radf_overlays</grid>
- <metadata>
- <title>CSPP Geo Geo2Grid Web Viewer - {{ PLATFORM_LONG_NAMES[plat] }} ABI</title>
- <abstract>{{ PLATFORM_LONG_NAMES[plat] }} ABI Basemap tiles</abstract>
- <keywords>
- <keyword>goesr</keyword>
- <keyword>abi</keyword>
- <keyword>cspp</keyword>
- <keyword>geo2grid</keyword>
- <keyword>ssec</keyword>
- </keywords>
- </metadata>
- <format>png_basic</format>
- <auto_expire>{{ basemap_autoexpire }}</auto_expire>
- </tileset>
- <tileset name="{{ plat }}_{{ inst }}_borders">
- <source>{{ plat }}_{{ inst }}_borders</source>
- <cache>disk</cache>
- <grid>grid_{{ plat }}_{{ inst }}_radf_overlays</grid>
- <metadata>
- <title>CSPP Geo Geo2Grid Web Viewer - {{ PLATFORM_LONG_NAMES[plat] }} ABI</title>
- <abstract>{{ PLATFORM_LONG_NAMES[plat] }} ABI Borders tiles</abstract>
- <keywords>
- <keyword>goesr</keyword>
- <keyword>abi</keyword>
- <keyword>cspp</keyword>
- <keyword>geo2grid</keyword>
- <keyword>ssec</keyword>
- </keywords>
- </metadata>
- <format>png_basic</format>
- {% if basemap_autoexpire != 0 %}
- <auto_expire>{{ basemap_autoexpire }}</auto_expire>
- {% endif %}
- </tileset>
- <tileset name="{{ plat }}_{{ inst }}_graticules">
- <source>{{ plat }}_{{ inst }}_graticules</source>
- <cache>disk</cache>
- <grid>grid_{{ plat }}_{{ inst }}_radf_overlays</grid>
- <metadata>
- <title>CSPP Geo Geo2Grid Web Viewer - {{ PLATFORM_LONG_NAMES[plat] }} ABI</title>
- <abstract>{{ PLATFORM_LONG_NAMES[plat] }} ABI Graticules tiles</abstract>
- <keywords>
- <keyword>goesr</keyword>
- <keyword>abi</keyword>
- <keyword>cspp</keyword>
- <keyword>geo2grid</keyword>
- <keyword>ssec</keyword>
- </keywords>
- </metadata>
- <format>png_basic</format>
- <auto_expire>{{ basemap_autoexpire }}</auto_expire>
- </tileset>
- {% endfor %}
- {% endfor %}
-
-
- <default_format>png_basic</default_format>
-
- <service type="wmts" enabled="true"/>
- <service type="demo" enabled="true"/>
-
- <!-- errors
- configure how error will be reported back to a client:
- - log : no error is reported back, except an http error code.
- - report : return the error message to the client in textual format
- - empty_img : return an empty image to the client. the actual error code is in the X-Mapcache-Error http header
- - report_img : return an image with the error text included inside. not implemented yet.
-
- the default setting is to report the error message back to the user. In production, you might want to set this to "log"
- if you're paranoid, or to "empty_img" if you want to play nice with non-conforming clients.
- -->
- <errors>report</errors>
-
-
-
- <locker type="disk"> <!-- this is the default -->
- <directory>{{ lock_disk_base }}</directory>
- <retry>0.01</retry> <!-- check back every .01 seconds -->
- </locker>
- <threaded_fetching>true</threaded_fetching>
- <connection_pool>
- <max_connections>1024</max_connections>
- <time_to_live_us>1000000</time_to_live_us>
- </connection_pool>
-
-
- <!-- fastcgi only -->
- <log_level>debug</log_level> <!-- logging verbosity -->
-<!-- <log_level>info</log_level> <!– logging verbosity –>-->
- <auto_reload>true</auto_reload> <!-- auto reload if config file changed -->
-
-</mapcache>
diff --git a/mapcache/render.py b/mapcache/render.py
deleted file mode 100644
index cd4ab46..0000000
--- a/mapcache/render.py
+++ /dev/null
@@ -1,130 +0,0 @@
-#!/usr/bin/env python3
-"""Convert a template XML mapcache config to a full config.
-
-We do this rendering when the container is started to allow for complete
-control per cache instance.
-
-"""
-
-import os
-import sys
-from collections import defaultdict
-import jinja2
-
-
-sectors = "radm1 radm2 radc radf"
-sectors = os.getenv("MAPCACHE_SECTORS", sectors).split(" ")
-products = ["C{:02d}".format(x) for x in range(1, 17)] + ["true_color"]
-products = os.getenv("MAPCACHE_PRODUCTS", " ".join(products)).split(" ")
-platforms = "g16 g17"
-platforms = os.getenv("MAPCACHE_PLATFORMS", platforms).split(" ")
-# 86400 = 24 hours
-# this does not control when tiles are deleted from the cache, but rather
-# when they should be re-requested. A value of 0 means never re-request.
-# A non-zero value can be a good safety net to just make sure things are updated.
-image_autoexpire = os.getenv("IMAGE_AUTOEXPIRE", str(86400 * 7))
-basemap_autoexpire = os.getenv("BASEMAP_AUTOEXPIRE", "0")
-
-wms_host = os.environ['WMS_HOST']
-wms_port = os.environ['WMS_PORT']
-cache_disk_base = os.environ['CACHE_DISK_BASE']
-lock_disk_base = os.getenv('LOCK_DISK_BASE', '/tmp')
-
-
-PLATFORM_LONG_NAMES = {
- 'g16': 'GOES-16',
- 'g17': 'GOES-17',
-}
-
-PLATFORM_EPSGS = {
- 'g16': 'EPSG:930916',
- 'g17': 'EPSG:930917',
-}
-
-PLATFORM_POS_NAMES = {
- 'g16': 'GOES-EAST',
- 'g17': 'GOES-WEST',
-}
-
-# instrument to platforms
-INSTRUMENTS = {
- 'abi': platforms,
-}
-
-# Full image at the base level -> divided by 2 gets us 7 zoom levels
-ABI_RESOLUTIONS_500 = "42460.1163 16032.138304 8016.069152 4008.034576 2004.017288 1002.008644 501.004322".split(" ")
-ABI_RESOLUTIONS_1000 = " ".join(ABI_RESOLUTIONS_500[:-1])
-ABI_RESOLUTIONS_2000 = " ".join(ABI_RESOLUTIONS_500[:-2])
-ABI_RESOLUTIONS_500 = " ".join(ABI_RESOLUTIONS_500)
-ABI_GRID_RESOLUTIONS = {
- "500m": ("500m", ABI_RESOLUTIONS_500),
- "1000m": ("1000m", ABI_RESOLUTIONS_1000),
- "2000m": ("2000m", ABI_RESOLUTIONS_2000),
-}
-ABI_CHANNEL_RESOLUTIONS = defaultdict(lambda: ABI_GRID_RESOLUTIONS['500m']) # default to 500m
-for chan in ["C04"] + [f"C{x:02d}" for x in range(6, 17)]:
- ABI_CHANNEL_RESOLUTIONS[chan] = ABI_GRID_RESOLUTIONS['2000m']
-for chan in ["C01", "C03", "C05"]:
- ABI_CHANNEL_RESOLUTIONS[chan] = ABI_GRID_RESOLUTIONS['1000m']
-for chan in ["true_color", "C02"]:
- ABI_CHANNEL_RESOLUTIONS[chan] = ABI_GRID_RESOLUTIONS['500m']
-
-CHANNEL_RESOLUTIONS = {
- 'abi': ABI_CHANNEL_RESOLUTIONS,
-}
-
-GRID_RESOLUTIONS = {
- 'abi': ABI_GRID_RESOLUTIONS,
-}
-
-OVERLAY_RESOLUTIONS = "16032.138304 8016.069152 2004.017288 501.004322"
-
-# assume images are rgb if not specified
-# these formats come from `output_formats.map` on the WMS server
-IMAGE_FORMATS = defaultdict(lambda: 'rgba')
-# The current 'gray' format is actually L + colortable which mapcache just
-# maps to an RGBA image anyway. Commenting out for now.
-# for i in range(1, 17):
-# IMAGE_FORMATS['C{:02d}'.format(i)] = 'gray'
-
-
-def main():
- import argparse
- parser = argparse.ArgumentParser(description="Render mapcache template to complete mapcache config.")
- parser.add_argument("template_fn",
- help="Template XML file to render")
- args = parser.parse_args()
-
- for rvar, rvar_name in ((sectors, 'MAPCACHE_SECTORS'),
- (products, 'MAPCACHE_PRODUCTS'),
- (platforms, 'MAPCACHE_PLATFORMS'),
- (cache_disk_base, 'CACHE_DISK_BASE')):
- if not rvar:
- raise ValueError(f"{rvar_name} was provided but is empty.")
-
- tmpl_vars = {
- 'sectors': sectors,
- 'products': products,
- 'instruments': INSTRUMENTS,
- 'grid_resolutions': GRID_RESOLUTIONS,
- 'channel_resolutions': CHANNEL_RESOLUTIONS,
- 'overlay_resolutions': OVERLAY_RESOLUTIONS,
- 'wms_host': wms_host,
- 'wms_port': wms_port,
- 'cache_disk_base': cache_disk_base,
- 'lock_disk_base': lock_disk_base,
- 'PLATFORM_LONG_NAMES': PLATFORM_LONG_NAMES,
- 'PLATFORM_EPSGS': PLATFORM_EPSGS,
- 'PLATFORM_POS_NAMES': PLATFORM_POS_NAMES,
- 'image_autoexpire': image_autoexpire,
- 'basemap_autoexpire': basemap_autoexpire,
- 'IMAGE_FORMATS': IMAGE_FORMATS,
- }
-
- with open(args.template_fn, 'r') as tmpl_file:
- template = jinja2.Template(tmpl_file.read())
- print(template.render(tmpl_vars))
-
-
-if __name__ == "__main__":
- sys.exit(main())
\ No newline at end of file
diff --git a/mapcache/render_config.sh b/mapcache/render_config.sh
deleted file mode 100755
index 9f4acc5..0000000
--- a/mapcache/render_config.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-
-if [[ $# -eq 2 ]]; then
- export WMS_HOST=$1
- export WMS_PORT=$2
-else
- echo "Usage: ./run.sh <wms_hostname> <wms_port>"
- exit 1
-fi
-
-# Load environment variable options to overwrite in the config
-export CACHE_DISK_BASE=${CACHE_DISK_BASE:-"/data/tile_cache"}
-
-# Update the mapcache.xml file with the real host and port name
-mc_tmpl="/work/mapcache_template.xml"
-mc_xml="/work/mapcache/mapcache.xml"
-mkdir -p $(dirname $mc_xml)
-python3 /work/render.py $mc_tmpl > $mc_xml
-if [[ $? -ne 0 ]]; then
- echo "Could not render mapcache template to config"
- exit 1
-fi
-
-# Create the tile directory as root
-if [[ ! -d ${CACHE_DISK_BASE} ]]; then
- mkdir -p ${CACHE_DISK_BASE} && chown www-data:www-data ${CACHE_DISK_BASE} \
- && chmod 775 ${CACHE_DISK_BASE}
-fi
-
diff --git a/mapcache/run.sh b/mapcache/run.sh
deleted file mode 100755
index baf557c..0000000
--- a/mapcache/run.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/bash -le
-
-# Render the template config to a full config
-render_config.sh "$@"
-# Make sure that the config is reasonable and print out any errors
-/usr/sbin/apache2ctl configtest
-# Start apache in the foreground
-/usr/sbin/apache2ctl -DFOREGROUND
diff --git a/mapcache/site-conf b/mapcache/site-conf
deleted file mode 100644
index 7d51ddd..0000000
--- a/mapcache/site-conf
+++ /dev/null
@@ -1,44 +0,0 @@
-<VirtualHost *:80>
- # The ServerName directive sets the request scheme, hostname and port that
- # the server uses to identify itself. This is used when creating
- # redirection URLs. In the context of virtual hosts, the ServerName
- # specifies what hostname must appear in the request's Host: header to
- # match this virtual host. For the default virtual host (this file) this
- # value is not decisive as it is used as a last resort host regardless.
- # However, you must set it for any further virtual host explicitly.
- #ServerName www.example.com
-
- ServerAdmin david.hoese@ssec.wisc.edu
- DocumentRoot /var/www/html
-
- # Available loglevels: trace8, ..., trace1, debug, info, notice, warn,
- # error, crit, alert, emerg.
- # It is also possible to configure the loglevel for particular
- # modules, e.g.
- #LogLevel info ssl:warn
- #LogLevel debug
-
- ErrorLog ${APACHE_LOG_DIR}/error.log
- CustomLog ${APACHE_LOG_DIR}/access.log combined
-
- # For most configuration files from conf-available/, which are
- # enabled or disabled at a global level, it is possible to
- # include a line for only one particular virtual host. For example the
- # following line enables the CGI configuration for this host only
- # after it has been globally disabled with "a2disconf".
- #Include conf-available/serve-cgi-bin.conf
-
- <IfModule mod_headers.c>
- Header set Access-Control-Allow-Origin "*"
- </IfModule>
- <IfModule mapcache_module>
- <Directory /work/mapcache>
- Require all granted
- </Directory>
- MapCacheAlias /mapcache "/work/mapcache/mapcache.xml"
- </IfModule>
- SetEnv "MAPCACHE_CONFIG_FILE" "/work/mapcache/mapcache.xml"
-
-</VirtualHost>
-
-# vim: syntax=apache ts=4 sw=4 sts=4 sr noet
\ No newline at end of file
diff --git a/mapcache/sql/goesr_crs.sql b/mapcache/sql/goesr_crs.sql
deleted file mode 100644
index 0cbf495..0000000
--- a/mapcache/sql/goesr_crs.sql
+++ /dev/null
@@ -1,2 +0,0 @@
-INSERT INTO projected_crs (auth_name, code, name, description, scope, coordinate_system_auth_name, coordinate_system_code, geodetic_crs_auth_name, geodetic_crs_code, conversion_auth_name, conversion_code, area_of_use_auth_name, area_of_use_code, text_definition, deprecated) VALUES ('EPSG', '930917', 'GOES-17 ABI Fixed Grid', 'GOES-17 (GOES-WEST) ABI Fixed Grid in the Geostationary projection', null, null, null, 'EPSG', '4269', null, null, null, null, 'PROJCRS["GOES-17 ABI Fixed Grid",BASEGEOGCRS["GOES-17 ABI Fixed Grid",DATUM["North American Datum 1983",ELLIPSOID["GRS 1980",6378137,298.257222101,LENGTHUNIT["metre",1]],ID["EPSG",6269]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8901]]],CONVERSION["unknown",METHOD["Geostationary Satellite (Sweep X)"],PARAMETER["Longitude of natural origin",-137,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8802]],PARAMETER["Satellite Height",35786023,LENGTHUNIT["metre",1,ID["EPSG",9001]]],PARAMETER["False easting",0,LENGTHUNIT["metre",1],ID["EPSG",8806]],PARAMETER["False northing",0,LENGTHUNIT["metre",1],ID["EPSG",8807]]],CS[Cartesian,2],AXIS["(E)",east,ORDER[1],LENGTHUNIT["metre",1,ID["EPSG",9001]]],AXIS["(N)",north,ORDER[2],LENGTHUNIT["metre",1,ID["EPSG",9001]]]]', 0);
-INSERT INTO projected_crs (auth_name, code, name, description, scope, coordinate_system_auth_name, coordinate_system_code, geodetic_crs_auth_name, geodetic_crs_code, conversion_auth_name, conversion_code, area_of_use_auth_name, area_of_use_code, text_definition, deprecated) VALUES ('EPSG', '930916', 'GOES-16 ABI Fixed Grid', 'GOES-16 (GOES-WEST) ABI Fixed Grid in the Geostationary projection', null, null, null, 'EPSG', '4269', null, null, null, null, 'PROJCRS["GOES-16 ABI Fixed Grid",BASEGEOGCRS["GOES-16 ABI Fixed Grid",DATUM["North American Datum 1983",ELLIPSOID["GRS 1980",6378137,298.257222101,LENGTHUNIT["metre",1]],ID["EPSG",6269]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8901]]],CONVERSION["unknown",METHOD["Geostationary Satellite (Sweep X)"],PARAMETER["Longitude of natural origin",-75,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8802]],PARAMETER["Satellite Height",35786023,LENGTHUNIT["metre",1,ID["EPSG",9001]]],PARAMETER["False easting",0,LENGTHUNIT["metre",1],ID["EPSG",8806]],PARAMETER["False northing",0,LENGTHUNIT["metre",1],ID["EPSG",8807]]],CS[Cartesian,2],AXIS["(E)",east,ORDER[1],LENGTHUNIT["metre",1,ID["EPSG",9001]]],AXIS["(N)",north,ORDER[2],LENGTHUNIT["metre",1,ID["EPSG",9001]]]]', 0);
\ No newline at end of file
diff --git a/mapcache_timedb/Dockerfile b/mapcache_timedb/Dockerfile
deleted file mode 100644
index 6da69ce..0000000
--- a/mapcache_timedb/Dockerfile
+++ /dev/null
@@ -1,19 +0,0 @@
-FROM gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapcache:latest
-
-ENV DEBIAN_FRONTEND=noninteractive
-ENV TZ=GMT
-ENV LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH}
-ENV LANG=C.UTF-8
-ENV LC_ALL=C.UTF-8
-
-WORKDIR /work
-
-# Install amqpfind
-RUN apt-get update && apt-get install -y unzip && \
- wget https://www.ssec.wisc.edu/~rayg/pub/amqpfind.zip && \
- unzip amqpfind.zip && \
- rm amqpfind.zip && \
- rm -rf /var/lib/apt/lists/*
-
-COPY time_updater.py /work
-COPY updater.sh /work
diff --git a/mapcache_timedb/README.md b/mapcache_timedb/README.md
deleted file mode 100644
index 876a90c..0000000
--- a/mapcache_timedb/README.md
+++ /dev/null
@@ -1,39 +0,0 @@
-# MapServer
-
-## Build
-
-```bash
-docker build -t gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapcache:latest mapcache_timedb/
-```
-
-## Usage
-
-```bash
-docker run -d --rm --name cspp-geo-mapcache-timedb --network cspp-geo-rabbit -v cspp-geo-tile-cache:/data gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapcache_timedb:latest ./updater.sh cspp-geo-mapserver 80
-```
-
-Note the host name and port of the MapServer passed to the `run.sh` at the end.
-The container will run, wait for RabbitMQ events for new tiles or removed
-tiles, and update a local SQLite database with these times. See the `run.sh`
-for more details.
-
-## Options
-
-**Environment Variables**
-
-* `AMQPFIND_ARGS`: Extra arguments to send to amqpfind.py
-* `AMQPFIND_TOPIC`: The RabbitMQ topic to listen to for amqpfind.py
-* `G2G_PRODUCTS`: What products to expect and process. Defaults to all channel
- products and the `true_color` RGB product.
-* `SATELLITES`: Satellite platforms to include in processing. Defaults to
- "g16 g17".
-* `MAPCACHE_INSTRUMENTS`: Instruments to process. Defaults to "abi".
-* `MAPCACHE_SECTORS`: Sectors to process. Defaults to "radf radc radm1 radm2".
-* `MAPCACHE_SEED`: Whether or not to pre-seed the cache when layers are added. Can be
- "0" for off (the default) or anything else for True.
-* `MAPCACHE_SEED_OVERLAYS`: Whether or not to pre-seed the basemap, borders, and
- graticules layers when this container starts. For an empty cache this can
- take a couple minutes per layer. Defaults to True, can be turned off by
- specifying as "0". This does not obey the `SEED_ARGS` environment variable
- (see below).
-* `MAPCACHE_SEED_ARGS`: Extra arguments to pass to `mapcache_seed`. Defaults to "".
diff --git a/mapcache_timedb/time_updater.py b/mapcache_timedb/time_updater.py
deleted file mode 100644
index b2ac582..0000000
--- a/mapcache_timedb/time_updater.py
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/usr/bin/env python3
-import os
-import sys
-import sqlite3
-import logging
-from datetime import datetime
-
-LOG = logging.getLogger(__name__)
-
-
-def main():
- """Command line main."""
- import argparse
- parser = argparse.ArgumentParser(description="""Add or remove time from SQLite database.""")
- parser.add_argument("-r", "--remove", action="store_true",
- help="Remove instead of add the provided times to the database.")
- parser.add_argument("--json", action="store_true",
- help="The provided times are in one JSON-formatted array.")
- parser.add_argument("dbfile",
- help="SQLite Database to create and/or modify.")
- parser.add_argument("table_name",
- help="Table name to update (ex. g16_abi_radf_true_color)")
- parser.add_argument("isotimes", nargs="*",
- help="Times to either remove or add (default) to the "
- "database. Times should be strings in the "
- "ISO8601 format (ex. YYYY-MM-DDTHH:MM:SS)")
- args = parser.parse_args()
- logging.basicConfig(level=logging.DEBUG)
-
- if args.json and len(args.isotimes) != 1:
- LOG.error("JSON formatted times must be provided as one array. Got "
- "multiple arguments.")
- return 1
- elif args.json:
- import json
- args.isotimes = json.loads(args.isotimes[0])
- dt_times = [datetime.strptime(x, "%Y-%m-%dT%H:%M:%S") for x in args.isotimes]
- exists = os.path.isfile(args.dbfile)
- table_exists = True
- conn = sqlite3.connect(args.dbfile)
- c = conn.cursor()
- if exists:
- # check if the table exists
- LOG.info("Checking if table exists...")
- res = c.execute("""SELECT name FROM sqlite_master WHERE type='table' AND name='{}'""".format(args.table_name))
- table_exists = bool(res.fetchall())
- if not exists or not table_exists:
- # Database doesn't exist yet, let's create it
- LOG.info("Initializing database table...")
- c.execute("""CREATE TABLE {} (ts text unique)""".format(args.table_name))
- conn.commit()
-
- if not dt_times:
- LOG.info("Empty time list provided, won't open database.")
- conn.close()
- return
-
- if args.remove:
- # Remove the provided times
- LOG.info("Removing {} times.".format(len(dt_times)))
- c.executemany('DELETE FROM {} WHERE ts == ?'.format(args.table_name), [(x,) for x in dt_times])
- else:
- # Add the provided times
- LOG.info("Adding {} times.".format(len(dt_times)))
- c.executemany('INSERT OR IGNORE INTO {} VALUES (?)'.format(args.table_name), [(x,) for x in dt_times])
- conn.commit()
- conn.close()
-
-
-if __name__ == "__main__":
- sys.exit(main())
diff --git a/mapcache_timedb/updater.sh b/mapcache_timedb/updater.sh
deleted file mode 100755
index f05d688..0000000
--- a/mapcache_timedb/updater.sh
+++ /dev/null
@@ -1,207 +0,0 @@
-#!/bin/bash -l
-# Usage: run.sh
-# Description: Wait for RabbitMQ messages and add available times to various
-# SQLite databases to be used by MapCache.
-# Environment variables used for configuration:
-# AMQPFIND_ARGS: Arguments to pass to amqpfind when listening for new input
-# events. Should not include the "-C" topic flag (see AMQPFIND_TOPIC).
-# Default: "-H cspp-geo-rabbit -X satellite -u guest -p guest"
-# AMQPFIND_TOPIC: Topic to use for incoming data events.
-# Default: "data.goes.*.abi.*.l1b.tiles.*.complete"
-# The first asterisk (3rd element) can limit processing to a particular
-# satellite (ex. `g16`). The second asterisk (5th element) can be used
-# to limit to a particular sector (choices: radf, radc, radm1, radm2).
-# The last asterisk represents the name of the product (c01, true_color,
-# etc).
-
-# Verify that the data mount is available
-test -d "/data"
-
-export AMQPFIND_ARGS=${AMQPFIND_ARGS:-"-H cspp-geo-rabbit -X satellite -u guest -p guest"}
-export AMQPFIND_TOPIC=${AMQPFIND_TOPIC:-'data.goes.*.abi.*.l1b.tiles.*.complete'}
-export G2G_PRODUCTS=${G2G_PRODUCTS:-"C01 C02 C03 C04 C05 C06 C07 C08 C09 C10 C11 C12 C13 C14 C15 C16 true_color"}
-export MAPCACHE_PRODUCTS=${MAPCACHE_PRODUCTS:-$G2G_PRODUCTS}
-export MAPCACHE_PLATFORMS=${MAPCACHE_PLATFORMS:-"g16 g17"}
-export MAPCACHE_INSTRUMENTS=${MAPCACHE_INSTRUMENTS:-"abi"}
-export MAPCACHE_SECTORS=${MAPCACHE_SECTORS:-"radf radc radm1 radm2"}
-export MAPCACHE_SEED=${MAPCACHE_SEED:-"0"} # 0 (False) or anything else (True)
-export MAPCACHE_SEED_OVERLAYS=${MAPCACHE_SEED_OVERLAYS:-"1"} # 0 (False) or anything else (True)
-export MAPCACHE_SEED_ARGS=${MAPCACHE_SEED_ARGS:-""}
-
-json_list_to_bash() {
- blist=$1
- blist=${blist//,/}
- blist=${blist//[/}
- blist=${blist//]/}
- echo "$blist"
-}
-export -f json_list_to_bash
-
-run_mapcache_seed() {
- CONFIG="/work/mapcache/mapcache.xml"
- NPROCS="4"
- TILESET=$1 # g16_abi_radf_l1b_true_color
- GRID=$2 # grid_g16_abi_radf_500m
- # optional (borders/overlays don't have a time dimension)
- T=$3 # YYYY-MM-DDTHH:MM:SS (Z will be added, assumes UTC)
- if [[ $T == "" ]]; then
- TIME=""
- else
- TIME="-D TIME=${T}Z" # Add Z timezone
- fi
- set -x
- su -l www-data -s /bin/bash -c "export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH; time mapcache_seed -c $CONFIG -t $TILESET -n $NPROCS -g $GRID $TIME $MAPCACHE_SEED_ARGS"
- status=$?
- set +x
- exit $status
-}
-export -f run_mapcache_seed
-
-run_time_updater() {
- if [[ $# -ne 7 ]]; then
- echo "Unexpected number of arguments (expected 7): $#"
- return 1
- fi
-
- satellite_family=${1,,}
- satellite_id=${2,,}
- instrument=${3,,}
- data_type=${4,,}
- product=${5}
- removed_times="$6"
- added_times="$7"
-
- # xargs strips quotes so lets just convert it to a normal array
- removed_times=$(json_list_to_bash "$removed_times")
- added_times=$(json_list_to_bash "$added_times")
-
- echo "Starting cache time updates for ${path}"
-
- # if /dst isn't defined then use /data
- if [[ -d "/dst" ]]; then
- dst_dir="/dst"
- else
- dst_dir="/data"
- fi
-
- dbfile="${dst_dir}/mapcache_time.sqlite"
- table_name="${satellite_id}_${instrument}_${data_type}_l1b_${product}"
- echo "Removing times for: ${dbfile}"
- echo "Removed times: ${removed_times}"
- python3 /work/time_updater.py -r "${dbfile}" "${table_name}" ${removed_times}
- echo "Adding times for: ${dbfile}"
- echo "Added times: ${added_times}"
- python3 /work/time_updater.py "${dbfile}" "${table_name}" ${added_times}
- echo "Done updating times for: ${dbfile}"
-
- if [[ ${MAPCACHE_SEED} -ne 0 ]]; then
- tileset="${satellite_id}_${instrument}_${data_type}_l1b_${product}"
- grid="grid_${satellite_id}_${instrument}_radf"
- case $product in
- C02 | true_color)
- grid="${grid}_500m"
- ;;
- C0[135])
- grid="${grid}_1000m"
- ;;
- C*)
- grid="${grid}_2000m"
- ;;
- *)
- grid="${grid}_500m"
- esac
- for t in ${added_times}; do
- echo "Seeding cache for ${tileset} @ ${t}"
- run_mapcache_seed $tileset $grid $t
- echo "Done seeding"
- done
- fi
-}
-export -f run_time_updater
-
-init_time_updater() {
- # if /dst isn't defined then use /data
- if [[ -d "/dst" ]]; then
- dst_dir="/dst"
- else
- dst_dir="/data"
- fi
-
- # Create an empty Database for every tileset we might process
- for sat_id in ${MAPCACHE_PLATFORMS}; do
- for inst in ${MAPCACHE_INSTRUMENTS}; do
- for sector in ${MAPCACHE_SECTORS}; do
- dbfile="${dst_dir}/mapcache_time.sqlite"
- for product in ${G2G_PRODUCTS}; do
- echo "Initializing database ${dbfile} for product ${product}"
- set -x
- current_times=$(wget -O - "http://${WMS_HOST}:${WMS_PORT}/wms_times/${sat_id}/${inst}/${sector}/${product}")
- table_name="${sat_id}_${inst}_${sector}_l1b_${product}"
- python3 /work/time_updater.py --json "${dbfile}" "${table_name}" "${current_times}"
- set +x
- done
- done
- done
- done
-}
-
-
-preseed_overlays() {
- # if /dst isn't defined then use /data
- if [[ -d "/dst" ]]; then
- dst_dir="/dst"
- else
- dst_dir="/data"
- fi
-
- # Create an empty Database for every tileset we might process
- for sat_id in ${MAPCACHE_PLATFORMS}; do
- for inst in ${MAPCACHE_INSTRUMENTS}; do
- # Preseed the overlays, graticules, and basemap layers
- if [[ $MAPCACHE_SEED_OVERLAYS -ne 0 ]]; then
- grid="grid_${sat_id}_${inst}_radf_overlays"
-
- echo "Seeding basemap layer"
- tileset="${sat_id}_${inst}_basemap"
- MAPCACHE_SEED_ARGS="" run_mapcache_seed $tileset $grid
- echo "Seeding borders layer"
- tileset="${sat_id}_${inst}_borders"
- MAPCACHE_SEED_ARGS="" run_mapcache_seed $tileset $grid
- echo "Seeding graticules layer"
- tileset="${sat_id}_${inst}_graticules"
- MAPCACHE_SEED_ARGS="" run_mapcache_seed $tileset $grid
- echo "Done seeding ${grid}'s overlay layers"
- fi
- done
- done
-}
-
-
-if [[ $# -eq 2 ]]; then
- WMS_HOST=$1
- WMS_PORT=$2
-else
- echo "Usage: ./run.sh <wms_hostname> <wms_port>"
- exit 1
-fi
-
-# Render the MapCache config so we can seed the cache later
-render_config.sh $WMS_HOST $WMS_PORT
-if [[ $? -ne 0 ]]; then
- echo "Could not render mapcache template to full config"
- exit 1
-fi
-
-# Initialize the database if it doesn't exist already
-echo "Initializing time database..."
-init_time_updater
-if [[ $? -ne 0 ]]; then
- echo "Could not initialize time databases. Exiting..."
- exit 1
-fi
-echo "Starting overlays preseed in the background"
-preseed_overlays &
-
-echo "Listening to AMQP messages with topic \"$AMQPFIND_TOPIC\""
-python3 amqpfind/amqpfind.py ${AMQPFIND_ARGS} -C "${AMQPFIND_TOPIC}" -j "{satellite_family} {satellite_ID} {instrument} {data_type} {product} \'{removed_times}\' \'{added_times}\'" | xargs -I{} -P4 -n1 bash -c "run_time_updater {}"
-
diff --git a/mapserver/Dockerfile b/mapserver/Dockerfile
deleted file mode 100644
index 7bb97b5..0000000
--- a/mapserver/Dockerfile
+++ /dev/null
@@ -1,108 +0,0 @@
-FROM tiledb/tiledb-geospatial:latest
-
-WORKDIR /work
-
-# FIXME: Remove once added to parent image
-RUN pip3 install shapely
-
-RUN apt-get -y update && \
- apt-get -y upgrade && \
- apt-get -y install gcc g++ cmake sqlite3 libsqlite3-dev pkg-config bash-completion curl subversion && \
- apt-get -y clean
-
-
-# FIXME: Remove if/when upstream image builds with fcgi
-# Install Mapserver (unstable)
-RUN mkdir -p /build_deps && cd /build_deps \
- && apt-get -y update \
-# && apt-get install -y libprotobuf17 zlib1g-dev libpng-dev libjpeg-turbo8 libfreetype6 libfribidi-dev libharfbuzz-dev libcairo2-dev libfcgi-dev libgeos++-dev postgresql postgis libxml2-dev libgif-dev libjpeg-turbo8-dev libprotobuf-dev protobuf-compiler libprotobuf-c-dev libprotobuf-c1 libprotobuf-dev protobuf-c-compiler \
- && apt-get install -y zlib1g-dev libpng-dev libjpeg-turbo8 libfreetype6 libfribidi-dev libharfbuzz-dev libcairo2-dev libfcgi-dev libgeos++-dev postgresql postgis libxml2-dev libgif-dev libjpeg-turbo8-dev libprotobuf-dev protobuf-compiler libprotobuf-c-dev libprotobuf-c1 libprotobuf-dev protobuf-c-compiler \
- && apt-get -y clean \
- && git clone https://github.com/mapserver/mapserver.git && cd mapserver \
- && git checkout 0fcc810f0b559c800f950db78a79fa6574799f23 \
- && mkdir -p build && cd build \
- && cmake .. -DWITH_PROTOBUFC=OFF -DWITH_POSTGIS=OFF \
- && make -j$(nproc) \
- && make install
-
-# apache
-# http://www.inanzzz.com/index.php/post/rhsb/running-apache-server-as-foreground-on-ubuntu-with-dockerfile
-# add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added
-ENV APACHE_RUN_USER www-data
-ENV APACHE_RUN_GROUP www-data
-ENV APACHE_LOCK_DIR /var/lock/apache2
-ENV APACHE_LOG_DIR /var/log/apache2
-ENV APACHE_PID_FILE /var/run/apache2/apache2.pid
-ENV APACHE_SERVER_NAME localhost
-#RUN groupadd -r www-data && useradd -r --create-home -g www-data www-data
-
-# install httpd runtime dependencies
-# https://httpd.apache.org/docs/2.4/install.html#requirements
-RUN cd /build_deps && apt-get -y install apache2 libapache2-mod-fcgid && \
- ls /etc/apache2/mods-available && \
- ls /etc/apache2/mods-enabled && \
- apt-get -y install libapache2-mod-php7.2 php7.2-common php7.2-cli php7.2-fpm php7.2 && \
- a2enmod actions proxy_fcgi setenvif cgi fcgid rewrite && \
- a2enconf php7.2-fpm serve-cgi-bin && \
- apt-get -y clean && \
- rm -rf /var/lib/apt/lists/*
-
-RUN apt-get -y update && apt-get install -y unzip \
- && curl -LOk https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_countries.zip \
- && curl -LOk https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_1_states_provinces_lines.zip \
- && curl -LOk https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_lakes.zip \
- && curl -LOk https://www2.census.gov/geo/tiger/GENZ2018/shp/cb_2018_us_county_500k.zip \
- && curl -LOk https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_graticules_1.zip \
- && curl -LOk https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_graticules_5.zip \
- && curl -LOk https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_graticules_10.zip \
- && mkdir -p /work/shapefiles \
- && unzip -d /work/shapefiles ne_10m_admin_0_countries.zip \
- && unzip -d /work/shapefiles ne_10m_admin_1_states_provinces_lines.zip \
- && unzip -d /work/shapefiles ne_10m_lakes.zip \
- && unzip -d /work/shapefiles cb_2018_us_county_500k.zip \
- && unzip -d /work/shapefiles ne_10m_graticules_1.zip \
- && unzip -d /work/shapefiles ne_10m_graticules_5.zip \
- && unzip -d /work/shapefiles ne_10m_graticules_10.zip \
- && chmod a+r /work/shapefiles/* \
- && rm -f ne_*.zip cb_*.zip \
- && apt-get -y clean \
- && rm -rf /var/lib/apt/lists/*
-
-# Install Fonts for mapserver labels
-RUN echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | debconf-set-selections \
- && mkdir -p /work/mapfiles \
- && apt-get -y update && apt-get install -y ttf-mscorefonts-installer \
- && bash -c 'for font_fn in `find /usr/share/fonts/truetype/ -name "*.ttf"`; do font_name=`basename $font_fn`; font_name=${font_name/.ttf/}; font_name=${font_name,,}; echo "$font_name $font_fn" >> /work/mapfiles/fonts.list; done' \
- && apt-get -y clean \
- && rm -rf /var/lib/apt/lists/*
-
-COPY site-conf /etc/apache2/sites-available/cspp_geo.conf
-# disable the default which would conflict with our custom
-RUN a2ensite cspp_geo && a2dissite 000-default
-COPY cgi-bin/* /usr/lib/cgi-bin/
-
-# Point apache to the mapserver binary
-RUN ln -s /usr/local/bin/mapserv /usr/lib/cgi-bin/mapserv && \
- ln -s /usr/local/bin/mapserv /usr/lib/cgi-bin/mapserv.fcgi && \
- chown ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/* && \
- chown -h ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/*
-
-RUN pip3 install jinja2
-COPY render.py run.sh abi_l1b_template.map /work/
-COPY mapfiles/ /work/mapfiles/
-COPY html/ /var/www/html/
-
-# Add our own custom EPSG codes (HACK)
-# GOES-16 ABI Full Disk = EPSG:930916
-# GOES-17 ABI Full Disk = EPSG:930917
-COPY sql/ /work/sql/
-RUN sqlite3 -init /work/sql/goesr_crs.sql /usr/local/share/proj/proj.db
-
-# Check the config before we finish
-RUN apache2ctl configtest
-
-# https://httpd.apache.org/docs/2.4/stopping.html#gracefulstop
-STOPSIGNAL WINCH
-
-EXPOSE 80
-CMD ["/work/run.sh"]
diff --git a/mapserver/README.md b/mapserver/README.md
deleted file mode 100644
index 93aa368..0000000
--- a/mapserver/README.md
+++ /dev/null
@@ -1,44 +0,0 @@
-# MapServer
-
-## Build
-
-```bash
-docker build -t gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapserver:latest mapserver/
-```
-
-## Usage
-
-```bash
-docker run -p 8888:80 -d --rm --name cspp-geo-mapserver --network cspp-geo-rabbit -v cspp-geo-abi-l1b-geotiffs:/data gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/mapserver:latest
-```
-
-Then the main mapserv CGI script can be accessed with:
-
-http://localhost:8888/cgi-bin/mapserv
-
-where MapServer parameters can be appended to the end (after a `?`). However,
-an easier to access API is available:
-
-```bash
-http://localhost:8888/wms/<satellite>/<instrument>/<sector>/<data_level>?...
-```
-
-An example URL of this including MapServer parameters is:
-
-```bash
-http://localhost:8888/wms/g16/abi/radf/l1b?request=GetMap&service=WMS&version=1.1.1&srs=EPSG:930916&layers=true_color_test&bbox=-5000000,-5000000,5000000,5000000&format=image/png&WIDTH=1000&HEIGHT=1000&TIME=2020-01-21T16:00:16
-```
-
-## Special Notes
-
-This image has had its installation of the PROJ library modified to include
-new non-standard EPSG codes to support the projections of certain satellite
-instruments. These additions are part of the Dockerfile build process and
-include the following definitions:
-
-1. EPSG:930916 - The GOES-16 ABI Fixed Grid geostationary coordinate reference
- system. The numbers are meant to represent "GEOG16" while also attempting
- to avoid conflicts with future additions to the official EPSG database.
-2. EPSG:930917 - The GOES-17 ABI Fixed Grid geostationary coordinate reference
- system. The numbers are meant to represent "GEOG17" while also attempting
- to avoid conflicts with future additions to the official EPSG database.
diff --git a/mapserver/abi_l1b_template.map b/mapserver/abi_l1b_template.map
deleted file mode 100644
index bab3262..0000000
--- a/mapserver/abi_l1b_template.map
+++ /dev/null
@@ -1,57 +0,0 @@
-MAP
- NAME "{{ platform_long_name }} ABI {{ sector_long_name }} L1b Layers"
- STATUS ON
- IMAGETYPE rgb
- SIZE 256 256
- EXTENT -5434894.885 -5434894.885 5434894.885 5434894.885
-
- PROJECTION
- "{{ projection }}"
- END
-
- WEB
- METADATA
- "wms_title" "CSPP Geo Geo2Grid Viewer"
- "wms_onlineresource" "https://myhost/mapserv"
- "wms_enable_request" "*"
- END
- END
-
- INCLUDE '/work/mapfiles/output_formats.map'
-
- {% for product in products %}
- # {{ product.replace('_', ' ').title() }} #
-
- LAYER
- NAME "{{ product }}_index"
- TYPE TILEINDEX
- DATA "{{ layer_base_dir }}/{{ platform }}/abi/{{ sector }}/{{ product }}/{{ product }}"
- END
-
- LAYER
- NAME "{{ product }}"
- TYPE RASTER
- TILEITEM "location"
- TILEINDEX "{{ product }}_index"
- # Comment below to default to transparency
- #OFFSITE 0 0 0
- PROJECTION
- "{{ projection }}"
- END
- PROCESSING "RESAMPLE=AVERAGE"
- # Used to keep FastCGI connections open longer
- PROCESSING "CLOSE_CONNECTION=DEFER"
- METADATA
- "wms_title" "{{ platform_long_name }} ABI {{ product.replace('_', ' ').title() }}"
- "wms_extent" "-180 -90 180 90"
- "wms_timeextent" "2017-01-01/2040-12-31"
- "wms_timeformat" "YYYY-MM-DDTHH:MM:SS"
- "wms_timeitem" "time" # time is a metadata item
- "wms_timedefault" "2019-12-12T19:20:18"
- "wms_enable_request" "*"
- END
- FILTER (`[time]` = `2019-12-12T19:20:18`)
- END # goes raster layer ends here
- {% endfor %}
-
-END # end of map file
diff --git a/mapserver/cgi-bin/layer_times.py b/mapserver/cgi-bin/layer_times.py
deleted file mode 100755
index 14c42f0..0000000
--- a/mapserver/cgi-bin/layer_times.py
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env python3
-import os
-import cgi
-import json
-import fiona
-
-form = cgi.FieldStorage()
-layer = form['layer'].value
-if not os.path.isfile(layer):
- times = []
-else:
- with fiona.open(layer, 'r') as shp_file:
- times = [x['properties']['time'] for x in shp_file]
-
-print("Content-Type: application/json")
-print("Access-Control-Allow-Origin: *")
-print() # blank line, end of headers
-print(json.dumps(times))
diff --git a/mapserver/html/index.html b/mapserver/html/index.html
deleted file mode 100644
index abecce6..0000000
--- a/mapserver/html/index.html
+++ /dev/null
@@ -1,21 +0,0 @@
-<html>
-<head></head>
-<body>
-<h1>CSPP Geo Geo2Grid Tile Server</h1>
-
-This tile server for the CSPP Geo Geo2Grid project.
-<br/>
-<br/>
-To access tiles using a standard WMS protocol:
-<br/>
-<br/>
-http://localhost/wms/goes16/abi/radf/l1b?...
-<br/>
-<br/>
-Where "radf" corresponds to Full Disk and "..." represents all remaining WMS
-parameters. Other sectors available are "radm01", "radm02", and "radc".
-<br/>
-<br/>
-
-</body>
-</html>
\ No newline at end of file
diff --git a/mapserver/mapfiles/basemap.map b/mapserver/mapfiles/basemap.map
deleted file mode 100644
index 958ba3a..0000000
--- a/mapserver/mapfiles/basemap.map
+++ /dev/null
@@ -1,130 +0,0 @@
-MAP
- NAME "Basemap"
- STATUS ON
- MAXSIZE 5000
- SIZE 1024 1024
- IMAGECOLOR 0 0 0
- IMAGETYPE png256
- EXTENT -5434894.885 -5434894.885 5434894.885 5434894.885
- FONTSET "/work/mapfiles/fonts.list"
-
- PROJECTION
- "init=epsg:930916"
- END
-
- WEB
- METADATA
- "wms_title" "BaseMap"
- INCLUDE "/work/mapfiles/generic_metadata.map"
- END
- END
-
- INCLUDE '/work/mapfiles/output_formats.map'
-
- LAYER
- NAME "m10_countries"
- DATA /work/shapefiles/ne_10m_admin_0_countries
- EXTENT -180 -89.9998 180 83.6338
- STATUS ON
- TYPE POLYGON
- PROJECTION
- "init=epsg:4326"
- END
- GROUP bw_map
- METADATA
- "wms_title" "m10_countries"
- END
- MINSCALEDENOM 1
- MAXSCALEDENOM 11000000000
- OFFSITE 128 128 128
- CLASS
- NAME '10m_countries'
- STYLE
- WIDTH 1
- OUTLINECOLOR 128 128 128
- COLOR 160 192 160
- END
- END
- END
-
- LAYER
- NAME "m10_states"
- DATA /work/shapefiles/ne_10m_admin_1_states_provinces_lines
- EXTENT -139.047 -49.2507 178.449 81.1382
- STATUS ON
- # don't show this for some countries to avoid conflicting lines between layers
- FILTER (("[adm0_name]" == "Australia") OR ("[adm0_name]" == "Canada") OR ("[adm0_name]" == "United States of America"))
- TYPE LINE
- PROJECTION
- "init=epsg:4326"
- END
- GROUP bw_map
- METADATA
- "wms_title" "m10_states"
- END
- MINSCALEDENOM 1
- MAXSCALEDENOM 7500000
- OFFSITE 128 128 128
- CLASS
- NAME '10m_states'
- STYLE
- WIDTH 1
- OUTLINECOLOR 128 128 128
- END
- END
- END
-
- LAYER
- NAME "m10_lakes"
- DATA /work/shapefiles/ne_10m_lakes
- EXTENT -165.966 -50.6696 176.094 81.9553
- STATUS ON
- TYPE POLYGON
- PROJECTION
- "init=epsg:4326"
- END
- GROUP bw_map
- METADATA
- "wms_title" "m10_lakes"
- END
- MINSCALEDENOM 1
- MAXSCALEDENOM 375000
- OFFSITE 128 128 128
- CLASS
- NAME '10m_lakes'
- STYLE
- WIDTH 1
- OUTLINECOLOR 128 128 128
- COLOR 192 192 192
- END
- END
- END
-
-
- LAYER
- NAME "cb_2018_us_county_500k"
- DATA /work/shapefiles/cb_2018_us_county_500k
- EXTENT -179.147 -14.5487 179.778 71.3526
- # Limit to 50 states based on FIPS codes (exclude Hawaii)
- FILTER ("[STATEFP]" <= "56" AND "[STATEFP]" != "03" AND "[STATEFP]" != "7" AND "[STATEFP]" != "14" AND "[STATEFP]" != "43" AND "[STATEFP]" != "52" AND "[STATEFP]" != "15")
- STATUS ON
- TYPE POLYGON
- PROJECTION
- "init=epsg:4269"
- END
- GROUP bw_map
- METADATA
- "wms_title" "cb_2018_us_county_500k"
- END
- MINSCALEDENOM 1
- MAXSCALEDENOM 1250000
- OFFSITE 128 128 128
- CLASS
- NAME 'cb_2018_us_county_500k'
- STYLE
- WIDTH 0.5
- OUTLINECOLOR 160 160 160
- END
- END
- END
-END
diff --git a/mapserver/mapfiles/borders.map b/mapserver/mapfiles/borders.map
deleted file mode 100644
index 291d8d2..0000000
--- a/mapserver/mapfiles/borders.map
+++ /dev/null
@@ -1,145 +0,0 @@
-MAP
- NAME "Borders"
- STATUS ON
- MAXSIZE 5000
- SIZE 1024 1024
- IMAGECOLOR 128 128 128
- IMAGETYPE rgb
- EXTENT -5434894.885 -5434894.885 5434894.885 5434894.885
- FONTSET "/work/mapfiles/fonts.list"
-
- PROJECTION
- "init=epsg:930916"
- END
-
- WEB
- METADATA
- "wms_title" "Borders"
- INCLUDE "/work/mapfiles/generic_metadata.map"
- END
- END
-
- INCLUDE '/work/mapfiles/output_formats.map'
-
- LAYER
- NAME "m10_countries"
- DATA /work/shapefiles/ne_10m_admin_0_countries
- EXTENT -180 -89.9998 180 83.6338
- STATUS ON
- TYPE POLYGON
- PROJECTION
- "init=epsg:4326"
- END
- GROUP bw_map
- METADATA
- "wms_title" "m10_countries"
- END
- MINSCALEDENOM 1
- MAXSCALEDENOM 11000000000
- OFFSITE 128 128 128
- CLASS
- NAME '10m_countries'
- STYLE
- WIDTH 2
- OUTLINECOLOR 0 0 0
- END
- STYLE
- WIDTH 1
- OUTLINECOLOR 255 255 255
- END
- END
- END
-
- LAYER
- NAME "m10_states"
- DATA /work/shapefiles/ne_10m_admin_1_states_provinces_lines
- EXTENT -139.047 -49.2507 178.449 81.1382
- STATUS ON
- # don't show this for some countries to avoid conflicting lines between layers
- FILTER (("[adm0_name]" == "Australia") OR ("[adm0_name]" == "Canada") OR ("[adm0_name]" == "United States of America"))
- TYPE LINE
- PROJECTION
- "init=epsg:4326"
- END
- GROUP bw_map
- METADATA
- "wms_title" "m10_states"
- END
- MINSCALEDENOM 1
- MAXSCALEDENOM 7500000
- OFFSITE 128 128 128
- CLASS
- NAME '10m_states'
- STYLE
- WIDTH 2
- COLOR 0 0 0
- END
- STYLE
- WIDTH 1
- COLOR 255 255 255
- END
- END
- END
-
- LAYER
- NAME "m10_lakes"
- DATA /work/shapefiles/ne_10m_lakes
- EXTENT -165.966 -50.6696 176.094 81.9553
- STATUS ON
- TYPE POLYGON
- PROJECTION
- "init=epsg:4326"
- END
- GROUP bw_map
- METADATA
- "wms_title" "m10_lakes"
- END
- # Lakes show up at zoom levels ~5 and 6
- MINSCALEDENOM 1
- MAXSCALEDENOM 3750000
- OFFSITE 128 128 128
- CLASS
- NAME '10m_lakes'
- STYLE
- WIDTH 2
- OUTLINECOLOR 0 0 0
- END
- STYLE
- WIDTH 1
- OUTLINECOLOR 255 255 255
- END
- END
- END
-
-
- LAYER
- NAME "cb_2018_us_county_500k"
- DATA /work/shapefiles/cb_2018_us_county_500k
- EXTENT -179.147 -14.5487 179.778 71.3526
- # Limit to 50 states based on FIPS codes (exclude Hawaii)
- FILTER ("[STATEFP]" <= "56" AND "[STATEFP]" != "03" AND "[STATEFP]" != "7" AND "[STATEFP]" != "14" AND "[STATEFP]" != "43" AND "[STATEFP]" != "52" AND "[STATEFP]" != "15")
- STATUS ON
- TYPE POLYGON
- PROJECTION
- "init=epsg:4269"
- END
- GROUP bw_map
- METADATA
- "wms_title" "cb_2018_us_county_500k"
- END
- MINSCALEDENOM 1
- MAXSCALEDENOM 1250000
- OFFSITE 128 128 128
- CLASS
- NAME 'cb_2018_us_county_500k'
- STYLE
- WIDTH 2
- OUTLINECOLOR 0 0 0
- END
- STYLE
- WIDTH 1
- OUTLINECOLOR 255 255 255
- END
- END
- END
-END
diff --git a/mapserver/mapfiles/generic_metadata.map b/mapserver/mapfiles/generic_metadata.map
deleted file mode 100644
index f130307..0000000
--- a/mapserver/mapfiles/generic_metadata.map
+++ /dev/null
@@ -1,3 +0,0 @@
- "wms_onlineresource" "https://myhost/mapserv"
- "wms_enable_request" "*"
- "wms_srs" "EPSG:4326 EPSG:930916 EPSG:930917"
diff --git a/mapserver/mapfiles/graticules.map b/mapserver/mapfiles/graticules.map
deleted file mode 100644
index 3b2cc82..0000000
--- a/mapserver/mapfiles/graticules.map
+++ /dev/null
@@ -1,159 +0,0 @@
-MAP
- NAME "latlongrid"
- STATUS ON
- MAXSIZE 5000
- SIZE 1024 1024
- EXTENT -180 -90 180 90
- UNITS DD
- FONTSET "/work/mapfiles/fonts.list"
-
- PROJECTION
- "init=epsg:4326"
- END
-
- WEB
- METADATA
- "wms_title" "CSPP Geo Geo2Grid Viewer - Graticule Lines"
- INCLUDE "/work/mapfiles/generic_metadata.map"
- END
- END
-
- INCLUDE '/work/mapfiles/output_formats.map'
-
- IMAGECOLOR 127 127 127
- IMAGETYPE rgb
-
- LAYER
- NAME "1degree_grid"
- DATA /work/shapefiles/ne_10m_graticules_1
- EXTENT -180 -90 180 90
- STATUS ON
- TYPE LINE
- PROJECTION
- "init=epsg:4326"
- END
- GROUP latlongrid
- METADATA
- "wms_title" "10m resolution 1 degree grid"
- END
- MINSCALEDENOM 1
- MAXSCALEDENOM 1250000
- OFFSITE 127 127 127
- CLASS
- NAME '1degree_grid'
- TEXT '[degrees]°'
- LABEL
- TYPE TRUETYPE
- FONT arial
- SIZE 10
- FORCE FALSE
- PARTIALS FALSE
- POSITION AUTO
- MINDISTANCE 1024
- COLOR 255 255 255
- OUTLINECOLOR 0 0 0
- MINSCALEDENOM 1
- MAXSCALEDENOM 1250000
- END
- STYLE
- PATTERN 16 16 END
- WIDTH 2
- COLOR 0 0 0
- END
- STYLE
- PATTERN 16 16 END
- WIDTH 1
- COLOR 255 255 255
- END
- END
- END
-
- LAYER
- NAME "5degree_grid"
- DATA /work/shapefiles/ne_10m_graticules_5
- EXTENT -180 -90 180 90
- STATUS ON
- TYPE LINE
- PROJECTION
- "init=epsg:4326"
- END
- GROUP latlongrid
- METADATA
- "wms_title" "10m resolution 5 degree grid"
- END
- MINSCALEDENOM 1250000
- MAXSCALEDENOM 4000000
- OFFSITE 127 127 127
- CLASS
- NAME '5degree_grid'
- TEXT '[degrees]°'
- LABEL
- TYPE TRUETYPE
- FONT arial
- SIZE 10
- FORCE FALSE
- PARTIALS FALSE
- POSITION AUTO
- MINDISTANCE 1024
- COLOR 255 255 255
- OUTLINECOLOR 0 0 0
- MINSCALEDENOM 1250000
- MAXSCALEDENOM 4000000
- END
- STYLE
- PATTERN 32 32 END
- WIDTH 2
- COLOR 0 0 0
- END
- STYLE
- PATTERN 32 32 END
- WIDTH 1
- COLOR 255 255 255
- END
- END
- END
-
- LAYER
- NAME "10degree_grid"
- DATA /work/shapefiles/ne_10m_graticules_10
- EXTENT -180 -90 180 90
- STATUS ON
- TYPE LINE
- PROJECTION
- "init=epsg:4326"
- END
- GROUP latlongrid
- METADATA
- "wms_title" "10m resolution 10 degree grid"
- END
- MINSCALEDENOM 4000000
- MAXSCALEDENOM 11000000000
- OFFSITE 127 127 127
- CLASS
- NAME '10degree_grid'
- TEXT '[degrees]°'
- LABEL
- TYPE TRUETYPE
- FONT arial
- SIZE 10
- FORCE FALSE
- PARTIALS FALSE
- POSITION AUTO
- MINDISTANCE 1024
- COLOR 255 255 255
- OUTLINECOLOR 0 0 0
- MINSCALEDENOM 4000000
- MAXSCALEDENOM 11000000000
- END
- STYLE
- WIDTH 2
- COLOR 0 0 0
- END
- STYLE
- WIDTH 1
- COLOR 255 255 255
- END
- END
- END
-
-END
diff --git a/mapserver/mapfiles/output_formats.map b/mapserver/mapfiles/output_formats.map
deleted file mode 100644
index 3076db4..0000000
--- a/mapserver/mapfiles/output_formats.map
+++ /dev/null
@@ -1,87 +0,0 @@
-# Full color with alpha
-
- OUTPUTFORMAT
- NAME "rgba"
- DRIVER AGG/PNG
- MIMETYPE "image/png"
- IMAGEMODE RGBA
- TRANSPARENT ON
- EXTENSION "png"
- END
-
- OUTPUTFORMAT
- NAME "rgb"
- DRIVER AGG/PNG
- MIMETYPE "image/png"
- IMAGEMODE RGB
- TRANSPARENT OFF
- EXTENSION "png"
- END
-
- OUTPUTFORMAT
- NAME "png"
- DRIVER AGG/PNG
- MIMETYPE "image/png"
- IMAGEMODE RGBA
- TRANSPARENT ON
- EXTENSION "png"
- END
-
- OUTPUTFORMAT
- NAME "png24"
- DRIVER AGG/PNG
- MIMETYPE "image/png"
- IMAGEMODE RGBA
- TRANSPARENT ON
- EXTENSION "png"
- END
-
-# 8 bit color with no transparency
-
- OUTPUTFORMAT
- NAME "png256"
- DRIVER AGG/PNG
- MIMETYPE "image/png"
- IMAGEMODE RGB
- FORMATOPTION "QUANTIZE_FORCE=ON"
- FORMATOPTION "QUANTIZE_DITHER=OFF"
- FORMATOPTION "QUANTIZE_COLORS=256"
- EXTENSION "png"
- END
-
- OUTPUTFORMAT
- NAME "png8"
- DRIVER AGG/PNG8
- MIMETYPE "image/png"
- IMAGEMODE PC256
- FORMATOPTION "QUANTIZE_FORCE=ON"
- FORMATOPTION "QUANTIZE_DITHER=OFF"
- FORMATOPTION "QUANTIZE_COLORS=256"
- FORMATOPTION "PALETTE_FORCE=ON"
- FORMATOPTION "COMPRESSION=9"
- TRANSPARENT ON
- EXTENSION "png"
- END
-
- OUTPUTFORMAT
- NAME "gray"
- DRIVER AGG/PNG8
- MIMETYPE "image/png"
- IMAGEMODE PC256
- FORMATOPTION "QUANTIZE_FORCE=ON"
- FORMATOPTION "QUANTIZE_DITHER=OFF"
- FORMATOPTION "QUANTIZE_COLORS=256"
- FORMATOPTION "PALETTE_FORCE=ON"
- FORMATOPTION "COMPRESSION=9"
- TRANSPARENT ON
- EXTENSION "png"
- END
-
-# Other
-
- OUTPUTFORMAT
- NAME "geojson"
- DRIVER "TEMPLATE"
- MIMETYPE "application/json"
- FORMATOPTION "FILE=/home/wms/localdata/mapfiles/templates/template0.html"
- END
diff --git a/mapserver/render.py b/mapserver/render.py
deleted file mode 100644
index 0c1864a..0000000
--- a/mapserver/render.py
+++ /dev/null
@@ -1,82 +0,0 @@
-#!/usr/bin/env python3
-"""Convert a template mapfile config to a full config.
-
-We do this rendering when the container is started to allow for complete
-control per mapserver instance (even though there is usually only one).
-
-"""
-
-import os
-import sys
-import jinja2
-
-
-sectors = "radm1 radm2 radc radf"
-sectors = os.getenv("WMS_SECTORS", sectors).split(" ")
-products = ["C{:02d}".format(x) for x in range(1, 17)] + ["true_color"]
-products = os.getenv("WMS_PRODUCTS", " ".join(products)).split(" ")
-platforms = "g16 g17"
-platforms = os.getenv("WMS_PLATFORMS", platforms).split(" ")
-
-layer_base_dir = os.environ['LAYER_BASE_DIR']
-
-
-SECTOR_LONG_NAMES = {
- 'radm1': 'Mesoscale 1',
- 'radm2': 'Mesoscale 2',
- 'radc': 'CONUS',
- 'radf': 'Full disk',
-}
-
-PLATFORM_LONG_NAMES = {
- 'g16': 'GOES-16',
- 'g17': 'GOES-17',
-}
-
-PLATFORM_EPSGS = {
- 'g16': 'EPSG:930916',
- 'g17': 'EPSG:930917',
-}
-
-PLATFORM_POS_NAMES = {
- 'g16': 'GOES-EAST',
- 'g17': 'GOES-WEST',
-}
-
-LAYER_TO_FORMAT = {
- 'C01': 'gray',
- 'true_color': 'rgba'
-}
-
-
-def main():
- import argparse
- parser = argparse.ArgumentParser(description="Render mapfile template to complete mapfile config.")
- parser.add_argument("template_fn",
- help="Template XML file to render")
- parser.add_argument('output_pattern',
- help='Format string for each output mapfiles.')
- args = parser.parse_args()
-
- for platform in platforms:
- for sector in sectors:
- tmpl_vars = {
- 'sector': sector,
- 'platform': platform,
- 'products': products,
- 'layer_base_dir': layer_base_dir,
- 'platform_long_name': PLATFORM_LONG_NAMES[platform],
- 'sector_long_name': SECTOR_LONG_NAMES[sector],
- 'projection': "init=" + PLATFORM_EPSGS[platform].lower(),
- 'PLATFORM_EPSGS': PLATFORM_EPSGS,
- 'PLATFORM_POS_NAMES': PLATFORM_POS_NAMES,
- }
- output_fn = args.output_pattern.format(**tmpl_vars)
-
- with open(args.template_fn, 'r') as tmpl_file, open(output_fn, 'w') as mapfile:
- template = jinja2.Template(tmpl_file.read())
- mapfile.write(template.render(tmpl_vars))
-
-
-if __name__ == "__main__":
- sys.exit(main())
\ No newline at end of file
diff --git a/mapserver/run.sh b/mapserver/run.sh
deleted file mode 100755
index 8f79599..0000000
--- a/mapserver/run.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash -le
-
-# Load environment variable options to overwrite in the config
-export LAYER_BASE_DIR=${LAYER_BASE_DIR:-"/data/tiles"}
-
-# Update the mapcache.xml file with the real host and port name
-mf_tmpl="/work/abi_l1b_template.map"
-export WMS_PLATFORMS=${WMS_PLATFORMS:-"g16 g17"}
-export WMS_SECTORS=${WMS_SECTORS:-"radm1 radm2 radc radf"}
-python3 /work/render.py $mf_tmpl "/work/mapfiles/{platform}_abi_{sector}_l1b.map"
-
-/usr/sbin/apache2ctl configtest
-
-/usr/sbin/apache2ctl -DFOREGROUND
diff --git a/mapserver/site-conf b/mapserver/site-conf
deleted file mode 100644
index 0c23c17..0000000
--- a/mapserver/site-conf
+++ /dev/null
@@ -1,56 +0,0 @@
-<VirtualHost *:80>
- # The ServerName directive sets the request scheme, hostname and port that
- # the server uses to identify itself. This is used when creating
- # redirection URLs. In the context of virtual hosts, the ServerName
- # specifies what hostname must appear in the request's Host: header to
- # match this virtual host. For the default virtual host (this file) this
- # value is not decisive as it is used as a last resort host regardless.
- # However, you must set it for any further virtual host explicitly.
- #ServerName www.example.com
-
- ServerAdmin david.hoese@ssec.wisc.edu
- DocumentRoot /var/www/html
-
- # Available loglevels: trace8, ..., trace1, debug, info, notice, warn,
- # error, crit, alert, emerg.
- # It is also possible to configure the loglevel for particular
- # modules, e.g.
- #LogLevel info ssl:warn
-
- ErrorLog ${APACHE_LOG_DIR}/error.log
- CustomLog ${APACHE_LOG_DIR}/access.log combined
-
- # For most configuration files from conf-available/, which are
- # enabled or disabled at a global level, it is possible to
- # include a line for only one particular virtual host. For example the
- # following line enables the CGI configuration for this host only
- # after it has been globally disabled with "a2disconf".
- #Include conf-available/serve-cgi-bin.conf
-
- # Custom CSPP Geo
- RewriteEngine on
- # /wms/g16/abi/radf/l1b?...
- # /wms/g16/abi/radf/borders?...
- # FCGI version of the URL
- RewriteRule "^/wms/([^/]+)/([^/]+)/([^/]+)/([^/]+)?(.*)" "/cgi-bin/mapserv.fcgi?map=/work/mapfiles/$1_$2_$3_$4.map&$5" [PT,QSA]
- RewriteRule "^/wms/([^/]+)?(.*)" "/cgi-bin/mapserv.fcgi?map=/work/mapfiles/$1.map&$2" [PT,QSA]
- #RewriteRule "^/fwms/([^/]+)/([^/]+)/([^/]+)/([^/]+)?(.*)" "/cgi-bin/mapserv.fcgi?map=/work/mapfiles/$1_$2_$3_$4.map&$5" [PT,QSA]
- #RewriteRule "^/fwms/([^/]+)?(.*)" "/cgi-bin/mapserv.fcgi?map=/work/mapfiles/$1.map&$2" [PT,QSA]
- # /wms_times/g16/abi/radf/true_color
- RewriteRule "^/wms_times/([^/]+)/([^/]+)/([^/]+)/([^/]+)" "/cgi-bin/layer_times.py?layer=/data/tiles/$1/$2/$3/$4/$4.shp" [PT,QSA]
- # Regular CGI version of the URL (put last since it should be used less often)
- RewriteRule "^/wms_cgi/([^/]+)/([^/]+)/([^/]+)/([^/]+)?(.*)" "/cgi-bin/mapserv?map=/work/mapfiles/$1_$2_$3_$4.map&$5" [PT,QSA]
- RewriteRule "^/wms_cgi/([^/]+)?(.*)" "/cgi-bin/mapserv?map=/work/mapfiles/$1.map&$2" [PT,QSA]
- #RewriteRule "^/wms/([^/]+)/([^/]+)/([^/]+)/([^/]+)?(.*)" "/cgi-bin/mapserv?map=/work/mapfiles/$1_$2_$3_$4.map&$5" [PT,QSA]
- #RewriteRule "^/wms/([^/]+)?(.*)" "/cgi-bin/mapserv?map=/work/mapfiles/$1.map&$2" [PT,QSA]
-
- # handler should be added by the enabled module
- #AddHandler fcgid-script fcgi
- <IfModule mod_fcgid.c>
- FcgidMaxProcessesPerClass 30
- FcgidInitialEnv PROJ_LIB /usr/local/share/proj
- FcgidInitialEnv LD_LIBRARY_PATH "/usr/local/lib:/usr/local/pgsql/lib:/usr3/pkg3/oracle9/lib"
- </IfModule>
-</VirtualHost>
-
-# vim: syntax=apache ts=4 sw=4 sts=4 sr noet
\ No newline at end of file
diff --git a/mapserver/sql/goesr_crs.sql b/mapserver/sql/goesr_crs.sql
deleted file mode 100644
index 0cbf495..0000000
--- a/mapserver/sql/goesr_crs.sql
+++ /dev/null
@@ -1,2 +0,0 @@
-INSERT INTO projected_crs (auth_name, code, name, description, scope, coordinate_system_auth_name, coordinate_system_code, geodetic_crs_auth_name, geodetic_crs_code, conversion_auth_name, conversion_code, area_of_use_auth_name, area_of_use_code, text_definition, deprecated) VALUES ('EPSG', '930917', 'GOES-17 ABI Fixed Grid', 'GOES-17 (GOES-WEST) ABI Fixed Grid in the Geostationary projection', null, null, null, 'EPSG', '4269', null, null, null, null, 'PROJCRS["GOES-17 ABI Fixed Grid",BASEGEOGCRS["GOES-17 ABI Fixed Grid",DATUM["North American Datum 1983",ELLIPSOID["GRS 1980",6378137,298.257222101,LENGTHUNIT["metre",1]],ID["EPSG",6269]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8901]]],CONVERSION["unknown",METHOD["Geostationary Satellite (Sweep X)"],PARAMETER["Longitude of natural origin",-137,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8802]],PARAMETER["Satellite Height",35786023,LENGTHUNIT["metre",1,ID["EPSG",9001]]],PARAMETER["False easting",0,LENGTHUNIT["metre",1],ID["EPSG",8806]],PARAMETER["False northing",0,LENGTHUNIT["metre",1],ID["EPSG",8807]]],CS[Cartesian,2],AXIS["(E)",east,ORDER[1],LENGTHUNIT["metre",1,ID["EPSG",9001]]],AXIS["(N)",north,ORDER[2],LENGTHUNIT["metre",1,ID["EPSG",9001]]]]', 0);
-INSERT INTO projected_crs (auth_name, code, name, description, scope, coordinate_system_auth_name, coordinate_system_code, geodetic_crs_auth_name, geodetic_crs_code, conversion_auth_name, conversion_code, area_of_use_auth_name, area_of_use_code, text_definition, deprecated) VALUES ('EPSG', '930916', 'GOES-16 ABI Fixed Grid', 'GOES-16 (GOES-WEST) ABI Fixed Grid in the Geostationary projection', null, null, null, 'EPSG', '4269', null, null, null, null, 'PROJCRS["GOES-16 ABI Fixed Grid",BASEGEOGCRS["GOES-16 ABI Fixed Grid",DATUM["North American Datum 1983",ELLIPSOID["GRS 1980",6378137,298.257222101,LENGTHUNIT["metre",1]],ID["EPSG",6269]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8901]]],CONVERSION["unknown",METHOD["Geostationary Satellite (Sweep X)"],PARAMETER["Longitude of natural origin",-75,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8802]],PARAMETER["Satellite Height",35786023,LENGTHUNIT["metre",1,ID["EPSG",9001]]],PARAMETER["False easting",0,LENGTHUNIT["metre",1],ID["EPSG",8806]],PARAMETER["False northing",0,LENGTHUNIT["metre",1],ID["EPSG",8807]]],CS[Cartesian,2],AXIS["(E)",east,ORDER[1],LENGTHUNIT["metre",1,ID["EPSG",9001]]],AXIS["(N)",north,ORDER[2],LENGTHUNIT["metre",1,ID["EPSG",9001]]]]', 0);
\ No newline at end of file
diff --git a/mapserver/tiledb_geospatial_Dockerfile b/mapserver/tiledb_geospatial_Dockerfile
deleted file mode 100644
index f80985e..0000000
--- a/mapserver/tiledb_geospatial_Dockerfile
+++ /dev/null
@@ -1,159 +0,0 @@
-# Based on
-
-FROM ubuntu:18.04
-LABEL maintainer="support@tiledb.io"
-
-ENV DEBIAN_FRONTEND=noninteractive
-ENV TZ=GMT
-ENV LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH}
-ENV LANG=C.UTF-8
-ENV LC_ALL=C.UTF-8
-
-RUN apt-get update && apt-get install -y \
- gosu \
- pwgen \
- tzdata \
- gcc \
- g++ \
- build-essential \
- cmake \
- sqlite \
- libsqlite3-dev \
- libxml2-dev \
- libjpeg-dev \
- libpng-dev \
- libfreetype6-dev \
- libzstd-dev \
- python3-pip \
- git \
- wget \
- && rm -rf /var/lib/apt/lists/*
-
-# Install tiledb using 1.7.2 release
-RUN mkdir -p /build_deps && cd /build_deps \
- && git clone https://github.com/TileDB-Inc/TileDB.git -b 1.7.2 && cd TileDB \
- && mkdir -p build && cd build \
- && cmake -DTILEDB_VERBOSE=ON -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. \
- && make -j$(nproc) \
- && make -C tiledb install
-
-# Install curl after building tiledb
-RUN apt-get update && apt-get install -y \
- libcurl4 \
- libcurl4-openssl-dev \
- && rm -rf /var/lib/apt/lists/*
-
-# Install OpenJPEG
-RUN cd /build_deps \
- && git clone https://github.com/uclouvain/openjpeg.git -b v2.2.0 && cd openjpeg \
- && mkdir -p build && cd build \
- && cmake .. \
- && make -j$(nproc) \
- && make install
-
-# Install libtiff
-RUN cd /build_deps \
- && wget --no-check-certificate https://download.osgeo.org/libtiff/tiff-4.1.0.tar.gz \
- && tar -zxf tiff-4.1.0.tar.gz \
- && cd tiff-4.1.0 \
- && ./configure \
- && make \
- && make install
-
-# Install Proj
-RUN cd /build_deps \
- && git clone https://github.com/OSGeo/PROJ.git -b 6.2.1 && cd PROJ \
- && mkdir -p build && cd build \
- && cmake .. \
- && make -j$(nproc) \
- && make install
-
-# Install libgeotiff
-RUN cd /build_deps \
- && wget --no-check-certificate https://download.osgeo.org/geotiff/libgeotiff/libgeotiff-1.5.1.tar.gz \
- && tar -zxf libgeotiff-1.5.1.tar.gz \
- && cd libgeotiff-1.5.1 \
- && mkdir -p build && cd build \
- && cmake .. \
- && make \
- && make install
-
-# Install GDAL
-RUN cd /build_deps \
- && git clone https://github.com/OSGeo/gdal.git && cd gdal/gdal \
- && git checkout c99a871a7bdedc751c503bb8cf508d9016510fe0 \
- && ./configure --with-crypto=no --with-curl=no \
- && make -j$(nproc) \
- && make install
-
-## Install TileDB-Py
-RUN cd /build_deps \
- && pip3 install numpy \
- && git clone https://github.com/TileDB-Inc/TileDB-Py.git -b 0.5.3 \
- && cd TileDB-Py && python3 setup.py install
-
-## Install XArray
-RUN cd /build_deps && pip3 install xarray
-
-## Install Dask
-RUN cd /build_deps \
- && pip3 install toolz && pip3 install dask_image \
- && git clone https://github.com/dask/dask.git && cd dask \
- && git checkout 807f3225cf840f28ce7cf89b88fea63d473889e7 \
- && python3 setup.py install \
- && pip3 install dask distributed --upgrade \
- && pip3 install dask-image
-
-# Install Rasterio
-RUN cd /build_deps && pip3 install cython
-RUN cd /build_deps \
- && git clone https://github.com/mapbox/rasterio.git -b 1.1.0 && cd rasterio \
- && python3 setup.py install
-
-# Install Fiona
-RUN cd /build_deps \
- && git clone https://github.com/Toblerity/Fiona.git && cd Fiona \
- && python3 setup.py install
-
-# Install TileDB-SAR
-RUN cd /build_deps \
- && git clone https://github.com/TileDB-Inc/TileDB-SAR.git && cd TileDB-SAR \
- && git checkout 888059a15d87ae95fff6dc01c8bd4343ee4eaee1 \
- && python3 setup.py install
-
-# Install Mapserver
-RUN cd /build_deps \
- && git clone https://github.com/mapserver/mapserver.git && cd mapserver \
- && git checkout 0fcc810f0b559c800f950db78a79fa6574799f23 \
- && mkdir -p build && cd build \
- && cmake .. -DWITH_GIF=OFF -DWITH_HARFBUZZ=OFF -DWITH_PROTOBUFC=OFF -DWITH_FRIBIDI=OFF -DWITH_POSTGIS=OFF -DWITH_GEOS=OFF -DWITH_FCGI=OFF -DWITH_CAIRO=OFF \
- && make \
- && make install
-
-# Install LasZIP
-RUN cd /build_deps \
- && wget https://github.com/LASzip/LASzip/releases/download/3.4.1/laszip-src-3.4.1.tar.gz \
- && tar -zxf laszip-src-3.4.1.tar.gz \
- && cd laszip-src-3.4.1 \
- && mkdir -p build && cd build \
- && cmake .. \
- && make \
- && make install
-
-# Install PDAL
-RUN cd /build_deps \
- && git clone https://github.com/PDAL/PDAL.git -b 2.0.1 && cd PDAL \
- && mkdir -p build && cd build \
- && cmake .. \
- && make \
- && make install
-
-# Install PDAL Python
-RUN pip3 install packaging \
- && git clone https://github.com/PDAL/python pdalextension \
- && cd pdalextension \
- && python3 setup.py build \
- && python3 setup.py install
-
-# Clean up
-RUN cd /tmp && rm -r /build_deps
diff --git a/tile_gen/Dockerfile b/tile_gen/Dockerfile
deleted file mode 100644
index 7393160..0000000
--- a/tile_gen/Dockerfile
+++ /dev/null
@@ -1,15 +0,0 @@
-FROM tiledb/tiledb-geospatial:latest
-
-WORKDIR /work
-
-RUN apt-get update && apt-get install -y unzip && \
- wget http://www.ssec.wisc.edu/~rayg/pub/amqpfind.zip && \
- unzip amqpfind.zip && \
- rm amqpfind.zip && \
- rm -rf /var/lib/apt/lists/*
-
-# FIXME: Remove once added to parent image
-RUN pip3 install shapely
-COPY tile_index.py .
-COPY generate_tiles.py .
-COPY run.sh .
diff --git a/tile_gen/README.md b/tile_gen/README.md
deleted file mode 100644
index a4f58e4..0000000
--- a/tile_gen/README.md
+++ /dev/null
@@ -1,31 +0,0 @@
-# Tile Generation
-
-## Build
-
-```bash
-docker build -t gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/tile_gen:latest tile_gen/
-docker push gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/tile_gen:latest
-```
-
-## Usage
-
-```bash
-docker run -d --rm --network cspp-geo-rabbit --cpus 2 --name cspp-geo-tilegen-g16-radf -e AMQPFIND_TOPIC="data.goes.g16.abi.radf.l1b.geotiff.all.complete" -v cspp-geo-abi-l1b-geotiffs:/data gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/tile_gen:latest ./run.sh
-```
-
-To run the version of tile generation that remaps to EPSG:4326:
-
-```bash
-docker run -d --rm --network cspp-geo-rabbit --cpus 6 --name cspp-geo-tilegen-g16-radf-ll -e AMQPFIND_TOPIC="data.goes.g16.abi.radf.l1b.geotiff.all.complete" -e G2G_PRODUCTS="true_color" -e TILE_ARGS="--remap --shape-file {product}_ll.shp" -v cspp-geo-abi-l1b-geotiffs:/data gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-web-viewer/tile_gen:latest ./run.sh
-```
-
-NOTE: For full disk data, remapping all products can take a really long
-time and fall behind. The above command limits processing to "true_color"
-only. To attempt to process all products remove the below portion of the
-command.
-
-```
--e G2G_PRODUCTS="true_color"
-```
-
-And adding more CPUs.
\ No newline at end of file
diff --git a/tile_gen/generate_tiles.py b/tile_gen/generate_tiles.py
deleted file mode 100644
index 65079a9..0000000
--- a/tile_gen/generate_tiles.py
+++ /dev/null
@@ -1,131 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import sys
-import warnings
-import logging
-import subprocess
-import shutil
-import json
-from glob import glob
-
-import tile_index
-
-LOG = logging.getLogger(__name__)
-
-
-def group_files(products, input_files):
- """Group input geotiff files by product."""
- groups = {}
- for prod in products:
- prods_files = [f for f in input_files if prod in f]
- if prods_files:
- groups[prod] = prods_files
- if len(groups) != len(products):
- warnings.warn("Not all product geotiffs were provided.")
- return groups
-
-
-def remap_to_lonlat(itif, otif):
- """Remap a single geotiff by calling gdalwarp."""
- try:
- subprocess.run(['gdalwarp', '-multi', '-wo', 'NUM_THREADS=ALL_CPUS', '-t_srs', 'EPSG:4326', itif, otif], check=True)
- except subprocess.CalledProcessError:
- LOG.error("Could not remap geotiff %s -> %s" % (itif, otif))
- return None
- return otif
-
-
-def remap_tifs(input_tifs, out_dir, remap_suffix):
- """Remap all input geotiffs to EPSG:4326."""
- for itif in input_tifs:
- ifn = os.path.basename(itif)
- otif = os.path.join(out_dir, ifn.replace('.tif', remap_suffix))
- otif = remap_to_lonlat(itif, otif)
- if otif is not None:
- yield otif
-
-
-def link_or_copy(input_tifs, out_dir):
- """Hardlink input tifs to output directory."""
- for prod_file in input_tifs:
- out_file = os.path.join(out_dir, os.path.basename(prod_file))
- if os.path.exists(out_file):
- yield out_file
- continue
- try:
- os.link(prod_file, out_file)
- except OSError:
- # on different mounts probably?
- shutil.copy2(prod_file, out_file)
- yield out_file
-
-
-def main():
- import argparse
- parser = argparse.ArgumentParser(description="Take input geotiffs and generate mapserver compatible tiles.")
- parser.add_argument('--remap', action='store_true',
- help="Remap input geotiffs to EPSG:4326")
- parser.add_argument('--remap-suffix', default='_ll.tif',
- help="Replace 'tif' with provided suffix when geotiffs are remapped.")
- parser.add_argument('-p', '--products', nargs="*",
- help="Product names to group together in each "
- "'layer'. Product name must be in the filename.")
- parser.add_argument('--shape-file', default='{product}.shp',
- help="Shapefile filename pattern to use and placed "
- "in the output directory. "
- "(default: '{product}.shp')")
- parser.add_argument('--json-times', action='store_true',
- help="Print a JSON dict to stdout containing removed "
- "and added times (ex. '{\'times_removed\': [], "
- "\'times_added\': []}'). Times are in the form: "
- "YYYY-MM-DDTHH:MM:SS.")
- parser.add_argument('--shared-bbox', action='store_true',
- help="Assume that all input geotiffs have the same "
- "bounding box so no need to compute it for every "
- "input.")
- parser.add_argument('out_dir',
- help="Output path to save tile information to (ex. '/data/tiles/{product}')")
- parser.add_argument('input_files', nargs="+",
- help="Input geotiffs to generate tiles for (separate from product lists with '--')")
- args = parser.parse_args()
-
- # allow for glob patterns to reduce number of files passed through command line
- args.input_files = [fn for glob_pat in args.input_files for fn in glob(glob_pat)]
- groups = group_files(args.products, args.input_files)
- all_added = set()
- all_removed = set()
- for prod, prod_files in groups.items():
- out_dir = args.out_dir.format(product=prod)
- os.makedirs(out_dir, exist_ok=True)
- shp_fn = args.shape_file.format(product=prod)
- shp_pathname = os.path.join(out_dir, shp_fn)
-
- if args.remap:
- # remap if needed
- prod_files = list(remap_tifs(prod_files, out_dir, args.remap_suffix))
- else:
- # hardlink if needed
- prod_files = list(link_or_copy(prod_files, out_dir))
-
- # get all products in the current directory
- ext = os.path.splitext(prod_files[-1])[-1]
- all_prod_files = sorted(glob(os.path.join(out_dir, '*' + prod + '*' + ext)))
-
- # create shape file
- LOG.info("Rebuilding shapefile index with:\n\t{}".format(", ".join(all_prod_files)))
- removed_times, added_times = tile_index.index(
- all_prod_files, shp_pathname, shared_bbox=args.shared_bbox)
- all_removed.update(removed_times)
- all_added.update(added_times)
-
- if args.json_times:
- # all_removed = [x.isoformat() for x in sorted(all_removed)]
- # all_added = [x.isoformat() for x in sorted(all_added)]
- all_removed = sorted(all_removed)
- all_added = sorted(all_added)
- print(json.dumps({'removed_times': all_removed, 'added_times': all_added}))
-
-
-if __name__ == "__main__":
- sys.exit(main())
\ No newline at end of file
diff --git a/tile_gen/run.sh b/tile_gen/run.sh
deleted file mode 100755
index d4c82c2..0000000
--- a/tile_gen/run.sh
+++ /dev/null
@@ -1,91 +0,0 @@
-#!/bin/bash -le
-# Usage: run.sh
-# Environment variables used for configuration:
-# AMQPFIND_ARGS: Arguments to pass to amqpfind when listening for new input
-# events. Should not include the "-C" topic flag (see AMQPFIND_TOPIC).
-# Default: "-H cspp-geo-rabbit -X satellite -u guest -p guest"
-# AMQPFIND_TOPIC: Topic to use for incoming data events.
-# Default: "data.goes.*.abi.*.l1b.netcdf.complete"
-# The first asterisk (3rd element) can limit processing to a particular
-# satellite (ex. `g16`). The second asterisk (5th element) can be used
-# to limit to a particular sector (choices: radf, radc, radm1, radm2)
-# AMQPSEND_ARGS: Arguments to pass to amqpsend when sending out new data
-# events. Default: "-H cspp-geo-rabbit -X satellite -u guest -p guest"
-
-# Verify that the data mount is available
-test -d "/data"
-
-export AMQPFIND_ARGS=${AMQPFIND_ARGS:-"-H cspp-geo-rabbit -X satellite -u guest -p guest"}
-export AMQPSEND_ARGS=${AMQPSEND_ARGS:-"-D 15 -H cspp-geo-rabbit -X satellite -u guest -p guest"}
-export AMQPFIND_TOPIC=${AMQPFIND_TOPIC:-'data.goes.*.abi.*.l1b.geotiff.all.complete'}
-export G2G_PRODUCTS=${G2G_PRODUCTS:-"C01 C02 C03 C04 C05 C06 C07 C08 C09 C10 C11 C12 C13 C14 C15 C16 true_color"}
-export TILE_ARGS=${TILE_ARGS:-""}
-export TMPDIR=${TMPDIR:-"/dst/tmp"}
-
-run_tile_gen() {
- if [[ $# -ne 5 ]]; then
- echo "Unexpected number of arguments (expected 5): $#"
- return 1
- fi
-
- satellite_family=${1,,}
- satellite_id=${2,,}
- instrument=${3,,}
- data_type=${4,,}
- path="$5"
- echo "Starting tile generation processing for ${path}"
- # convert path from a relative path to an absolute path
- path="/data/${path}"
-
- # if /dst isn't defined then use /data
- if [[ -d "/dst" ]]; then
- dst_dir="/dst"
- else
- dst_dir="/data"
- fi
-
- # update shapefile in a temporary directory
- # and resample geotiff if necessary
- # FUTURE: TileDB will be updated in-place
- # generate_tiles.py will make a temporary directory
- out_dir="${dst_dir}/tiles/${satellite_id}/${instrument}/${data_type}"
- # if we are going to remap, separate the geotiffs and shapefile from unremapped
- if [[ ${TILE_ARGS} == *"--remap"* ]]; then
- echo "Adding '_ll' prefix to tile output directory because of remapping"
- out_dir="${out_dir}_ll"
- mkdir -p ${out_dir}
- # add string formatting portion to separate add 'product' sub-directory
- out_dir="${out_dir}/{product}_ll"
- else
- mkdir -p ${out_dir}
- # add string formatting portion to separate add 'product' sub-directory
- out_dir="${out_dir}/{product}"
- fi
- # Add optimization to only compute the bboxonce
- if ([ "$data_type" == "radf" ] || [ "$data_type" == "radc" ]) && [ $TILE_ARGS =~ "shared" ]; then
- TILE_ARGS="${TILE_ARGS} --shared-bbox"
- fi
- echo "Generating tiles in directory: ${out_dir}"
- for product in ${G2G_PRODUCTS}; do
- set -x
- json_times_changed=$(python3 generate_tiles.py ${TILE_ARGS} --json-times -p ${product} -- ${out_dir} "${path}")
- set +x
- # OUT/<product>/<product>.shp
- glob_pattern="${out_dir/\{product\}/${product}}/*.shp"
- # Remove the /data prefix
- glob_pattern="${glob_pattern/${dst_dir}\//}"
-
- amqpsend_topic="data.${satellite_family}.${satellite_id}.${instrument}.${data_type}.l1b.tiles.${product,,}.complete"
- json_info="{\"path\": \"${glob_pattern}\", \"satellite_family\": \"${satellite_family}\", \"satellite_ID\": \"${satellite_id}\", \"instrument\": \"${instrument}\", \"data_type\": \"${data_type}\", \"product\": \"${product}\"}"
- # append the JSON returned by the python (remove the curly braces at the ends)
- json_info="${json_info:0:-1}, ${json_times_changed:1}"
- echo -e "[[\"$amqpsend_topic\", $json_info]]"
- echo -e "[[\"$amqpsend_topic\", $json_info]]" | python3 /work/amqpfind/amqpsend.py ${AMQPSEND_ARGS}
- done
- echo "Done generating tiles for ${path}"
-}
-
-export -f run_tile_gen
-echo "Listening to AMQP messages with topic \"$AMQPFIND_TOPIC\""
-python3 amqpfind/amqpfind.py ${AMQPFIND_ARGS} -C "${AMQPFIND_TOPIC}" -j "{satellite_family} {satellite_ID} {instrument} {data_type} \'{path}\'" | xargs -I{} -P4 -n1 bash -c "run_tile_gen {}"
-
diff --git a/tile_gen/tile_index.py b/tile_gen/tile_index.py
deleted file mode 100644
index 1879670..0000000
--- a/tile_gen/tile_index.py
+++ /dev/null
@@ -1,177 +0,0 @@
-import sys
-import fiona
-from fiona.crs import from_epsg
-import rasterio
-from shapely.geometry import mapping, box
-
-import argparse
-import datetime
-import glob
-import logging
-import os
-import re
-import shutil
-
-# Remap geostationary to EPSG 4326
-# gdalwarp -t_srs EPSG:4326 in.tif out.tif
-
-
-# Note if using docker tiledb-geospatial image then requires shapely - `pip3 install shapely`
-# export CPL_DEBUG=ON - GDAL
-# export MS_DEBUGLEVEL=6 - MAPSERVER
-# python3 tile_index.py -dir data
-# sample mapserver queries
-# mapserv -nh "QUERY_STRING=map=goes.map&request=GetCapabilities&service=WMS&version=1.1.1"
-# mapserv -nh "QUERY_STRING=map=goes.map&request=GetMap&service=WMS&version=1.1.1&layers=goes_abi&srs=EPSG:4326&bbox=-180,-90,180,90&format=image/jpeg&WIDTH=1000&HEIGHT=1000&TIME=2019-12-12T19:10:18" > out.jpg
-# TODO add overview example to mapserver
-
-logger = logging.getLogger(__name__)
-
-temporal_schema = {
- 'geometry': 'Polygon',
- 'properties': {
- 'location': 'str',
- 'time': 'str:19'
- }
-}
-
-possible_time_regex = (
- (re.compile(r'\d{4}\d{2}\d{2}_\d{2}\d{2}\d{2}'), '%Y%m%d_%H%M%S'),
- (re.compile(r'\d{4}\d{2}\d{2}T\d{2}\d{2}\d{2}'), '%Y%m%dT%H%M%S'),
-)
-
-
-def previous_shapes_generator(shp_filename):
- """Get currently available times in the shapefile."""
- if not os.path.isfile(shp_filename):
- # no shapefile, no times
- return
-
- with fiona.open(shp_filename, 'r') as shp_file:
- for time_step in shp_file:
- yield time_step['properties']['location'], time_step
-
-
-def get_file_time(fn):
- for regex, time_fmt in possible_time_regex:
- matches = regex.findall(fn)
- if matches:
- return datetime.datetime.strptime(matches[-1], time_fmt)
- else:
- raise ValueError("Unknown filename scheme, can't determine file time.")
-
-
-def shapes_generator(input_files, prev_shapefile, shared_bbox=False):
- prev_shapes = previous_shapes_generator(prev_shapefile)
- default_shape = (None, None)
- location, shape_info = next(prev_shapes, default_shape)
- while location is not None:
- if location != input_files[0]:
- # the old shapefile includes a file that we no longer want
- # keep going through the existing shapes until we find one we
- # do want
- yield 'removed', location, shape_info
- location, shape_info = next(prev_shapes, default_shape)
- continue
- yield 'existing', location, shape_info
- location, shape_info = next(prev_shapes, default_shape)
- input_files = input_files[1:]
-
- # from here on out these are all newly added files
- bbox_cache = None
- for f in input_files:
- try:
- dt = get_file_time(f)
- except ValueError:
- logger.error(f"Can't time for file {f}")
- continue
-
- logger.info(f"Indexing {f} {dt.isoformat()}")
- if bbox_cache is None or not shared_bbox:
- with rasterio.open(f) as src:
- bbox_cache = mapping(box(*src.bounds))
-
- shape_info = {
- 'geometry': bbox_cache,
- 'properties': {
- 'location': f,
- 'time': dt.isoformat(),
- },
- }
- yield 'added', f, shape_info
-
-
-def index(input_files, output_shapefile, shared_bbox=False):
- """Create shapefile for location and times of provided geotiffs or tileDB arrays.
-
- Note: All layers to be included in the shapefile must be provided all at
- once. Repeated calls to this function will overwrite existing
- shapefile information.
-
- Returns:
- (times_removed, times_added): Two lists of times removed by this index
- being updated and times added by this index being updated.
-
- """
- import tempfile
- out_dir, shp_fn = os.path.split(output_shapefile)
- tmp_dir = tempfile.mkdtemp("_tile_index")
- tmp_shapefile = os.path.join(tmp_dir, shp_fn)
- times_added = []
- times_removed = []
- with fiona.open(tmp_shapefile, 'w', driver='ESRI Shapefile',
- schema=temporal_schema) as output:
- for shape_state, location, shape_info in shapes_generator(input_files, output_shapefile, shared_bbox=shared_bbox):
- if shape_state == 'removed':
- times_removed.append(shape_info['properties']['time'])
- continue
- if shape_state == 'added':
- times_added.append(shape_info['properties']['time'])
- output.write(shape_info)
-
- # move the shapefile contents to the final destination
- for fn in os.listdir(tmp_dir):
- shutil.move(os.path.join(tmp_dir, fn), os.path.join(out_dir, fn))
- # we don't need the temporary directory anymore
- shutil.rmtree(tmp_dir, ignore_errors=True)
-
- return times_removed, times_added
-
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument('-d', '--directory',
- help='Single directory to search for .tif files or TileDB array directories.')
- parser.add_argument('-o', '--output', default='img_index.shp')
- parser.add_argument('input_files', nargs='*',
- help='TileDB directories or GeoTIFF files to ingest.')
- args = parser.parse_args()
-
- logging.basicConfig(
- level=logging.INFO,
- format="%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s"
- )
-
- if args.input_files:
- logger.info("Using input files...")
- it = args.input_files
- else:
- src_dir = args.directory
- files = glob.glob(os.path.join(src_dir, '*.tif'))
- folders = [os.path.join(src_dir, o) for o in os.listdir(src_dir) if os.path.isdir(os.path.join(src_dir, o))]
- # simple toggle between indexing tiff files or tiledb arrays
- if len(files) > 0:
- it = files
- else:
- it = folders
-
- if not it:
- raise ValueError("No valid inputs provided.")
-
- if args.directory:
- logger.info('Indexer starting')
- index(it, args.output)
-
-
-if __name__ == "__main__":
- sys.exit(main())
--
GitLab