Skip to content
Snippets Groups Projects
Verified Commit 1d8c297e authored by David Hoese's avatar David Hoese
Browse files

Update to use YAML configuration file for products and add support for ABI L2

parent cbd57f77
No related branches found
No related tags found
1 merge request!5Add initial attempt at S3 geotiff reading
Pipeline #39465 failed
...@@ -258,6 +258,7 @@ curl_layer_times() { ...@@ -258,6 +258,7 @@ curl_layer_times() {
curl_empty_tile() { curl_empty_tile() {
debug "Starting curl basic mapfile request for non-existent tile..." debug "Starting curl basic mapfile request for non-existent tile..."
# NOTE: The time doesn't actually exist and no image data is available. A blank image should be returned # NOTE: The time doesn't actually exist and no image data is available. A blank image should be returned
# docker exec -it test bash
curl --fail -sS -o "empty_tile.png" "http://localhost:8888/wms/g16/abi/radf/l1b?VERSION=1.1.1&REQUEST=GetMap&SERVICE=WMS&STYLES=&BBOX=-2500000%2c-2500000%2c2500000%2c2500000&WIDTH=256&HEIGHT=256&FORMAT=rgba&SRS=EPSG%3a930916&LAYERS=C01&TIME=2022-04-20T16:00:12Z" >/dev/null curl --fail -sS -o "empty_tile.png" "http://localhost:8888/wms/g16/abi/radf/l1b?VERSION=1.1.1&REQUEST=GetMap&SERVICE=WMS&STYLES=&BBOX=-2500000%2c-2500000%2c2500000%2c2500000&WIDTH=256&HEIGHT=256&FORMAT=rgba&SRS=EPSG%3a930916&LAYERS=C01&TIME=2022-04-20T16:00:12Z" >/dev/null
check_image_content "empty_tile.png" 1 check_image_content "empty_tile.png" 1
} }
......
...@@ -169,9 +169,11 @@ RUN ln -s /usr/local/bin/mapserv /usr/lib/cgi-bin/mapserv && \ ...@@ -169,9 +169,11 @@ RUN ln -s /usr/local/bin/mapserv /usr/lib/cgi-bin/mapserv && \
chown -h ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/* chown -h ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/*
RUN pip3 install jinja2 psycopg2 && rm -r /root/.cache/pip RUN pip3 install jinja2 psycopg2 && rm -r /root/.cache/pip
COPY render.py run.sh abi_l1b_template.map /work/ COPY render.py run.sh abi_l1b_template.map /work/
COPY mapfiles/ /work/mapfiles/ COPY mapfiles/ /work/mapfiles/
COPY html/ /var/www/html/ COPY html/ /var/www/html/
COPY default_settings.yaml /work/geosphere_settings.yaml
# Add our own custom EPSG codes (HACK) # Add our own custom EPSG codes (HACK)
# GOES-16 ABI Full Disk = EPSG:930916 # GOES-16 ABI Full Disk = EPSG:930916
......
MAP MAP
NAME "{{ platform_long_name }} ABI {{ sector_long_name }} L1b Layers" NAME "{{ satellite_info["long_name"] }} ABI {{ sector_info["long_name"] }} L1b Layers"
STATUS ON STATUS ON
IMAGETYPE rgb IMAGETYPE rgb
SIZE 256 256 SIZE 256 256
...@@ -38,24 +38,24 @@ MAP ...@@ -38,24 +38,24 @@ MAP
INCLUDE '/work/mapfiles/output_formats.map' INCLUDE '/work/mapfiles/output_formats.map'
{% for product in products %} {% for product_info in products %}
# {{ product.replace('_', ' ').title() }} # # {{ product_info["short_name"] }} #
LAYER LAYER
NAME "{{ product }}_shapefile_index" NAME "{{ product_info["identifier"] }}_shapefile_index"
TYPE TILEINDEX TYPE TILEINDEX
DATA "{{ layer_base_dir }}/{{ platform }}/abi/{{ sector }}/{{ product }}/{{ product }}" DATA "{{ layer_base_dir }}/{{ satellite_info["identifier"] }}/abi/{{ sector_info["identifier"] }}/{{ product_info["short_name"] }}/{{ product_info["short_name"] }}"
END END
LAYER LAYER
NAME "{{ product }}_postgres_index" NAME "{{ product_info["identifier"] }}_postgres_index"
TYPE POLYGON TYPE POLYGON
DATA "{{ postgis_geom_column }} from {{ platform }}_abi_{{ sector }}_l1b_{{ product.lower() }} using SRID={{ epsg_code }} using unique gid" DATA "{{ postgis_geom_column }} from {{ satellite_info["identifier"] }}_abi_{{ sector_info["identifier"] }}_{{ product_info["processing_level"] }}_{{ product_info["identifier"] }} using SRID={{ epsg_code }} using unique gid"
CONNECTIONTYPE postgis CONNECTIONTYPE postgis
CONNECTION "{{ postgis_connection_params }}" CONNECTION "{{ postgis_connection_params }}"
PROCESSING "CLOSE_CONNECTION=DEFER" PROCESSING "CLOSE_CONNECTION=DEFER"
METADATA METADATA
"wms_title" "{{ platform_long_name }} ABI {{ product.replace('_', ' ').title() }} Time Index" "wms_title" "{{ satellite_info["long_name"] }} ABI {{ product_info["long_name"] }} Time Index"
"wms srs" "EPSG:{{ epsg_code }}" "wms srs" "EPSG:{{ epsg_code }}"
"wms_extent" "-180 -90 180 90" "wms_extent" "-180 -90 180 90"
"wms_timeextent" "2017-01-01/2040-12-31" "wms_timeextent" "2017-01-01/2040-12-31"
...@@ -67,14 +67,14 @@ MAP ...@@ -67,14 +67,14 @@ MAP
END END
LAYER LAYER
NAME "{{ product }}" NAME "{{ product_info["identifier"] }}"
TYPE RASTER TYPE RASTER
{% if postgis_connection_params %} {% if postgis_connection_params %}
TILEITEM "location" TILEITEM "location"
TILEINDEX "{{ product }}_postgres_index" TILEINDEX "{{ product_info["identifier"] }}_postgres_index"
{% else %} {% else %}
TILEITEM "location" TILEITEM "location"
TILEINDEX "{{ product }}_shapefile_index" TILEINDEX "{{ product_info["identifier"] }}_shapefile_index"
{% endif %} {% endif %}
# Comment below to default to transparency # Comment below to default to transparency
...@@ -83,7 +83,7 @@ MAP ...@@ -83,7 +83,7 @@ MAP
# Used to keep FastCGI connections open longer # Used to keep FastCGI connections open longer
PROCESSING "CLOSE_CONNECTION=DEFER" PROCESSING "CLOSE_CONNECTION=DEFER"
METADATA METADATA
"wms_title" "{{ platform_long_name }} ABI {{ product.replace('_', ' ').title() }}" "wms_title" "{{ satellite_info["long_name"] }} ABI {{ product_info["long_name"] }}"
"wms srs" "EPSG:{{ epsg_code }}" "wms srs" "EPSG:{{ epsg_code }}"
"wms_extent" "-180 -90 180 90" "wms_extent" "-180 -90 180 90"
"wms_timeextent" "2017-01-01/2040-12-31" "wms_timeextent" "2017-01-01/2040-12-31"
......
This diff is collapsed.
...@@ -5,48 +5,24 @@ We do this rendering when the container is started to allow for complete ...@@ -5,48 +5,24 @@ We do this rendering when the container is started to allow for complete
control per mapserver instance (even though there is usually only one). control per mapserver instance (even though there is usually only one).
""" """
import itertools
import os import os
import sys import sys
import jinja2 import jinja2
import yaml
sectors = "radm1 radm2 radc radf" sectors_filter = os.getenv("WMS_SECTORS", "")
sectors = os.getenv("WMS_SECTORS", sectors).split(" ") if sectors_filter:
products = ["C{:02d}".format(x) for x in range(1, 17)] + ["true_color"] sectors_filter = sectors_filter.split(" ")
products = os.getenv("WMS_PRODUCTS", " ".join(products)).split(" ") products_filter = os.getenv("WMS_PRODUCTS", "")
platforms = "g16 g17 g18" if products_filter:
platforms = os.getenv("WMS_PLATFORMS", platforms).split(" ") products_filter = products_filter.split(" ")
satellites_filter = os.getenv("WMS_PLATFORMS", "")
if satellites_filter:
satellites_filter = satellites_filter.split(" ")
layer_base_dir = os.environ['LAYER_BASE_DIR'] layer_base_dir = os.environ['LAYER_BASE_DIR']
SECTOR_LONG_NAMES = {
'radm1': 'Mesoscale 1',
'radm2': 'Mesoscale 2',
'radc': 'CONUS',
'radf': 'Full disk',
}
PLATFORM_LONG_NAMES = {
'g16': 'GOES-16',
'g17': 'GOES-17',
'g18': 'GOES-18',
}
PLATFORM_EPSGS = {
'g16': 'EPSG:930916',
'g17': 'EPSG:930917',
'g18': 'EPSG:930917',
}
PLATFORM_POS_NAMES = {
'g16': 'GOES-EAST',
'g17': 'GOES-WEST',
'g18': 'GOES-WEST',
}
def _get_connect_str(): def _get_connect_str():
pw_filename = "__POSTGRES_PASSWORD_FILE__" pw_filename = "__POSTGRES_PASSWORD_FILE__"
connect_str = "host=__POSTGRES_HOST__ " \ connect_str = "host=__POSTGRES_HOST__ " \
...@@ -68,35 +44,77 @@ def _get_connect_str(): ...@@ -68,35 +44,77 @@ def _get_connect_str():
def main(): def main():
import argparse import argparse
parser = argparse.ArgumentParser(description="Render mapfile template to complete mapfile config.") parser = argparse.ArgumentParser(description="Render mapfile template to complete mapfile config.")
parser.add_argument("product_yaml_config",
help="GeoSphere YAML configuration file. Matches the YAML used by GeoSphere API, "
"but not all the information may be used.")
parser.add_argument("template_fn", parser.add_argument("template_fn",
help="Template XML file to render") help="Template XML file to render")
parser.add_argument('output_pattern', parser.add_argument('output_pattern',
help='Format string for each output mapfiles.') help='Format string for each output mapfiles. Should be per-sector per-processing level.')
args = parser.parse_args() args = parser.parse_args()
# Check if this script was properly sed'd with database information
connect_params = _get_connect_str() if "__" not in "__POSTGRES_HOST__" else "" connect_params = _get_connect_str() if "__" not in "__POSTGRES_HOST__" else ""
for platform in platforms: with open(args.product_yaml_config) as product_yaml:
for sector in sectors: geosphere_config = yaml.load(product_yaml, Loader=yaml.SafeLoader)
tmpl_vars = { for template_info in template_info_from_config_and_db_info(geosphere_config, connect_params):
'sector': sector, output_fn = args.output_pattern.format(**template_info)
'platform': platform, render_mapfile_template_with_template_info(args.template_fn, output_fn, template_info)
'products': products,
'layer_base_dir': layer_base_dir,
'platform_long_name': PLATFORM_LONG_NAMES[platform], def template_info_from_config_and_db_info(geosphere_config: dict, connect_params: str):
'sector_long_name': SECTOR_LONG_NAMES[sector], for satellite_info in geosphere_config["satellites"]:
'projection': "init=" + PLATFORM_EPSGS[platform].lower(), if satellites_filter and satellite_info["identifier"] not in satellites_filter:
'epsg_code': PLATFORM_EPSGS[platform].replace('EPSG:', ''), continue
'PLATFORM_EPSGS': PLATFORM_EPSGS,
'PLATFORM_POS_NAMES': PLATFORM_POS_NAMES, for instrument_info in satellite_info["instruments"]:
'postgis_connection_params': connect_params, for sector_info in instrument_info["sectors"]:
'postgis_geom_column': 'bbox_geometry', if sectors_filter and sector_info["identifier"] not in sectors_filter:
} continue
output_fn = args.output_pattern.format(**tmpl_vars)
projection_info = projection_info_for_id(geosphere_config["projections"], sector_info["projection_id"])
with open(args.template_fn, 'r') as tmpl_file, open(output_fn, 'w') as mapfile: product_infos_groups = per_processing_level_product_infos(sector_info["products"])
template = jinja2.Template(tmpl_file.read()) for processing_level, product_infos_for_plevel in product_infos_groups:
mapfile.write(template.render(tmpl_vars, env=os.environ)) # print(len(sector_info["products"]))
# product_infos_for_plevel = list(product_infos_for_plevel)
# print(processing_level, product_infos_for_plevel)
product_infos = filter_products(product_infos_for_plevel, products_filter)
template_info = {
'sector_info': sector_info,
'satellite_info': satellite_info,
'instrument_info': instrument_info,
'processing_level': processing_level,
'products': product_infos,
'layer_base_dir': layer_base_dir,
'projection': f"init=epsg:{projection_info['epsg_code']}",
'epsg_code': str(projection_info["epsg_code"]),
'postgis_connection_params': connect_params,
'postgis_geom_column': 'bbox_geometry',
}
yield template_info
def projection_info_for_id(projection_infos, projection_id):
return [proj_info for proj_info in projection_infos if proj_info["identifier"] == projection_id][0]
def per_processing_level_product_infos(sector_product_infos):
"""Split a list of product information dictionaries into groups by ``processing_level``."""
yield from itertools.groupby(sector_product_infos, key=lambda product_info: product_info["processing_level"])
def filter_products(product_infos, product_identifiers_to_keep):
if not product_identifiers_to_keep:
return product_infos
return [product_info for product_info in product_infos if
product_info["identifier"] in product_identifiers_to_keep]
def render_mapfile_template_with_template_info(template_fn, output_fn, template_info):
with open(template_fn, 'r') as tmpl_file, open(output_fn, 'w') as mapfile:
template = jinja2.Template(tmpl_file.read())
mapfile.write(template.render(template_info, env=os.environ))
if __name__ == "__main__": if __name__ == "__main__":
......
...@@ -15,8 +15,6 @@ export LAYER_BASE_DIR=${LAYER_BASE_DIR:-"/data/tiles"} ...@@ -15,8 +15,6 @@ export LAYER_BASE_DIR=${LAYER_BASE_DIR:-"/data/tiles"}
# Update the mapcache.xml file with the real host and port name # Update the mapcache.xml file with the real host and port name
mf_tmpl="/work/abi_l1b_template.map" mf_tmpl="/work/abi_l1b_template.map"
export WMS_PLATFORMS=${WMS_PLATFORMS:-"g16 g17 g18"}
export WMS_SECTORS=${WMS_SECTORS:-"radm1 radm2 radc radf"}
sed -i "s:__LAYER_BASE_DIR__:$LAYER_BASE_DIR:g" /etc/apache2/sites-available/cspp_geo.conf sed -i "s:__LAYER_BASE_DIR__:$LAYER_BASE_DIR:g" /etc/apache2/sites-available/cspp_geo.conf
if [[ ${POSTGRES_HOST} != "" ]]; then if [[ ${POSTGRES_HOST} != "" ]]; then
...@@ -27,7 +25,8 @@ else ...@@ -27,7 +25,8 @@ else
sed -i "s:wms_times_shapes:wms_times:g" /etc/apache2/sites-available/cspp_geo.conf sed -i "s:wms_times_shapes:wms_times:g" /etc/apache2/sites-available/cspp_geo.conf
fi fi
python3 /work/render.py $mf_tmpl "/work/mapfiles/{platform}_abi_{sector}_l1b.map" export GEOSPHERE_CONFIG=${GEOSPHERE_CONFIG:-"/work/geosphere_settings.yaml"}
python3 /work/render.py ${GEOSPHERE_CONFIG} $mf_tmpl "/work/mapfiles/{satellite_info[identifier]}_{instrument_info[identifier]}_{sector_info[identifier]}_{processing_level}.map"
/usr/sbin/apache2ctl configtest /usr/sbin/apache2ctl configtest
......
...@@ -37,8 +37,6 @@ ...@@ -37,8 +37,6 @@
# FCGI version of the URL # FCGI version of the URL
RewriteRule "^/wms/([^/]+)/([^/]+)/([^/]+)/([^/]+)?(.*)" "/cgi-bin/mapserv.fcgi?map=/work/mapfiles/$1_$2_$3_$4.map&$5" [PT,QSA] RewriteRule "^/wms/([^/]+)/([^/]+)/([^/]+)/([^/]+)?(.*)" "/cgi-bin/mapserv.fcgi?map=/work/mapfiles/$1_$2_$3_$4.map&$5" [PT,QSA]
RewriteRule "^/wms/([^/]+)?(.*)" "/cgi-bin/mapserv.fcgi?map=/work/mapfiles/$1.map&$2" [PT,QSA] RewriteRule "^/wms/([^/]+)?(.*)" "/cgi-bin/mapserv.fcgi?map=/work/mapfiles/$1.map&$2" [PT,QSA]
#RewriteRule "^/fwms/([^/]+)/([^/]+)/([^/]+)/([^/]+)?(.*)" "/cgi-bin/mapserv.fcgi?map=/work/mapfiles/$1_$2_$3_$4.map&$5" [PT,QSA]
#RewriteRule "^/fwms/([^/]+)?(.*)" "/cgi-bin/mapserv.fcgi?map=/work/mapfiles/$1.map&$2" [PT,QSA]
# /wms_times/g16/abi/radf/true_color # /wms_times/g16/abi/radf/true_color
# LAYER_BASE_DIR defaults to /data/tiles # LAYER_BASE_DIR defaults to /data/tiles
RewriteRule "^/wms_times_postgres/([^/]+)/([^/]+)/([^/]+)/([^/]+)" "/cgi-bin/layer_times_postgres.py?layer=$1_$2_$3_l1b_${lc:$4}" [PT,QSA] RewriteRule "^/wms_times_postgres/([^/]+)/([^/]+)/([^/]+)/([^/]+)" "/cgi-bin/layer_times_postgres.py?layer=$1_$2_$3_l1b_${lc:$4}" [PT,QSA]
...@@ -46,8 +44,6 @@ ...@@ -46,8 +44,6 @@
# Regular CGI version of the URL (put last since it should be used less often) # Regular CGI version of the URL (put last since it should be used less often)
RewriteRule "^/wms_cgi/([^/]+)/([^/]+)/([^/]+)/([^/]+)?(.*)" "/cgi-bin/mapserv?map=/work/mapfiles/$1_$2_$3_$4.map&$5" [PT,QSA] RewriteRule "^/wms_cgi/([^/]+)/([^/]+)/([^/]+)/([^/]+)?(.*)" "/cgi-bin/mapserv?map=/work/mapfiles/$1_$2_$3_$4.map&$5" [PT,QSA]
RewriteRule "^/wms_cgi/([^/]+)?(.*)" "/cgi-bin/mapserv?map=/work/mapfiles/$1.map&$2" [PT,QSA] RewriteRule "^/wms_cgi/([^/]+)?(.*)" "/cgi-bin/mapserv?map=/work/mapfiles/$1.map&$2" [PT,QSA]
#RewriteRule "^/wms/([^/]+)/([^/]+)/([^/]+)/([^/]+)?(.*)" "/cgi-bin/mapserv?map=/work/mapfiles/$1_$2_$3_$4.map&$5" [PT,QSA]
#RewriteRule "^/wms/([^/]+)?(.*)" "/cgi-bin/mapserv?map=/work/mapfiles/$1.map&$2" [PT,QSA]
# handler should be added by the enabled module # handler should be added by the enabled module
#AddHandler fcgid-script fcgi #AddHandler fcgid-script fcgi
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment