Update to use YAML configuration file for products and add support for ABI L2

ci/test_mapserver_image.sh

@@ -258,6 +258,7 @@ curl_layer_times() {
 curl_empty_tile() {
     debug "Starting curl basic mapfile request for non-existent tile..."
     # NOTE: The time doesn't actually exist and no image data is available. A blank image should be returned
+#    docker exec -it test bash
     curl --fail -sS -o "empty_tile.png" "http://localhost:8888/wms/g16/abi/radf/l1b?VERSION=1.1.1&REQUEST=GetMap&SERVICE=WMS&STYLES=&BBOX=-2500000%2c-2500000%2c2500000%2c2500000&WIDTH=256&HEIGHT=256&FORMAT=rgba&SRS=EPSG%3a930916&LAYERS=C01&TIME=2022-04-20T16:00:12Z" >/dev/null
     check_image_content "empty_tile.png" 1
 }

mapserver/Dockerfile

@@ -169,9 +169,11 @@ RUN ln -s /usr/local/bin/mapserv /usr/lib/cgi-bin/mapserv && \
     chown -h ${APACHE_RUN_USER}:${APACHE_RUN_GROUP} /usr/lib/cgi-bin/*
 
 RUN pip3 install jinja2 psycopg2 && rm -r /root/.cache/pip
+
 COPY render.py run.sh abi_l1b_template.map /work/
 COPY mapfiles/ /work/mapfiles/
 COPY html/ /var/www/html/
+COPY default_settings.yaml /work/geosphere_settings.yaml
 
 # Add our own custom EPSG codes (HACK)
 # GOES-16 ABI Full Disk = EPSG:930916

mapserver/abi_l1b_template.map

 MAP
-  NAME "{{ platform_long_name }} ABI {{ sector_long_name }} L1b Layers"
+  NAME "{{ satellite_info["long_name"] }} ABI {{ sector_info["long_name"] }} L1b Layers"
   STATUS ON
   IMAGETYPE rgb
   SIZE 256 256
@@ -38,24 +38,24 @@ MAP
 
   INCLUDE '/work/mapfiles/output_formats.map'
 
-  {% for product in products %}
-  # {{ product.replace('_', ' ').title() }} #
+  {% for product_info in products %}
+  # {{ product_info["short_name"] }} #
 
   LAYER
-    NAME "{{ product }}_shapefile_index"
+    NAME "{{ product_info["identifier"] }}_shapefile_index"
     TYPE TILEINDEX
-    DATA "{{ layer_base_dir }}/{{ platform }}/abi/{{ sector }}/{{ product }}/{{ product }}"
+    DATA "{{ layer_base_dir }}/{{ satellite_info["identifier"] }}/abi/{{ sector_info["identifier"] }}/{{ product_info["short_name"] }}/{{ product_info["short_name"] }}"
   END
 
   LAYER
-    NAME "{{ product }}_postgres_index"
+    NAME "{{ product_info["identifier"] }}_postgres_index"
     TYPE POLYGON
-    DATA "{{ postgis_geom_column }} from {{ platform }}_abi_{{ sector }}_l1b_{{ product.lower() }} using SRID={{ epsg_code }} using unique gid"
+    DATA "{{ postgis_geom_column }} from {{ satellite_info["identifier"] }}_abi_{{ sector_info["identifier"] }}_{{ product_info["processing_level"] }}_{{ product_info["identifier"] }} using SRID={{ epsg_code }} using unique gid"
     CONNECTIONTYPE postgis
     CONNECTION "{{ postgis_connection_params }}"
     PROCESSING "CLOSE_CONNECTION=DEFER"
     METADATA
-      "wms_title" "{{ platform_long_name }} ABI {{ product.replace('_', ' ').title() }} Time Index"
+      "wms_title" "{{ satellite_info["long_name"] }} ABI {{ product_info["long_name"] }} Time Index"
       "wms srs" "EPSG:{{ epsg_code }}"
       "wms_extent" "-180 -90 180 90"
       "wms_timeextent" "2017-01-01/2040-12-31"
@@ -67,14 +67,14 @@ MAP
   END
 
   LAYER
-    NAME "{{ product }}"
+    NAME "{{ product_info["identifier"] }}"
     TYPE RASTER
     {% if postgis_connection_params %}
     TILEITEM "location"
-    TILEINDEX "{{ product }}_postgres_index"
+    TILEINDEX "{{ product_info["identifier"] }}_postgres_index"
     {% else %}
     TILEITEM "location"
-    TILEINDEX "{{ product }}_shapefile_index"
+    TILEINDEX "{{ product_info["identifier"] }}_shapefile_index"
     {% endif %}
 
     # Comment below to default to transparency
@@ -83,7 +83,7 @@ MAP
     # Used to keep FastCGI connections open longer
     PROCESSING "CLOSE_CONNECTION=DEFER"
     METADATA
-      "wms_title" "{{ platform_long_name }} ABI {{ product.replace('_', ' ').title() }}"
+      "wms_title" "{{ satellite_info["long_name"] }} ABI {{ product_info["long_name"] }}"
       "wms srs" "EPSG:{{ epsg_code }}"
       "wms_extent" "-180 -90 180 90"
       "wms_timeextent" "2017-01-01/2040-12-31"

mapserver/render.py

@@ -5,48 +5,24 @@ We do this rendering when the container is started to allow for complete
 control per mapserver instance (even though there is usually only one).
 
 """
-
+import itertools
 import os
 import sys
 import jinja2
-
-
-sectors = "radm1 radm2 radc radf"
-sectors = os.getenv("WMS_SECTORS", sectors).split(" ")
-products = ["C{:02d}".format(x) for x in range(1, 17)] + ["true_color"]
-products = os.getenv("WMS_PRODUCTS", " ".join(products)).split(" ")
-platforms = "g16 g17 g18"
-platforms = os.getenv("WMS_PLATFORMS", platforms).split(" ")
-
+import yaml
+
+sectors_filter = os.getenv("WMS_SECTORS", "")
+if sectors_filter:
+    sectors_filter = sectors_filter.split(" ")
+products_filter = os.getenv("WMS_PRODUCTS", "")
+if products_filter:
+    products_filter = products_filter.split(" ")
+satellites_filter = os.getenv("WMS_PLATFORMS", "")
+if satellites_filter:
+    satellites_filter = satellites_filter.split(" ")
 layer_base_dir = os.environ['LAYER_BASE_DIR']
 
 
-SECTOR_LONG_NAMES = {
-    'radm1': 'Mesoscale 1',
-    'radm2': 'Mesoscale 2',
-    'radc': 'CONUS',
-    'radf': 'Full disk',
-}
-
-PLATFORM_LONG_NAMES = {
-    'g16': 'GOES-16',
-    'g17': 'GOES-17',
-    'g18': 'GOES-18',
-}
-
-PLATFORM_EPSGS = {
-    'g16': 'EPSG:930916',
-    'g17': 'EPSG:930917',
-    'g18': 'EPSG:930917',
-}
-
-PLATFORM_POS_NAMES = {
-    'g16': 'GOES-EAST',
-    'g17': 'GOES-WEST',
-    'g18': 'GOES-WEST',
-}
-
-
 def _get_connect_str():
     pw_filename = "__POSTGRES_PASSWORD_FILE__"
     connect_str = "host=__POSTGRES_HOST__ " \
@@ -68,35 +44,77 @@ def _get_connect_str():
 def main():
     import argparse
     parser = argparse.ArgumentParser(description="Render mapfile template to complete mapfile config.")
+    parser.add_argument("product_yaml_config",
+                        help="GeoSphere YAML configuration file. Matches the YAML used by GeoSphere API, "
+                             "but not all the information may be used.")
     parser.add_argument("template_fn",
                         help="Template XML file to render")
     parser.add_argument('output_pattern',
-                        help='Format string for each output mapfiles.')
+                        help='Format string for each output mapfiles. Should be per-sector per-processing level.')
     args = parser.parse_args()
 
+    # Check if this script was properly sed'd with database information
     connect_params = _get_connect_str() if "__" not in "__POSTGRES_HOST__" else ""
 
-    for platform in platforms:
-        for sector in sectors:
-            tmpl_vars = {
-                'sector': sector,
-                'platform': platform,
-                'products': products,
-                'layer_base_dir': layer_base_dir,
-                'platform_long_name': PLATFORM_LONG_NAMES[platform],
-                'sector_long_name': SECTOR_LONG_NAMES[sector],
-                'projection': "init=" + PLATFORM_EPSGS[platform].lower(),
-                'epsg_code': PLATFORM_EPSGS[platform].replace('EPSG:', ''),
-                'PLATFORM_EPSGS': PLATFORM_EPSGS,
-                'PLATFORM_POS_NAMES': PLATFORM_POS_NAMES,
-                'postgis_connection_params': connect_params,
-                'postgis_geom_column': 'bbox_geometry',
-            }
-            output_fn = args.output_pattern.format(**tmpl_vars)
-
-            with open(args.template_fn, 'r') as tmpl_file, open(output_fn, 'w') as mapfile:
-                template = jinja2.Template(tmpl_file.read())
-                mapfile.write(template.render(tmpl_vars, env=os.environ))
+    with open(args.product_yaml_config) as product_yaml:
+        geosphere_config = yaml.load(product_yaml, Loader=yaml.SafeLoader)
+    for template_info in template_info_from_config_and_db_info(geosphere_config, connect_params):
+        output_fn = args.output_pattern.format(**template_info)
+        render_mapfile_template_with_template_info(args.template_fn, output_fn, template_info)
+
+
+def template_info_from_config_and_db_info(geosphere_config: dict, connect_params: str):
+    for satellite_info in geosphere_config["satellites"]:
+        if satellites_filter and satellite_info["identifier"] not in satellites_filter:
+            continue
+
+        for instrument_info in satellite_info["instruments"]:
+            for sector_info in instrument_info["sectors"]:
+                if sectors_filter and sector_info["identifier"] not in sectors_filter:
+                    continue
+
+                projection_info = projection_info_for_id(geosphere_config["projections"], sector_info["projection_id"])
+                product_infos_groups = per_processing_level_product_infos(sector_info["products"])
+                for processing_level, product_infos_for_plevel in product_infos_groups:
+                    # print(len(sector_info["products"]))
+                    # product_infos_for_plevel = list(product_infos_for_plevel)
+                    # print(processing_level, product_infos_for_plevel)
+                    product_infos = filter_products(product_infos_for_plevel, products_filter)
+                    template_info = {
+                        'sector_info': sector_info,
+                        'satellite_info': satellite_info,
+                        'instrument_info': instrument_info,
+                        'processing_level': processing_level,
+                        'products': product_infos,
+                        'layer_base_dir': layer_base_dir,
+                        'projection': f"init=epsg:{projection_info['epsg_code']}",
+                        'epsg_code': str(projection_info["epsg_code"]),
+                        'postgis_connection_params': connect_params,
+                        'postgis_geom_column': 'bbox_geometry',
+                    }
+                    yield template_info
+
+
+def projection_info_for_id(projection_infos, projection_id):
+    return [proj_info for proj_info in projection_infos if proj_info["identifier"] == projection_id][0]
+
+
+def per_processing_level_product_infos(sector_product_infos):
+    """Split a list of product information dictionaries into groups by ``processing_level``."""
+    yield from itertools.groupby(sector_product_infos, key=lambda product_info: product_info["processing_level"])
+
+
+def filter_products(product_infos, product_identifiers_to_keep):
+    if not product_identifiers_to_keep:
+        return product_infos
+    return [product_info for product_info in product_infos if
+            product_info["identifier"] in product_identifiers_to_keep]
+
+
+def render_mapfile_template_with_template_info(template_fn, output_fn, template_info):
+    with open(template_fn, 'r') as tmpl_file, open(output_fn, 'w') as mapfile:
+        template = jinja2.Template(tmpl_file.read())
+        mapfile.write(template.render(template_info, env=os.environ))
 
 
 if __name__ == "__main__":

mapserver/run.sh

@@ -15,8 +15,6 @@ export LAYER_BASE_DIR=${LAYER_BASE_DIR:-"/data/tiles"}
 
 # Update the mapcache.xml file with the real host and port name
 mf_tmpl="/work/abi_l1b_template.map"
-export WMS_PLATFORMS=${WMS_PLATFORMS:-"g16 g17 g18"}
-export WMS_SECTORS=${WMS_SECTORS:-"radm1 radm2 radc radf"}
 sed -i "s:__LAYER_BASE_DIR__:$LAYER_BASE_DIR:g" /etc/apache2/sites-available/cspp_geo.conf
 
 if [[ ${POSTGRES_HOST} != "" ]]; then
@@ -27,7 +25,8 @@ else
     sed -i "s:wms_times_shapes:wms_times:g" /etc/apache2/sites-available/cspp_geo.conf
 fi
 
-python3 /work/render.py $mf_tmpl "/work/mapfiles/{platform}_abi_{sector}_l1b.map"
+export GEOSPHERE_CONFIG=${GEOSPHERE_CONFIG:-"/work/geosphere_settings.yaml"}
+python3 /work/render.py ${GEOSPHERE_CONFIG} $mf_tmpl "/work/mapfiles/{satellite_info[identifier]}_{instrument_info[identifier]}_{sector_info[identifier]}_{processing_level}.map"
 
 /usr/sbin/apache2ctl configtest
 

mapserver/site-conf

@@ -37,8 +37,6 @@
         # FCGI version of the URL
         RewriteRule   "^/wms/([^/]+)/([^/]+)/([^/]+)/([^/]+)?(.*)" "/cgi-bin/mapserv.fcgi?map=/work/mapfiles/$1_$2_$3_$4.map&$5" [PT,QSA]
         RewriteRule   "^/wms/([^/]+)?(.*)" "/cgi-bin/mapserv.fcgi?map=/work/mapfiles/$1.map&$2" [PT,QSA]
-        #RewriteRule   "^/fwms/([^/]+)/([^/]+)/([^/]+)/([^/]+)?(.*)" "/cgi-bin/mapserv.fcgi?map=/work/mapfiles/$1_$2_$3_$4.map&$5" [PT,QSA]
-        #RewriteRule   "^/fwms/([^/]+)?(.*)" "/cgi-bin/mapserv.fcgi?map=/work/mapfiles/$1.map&$2" [PT,QSA]
         # /wms_times/g16/abi/radf/true_color
         # LAYER_BASE_DIR defaults to /data/tiles
         RewriteRule   "^/wms_times_postgres/([^/]+)/([^/]+)/([^/]+)/([^/]+)" "/cgi-bin/layer_times_postgres.py?layer=$1_$2_$3_l1b_${lc:$4}" [PT,QSA]
@@ -46,8 +44,6 @@
         # Regular CGI version of the URL (put last since it should be used less often)
         RewriteRule   "^/wms_cgi/([^/]+)/([^/]+)/([^/]+)/([^/]+)?(.*)" "/cgi-bin/mapserv?map=/work/mapfiles/$1_$2_$3_$4.map&$5" [PT,QSA]
         RewriteRule   "^/wms_cgi/([^/]+)?(.*)" "/cgi-bin/mapserv?map=/work/mapfiles/$1.map&$2" [PT,QSA]
-        #RewriteRule   "^/wms/([^/]+)/([^/]+)/([^/]+)/([^/]+)?(.*)" "/cgi-bin/mapserv?map=/work/mapfiles/$1_$2_$3_$4.map&$5" [PT,QSA]
-        #RewriteRule   "^/wms/([^/]+)?(.*)" "/cgi-bin/mapserv?map=/work/mapfiles/$1.map&$2" [PT,QSA]
 
         # handler should be added by the enabled module
         #AddHandler fcgid-script fcgi