Merge tag 'v20180216'

optional attributes and filename fixes

build.sh

+#!/bin/bash
+# docker run --rm -it -v $PWD:/docker -w /docker centos6dev /bin/bash
+# yum install xz git
+# export AXITOOLS_HOME=$PWD/test_install
+# export HIMAWARI_TAG=develop
+# export GOESR_TAG=develop
+# bash $HOME/Repos/git/himawari/build.sh
+#
+# example go-script
+# #!/bin/bash
+# version="v$(date '+%Y%m%d')"
+# export AXITOOLS_HOME=$PWD/axi-tools-${version}
+# test -f build.sh && rm build.sh
+# wget https://gitlab.ssec.wisc.edu/rayg/himawari/raw/develop/build.sh
+# bash build.sh 
+
+oops() {
+    echo "ERROR: $*"
+    exit 1
+}
+
+# find where we are
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
+BASE="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+test -n "$AXITOOLS_HOME" || oops "Please set AXITOOLS_HOME destination variable, e.g. export AXITOOLS_HOME=$PWD/axi-tools-v20170831"
+test -n "$PYTHON" || export PYTHON=$AXITOOLS_HOME/ShellB3/bin/python3
+
+# scrape warts out of environment
+unset LD_LIBRARY_PATH
+unset PYTHONPATH
+
+export REPO_HIMAWARI='https://gitlab.ssec.wisc.edu/rayg/himawari.git'
+export REPO_GOESR='https://gitlab.ssec.wisc.edu/rayg/goesr.git'
+test -n "$TAG" || export TAG="v$(date '+%Y%m%dT%H%M%S')"
+test -n "$HIMAWARI_TAG" || export HIMAWARI_TAG=develop
+test -n "$GOESR_TAG" || export GOESR_TAG=develop
+
+test -n "$FTP_SB3" || export FTP_SB3="ftp://ftp.ssec.wisc.edu/pub/shellb3"
+test -n "$SB3_TARBALL" || export SB3_TARBALL="ShellB3-Linux-x86_64-CentOS6-20170801-rGIT8dfd19cb-py3-portable-core-cspp.tar.xz"
+test -n "$RUNTIME" || export RUNTIME="$HOME/Downloads"
+
+abspath() {
+    python -c 'import sys,os; print("\n".join([os.path.abspath(x) for x in sys.argv[1:]]))' "$@"
+}
+
+echo "AXITOOLS_HOME=$AXITOOLS_HOME"
+echo "PYTHON=$PYTHON"
+echo "HIMAWARI_TAG=$HIMAWARI_TAG"
+echo "GOESR_TAG=$GOESR_TAG"
+
+
+# runtime requires effective install location of shellB3 is /docker/ShellB3.git/ShellB3 in order for linking to work
+RUNTIME_EXPECTED_LOCATION="/docker/ShellB3.git"
+install_runtime() {
+    test -d $RUNTIME || mkdir $RUNTIME
+    test -f "${RUNTIME}/${SB3_TARBALL}" || ( cd $RUNTIME; curl -O "${FTP_SB3}/${SB3_TARBALL}" )
+    test -f "${RUNTIME}/${SB3_TARBALL}" || oops "runtime snapshot could not be found"
+    test -d "${AXITOOLS_HOME}" || mkdir -p "${AXITOOLS_HOME}/lib"
+    pushd "${AXITOOLS_HOME}"
+    echo "installing ShellB3 tar ball into ${AXITOOLS_HOME}..."
+    test -d ShellB3 || tar xf "${RUNTIME}/${SB3_TARBALL}" || oops "SB3 tar ball would not extract"
+    popd
+    test -d "${RUNTIME_EXPECTED_LOCATION}/ShellB3/bin" || (mkdir -p ${RUNTIME_EXPECTED_LOCATION}; ln -s $AXITOOLS_HOME/ShellB3 $RUNTIME_EXPECTED_LOCATION)
+    $RUNTIME_EXPECTED_LOCATION/ShellB3/bin/python3 -c "import netCDF4, numpy, cartopy" || oops "non-viable python runtime install"
+}
+
+lncp() {
+  ln "$@" || cp -a "$@" || oops "could not link or copy $*"
+}
+
+clone() {
+    dest="$1"
+    repo="$2"
+    tag="$3"
+    if [ -n "$no_checkout" ]; then
+        test -d "$dest" || oops "no pre-existing checkout of $repo; will not attempt to check out"
+        return
+    fi
+    test -d $dest || git clone $repo $dest
+    test -d $dest || oops "unable to find repo $repo in directory $dest"
+    pushd $dest
+    git fetch || oops "could not fetch $repo"
+    git checkout $tag || oops "could not checkout $tag in $repo"
+    popd
+}
+
+install_goesr() {
+    clone goesr $REPO_GOESR $GOESR_TAG
+    pushd goesr
+    $PYTHON setup.py install || oops "could not install goesr module"
+    test -d "$AXITOOLS_HOME/bin" || mkdir -p "$AXITOOLS_HOME/bin"
+    cp bin/cmi_changer.sh "$AXITOOLS_HOME/bin/" || oops "could not install wrapper script"
+    popd
+}
+
+install_himawari() {
+    clone himawari $REPO_HIMAWARI $HIMAWARI_TAG
+    pushd himawari
+    $PYTHON setup.py install || oops "could not install himawari module"
+    popd
+}
+
+set_permissions() {
+    test -d $AXITOOLS_HOME || oops "output directory does not exist"
+    chmod -R a+rX $AXITOOLS_HOME
+}
+
+record_versions() {
+    test -n "$home" || home=$AXITOOLS_HOME
+    test -d $home || mkdir -p $home
+    cat >$home/VERSION.txt <<SOSAYWEALL
+AxI SCMI conversion utilities
+Copyright 2017, University of Wisconsin Regents
+License: GPLv3
+Packaged $(date) on $(hostname)
+himawari version tag: $HIMAWARI_TAG
+goesr version tag: $GOESR_TAG
+SOSAYWEALL
+}
+
+
+
+all() {
+    install_runtime
+    install_goesr
+    install_himawari
+    set_permissions
+    record_versions
+}
+
+
+set -x
+unset did_act
+
+for kw in $*; do
+    $kw
+    did_act=yup
+done
+
+test -n "$did_act" || all

go.sh

+tag=$1
+test -n "$tag" || tag=develop
+if docker images |grep centos6dev; then
+  image=centos6dev:0
+else
+  image=centos:6
+fi
+test -f go.sh && rm go.sh
+curl -O https://gitlab.ssec.wisc.edu/rayg/himawari/raw/${tag}/godo.sh
+docker run --rm -it -v $PWD:/dock -w /dock $image /bin/bash godo.sh ${tag}
+
+

godo.sh

+#!/bin/bash
+set -ex
+yum -y install gcc-gfortran gcc-c++ unzip perl patch yacc bison flex glibc-devel.i686 glibc-devel.x86_64 xz git
+#version="v$(date '+%Y%m%d')"
+tag=$1
+export DEST=$PWD
+mkdir /build
+mkdir /docker
+cd /build
+export AXITOOLS_HOME=$PWD/axi-tools-${tag}
+rm -fr goesr himawari build.sh
+curl -O https://gitlab.ssec.wisc.edu/rayg/himawari/raw/${tag}/build.sh
+export TAG=$tag
+test -n "$HIMAWARI_TAG" || export HIMAWARI_TAG=${tag}
+test -n "$GOESR_TAG" || export GOESR_TAG=${tag}
+test -n "$TAG" || export TAG="v$(date '+%Y%m%dT%H%M%S')"
+env |grep TAG
+bash build.sh
+tar jcvf $DEST/axi-tools-${tag}.tar.bz2 axi-tools-${tag}
+

himawari/HimawariInterface.py

@@ -113,6 +113,9 @@ struct HimawariMetadata {
     int resolution;
     int32_t wmo_sat_id;
     char satellite_name[17];
+    char processing_center_name[17];
+    char observation_area[5];
+    char format_version[33];
 };
 
 

himawari/HimawariScene.py

@@ -87,6 +87,16 @@ class HimawariScene(object):
         name = ffi.string(meta.satellite_name).decode('UTF-8').strip(' \0')
         return name
 
+    @property
+    def processing_center_name(self):
+        meta = self.metadata
+        return ffi.string(meta.processing_center_name).decode('UTF-8').strip(' \0')
+
+    @property
+    def observation_area(self):
+        meta = self.metadata
+        return ffi.string(meta.observation_area).decode('UTF-8').strip(' \0')
+
     def __init__(self, path, sentinel_invalid=None, sentinel_outside_scan=None,
                  downsample_to_band=0, nav_options=DEFAULT_NAVOPT):
         sentinels = ffi.new('struct HimawariSentinels *')

himawari/fortify.py

@@ -22,6 +22,8 @@ __docformat__ = 'reStructuredText'
 
 import os, sys
 import logging, unittest, argparse
+from functools import reduce
+from datetime import datetime
 from pycparser import c_ast, parse_file
 
 LOG = logging.getLogger(__name__)
@@ -171,7 +173,7 @@ def f9x_structs(ast):
     return txt
 
 
-HSD_STRUCTS = """! AUTOGENERATED using fortify.py on C APIS! DO NOT EDIT
+HSD_STRUCTS = """! AUTOGENERATED %sZ using fortify.py on C APIS! DO NOT EDIT
 module hsd_structs
   use iso_c_binding
   implicit none
@@ -184,7 +186,8 @@ end module hsd_structs
 
 def f9x_structs_module(ast):
     txt = '\n'.join(f9x_structs(ast))
-    return HSD_STRUCTS % txt
+    now = datetime.utcnow().isoformat()
+    return HSD_STRUCTS % (now, txt)
 
 
 HSD_FUNC = """    interface
@@ -198,7 +201,7 @@ HSD_FUNC = """    interface
 """
 
 
-HSD_FUNCS = """! AUTOGENERATED using fortify.py on C APIs! DO NOT EDIT
+HSD_FUNCS = """! AUTOGENERATED %sZ using fortify.py on C APIs! DO NOT EDIT
 module hsd_scene
     implicit none
 
@@ -257,7 +260,8 @@ def f9x_funcs(ast):
 
 def f9x_funcs_module(ast):
     txt = '\n'.join(f9x_funcs(ast))
-    return HSD_FUNCS % txt
+    now = datetime.utcnow().isoformat()
+    return HSD_FUNCS % (now, txt)
 
 
 class tests(unittest.TestCase):

himawari/hcast2scmi.sh

@@ -16,9 +16,10 @@ SOURCE="${BASH_SOURCE[0]}"
 # while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
 BASE="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 
+test -x "$PYTHON" || PYTHON=${BASE}/ShellB3/bin/python
 # environment settings
-export PYTHONPATH=${BASE}/himawari/py
-export PATH=${BASE}/ShellB3/bin:$PATH
+# export PYTHONPATH=${BASE}/himawari/py
+# export PATH=${BASE}/ShellB3/bin:$PATH
 unset LD_LIBRARY_PATH
 unset LD_PRELOAD
 
@@ -41,7 +42,7 @@ set -x
 
 # execute on multiple scenes in parallel, writing output to CWD
 for scene in "$@"; do
-    ${BASE}/ShellB3/bin/python -m scmi -v "$scene" $(additional_args "$scene") &
+    $PYTHON -m himawari.scmi -v "$scene" $(additional_args "$scene") &
     echo ">>> started processing job for $scene"
 done
 

himawari/hsd2nc.py

@@ -112,7 +112,10 @@ import os, sys
 import logging, unittest, argparse
 import numpy as np
 from datetime import datetime
-import HimawariScene as himawari
+try:  # new style module first
+    import himawari.HimawariScene as himawari
+except ImportError:  # old ad hoc install style
+    import HimawariScene as himawari
 
 
 LOG = logging.getLogger(__name__)
@@ -476,9 +479,10 @@ def hsd2nc(scene, filename='test.nc', include_rad=True,
     if include_geo and nav_scene is None:
         geo = scene.geo(**qkw)
         # FIXME: revisit best way to interpolate geo - possibly at FGF y/x for instance
-        # note that we're doing striding in the generation above and downsample below: 
-        nc.set_geo( lat=geo.latitude[downsample_factor/2::downsample_factor, downsample_factor/2::downsample_factor], 
-                    lon=geo.longitude[downsample_factor/2::downsample_factor, downsample_factor/2::downsample_factor] )
+        # note that we're doing striding in the generation above and downsample below:
+        dfo2 = int(downsample_factor / 2)
+        nc.set_geo( lat=geo.latitude[dfo2::downsample_factor, dfo2::downsample_factor],
+                    lon=geo.longitude[dfo2::downsample_factor, dfo2::downsample_factor] )
         del geo
     elif include_geo and nav_scene is not None:
         geo = nav_scene.geo(**qkw)
@@ -492,8 +496,9 @@ def hsd2nc(scene, filename='test.nc', include_rad=True,
                                     my=fgf.my,by=fgf.by,
                                     mx=fgf.mx,bx=fgf.bx)
         if reduction_factor>1:
-            fgf = himawari.fgf_yxmb(fgf.y[reduction_factor/2::reduction_factor],
-                                    fgf.x[reduction_factor/2::reduction_factor],
+            rfo2 = int(reduction_factor / 2)
+            fgf = himawari.fgf_yxmb(fgf.y[rfo2::reduction_factor],
+                                    fgf.x[rfo2::reduction_factor],
                                     my=fgf.my,by=fgf.by,
                                     mx=fgf.mx,bx=fgf.bx)
         nc.set_fgf(fgf, reduction_factor)
@@ -502,7 +507,7 @@ def hsd2nc(scene, filename='test.nc', include_rad=True,
         fgf = nav_scene.fgf
         if meta.begin_line>0:
             LOG.warning('partial XY from donor file not fully tested')
-            fgf = himawari.fgf_yxmb(fgf.y[meta.begin_line/downsample_factor:meta.end_line/downsample_factor], fgf.x,
+            fgf = himawari.fgf_yxmb(fgf.y[int(meta.begin_line/downsample_factor):int(meta.end_line/downsample_factor)], fgf.x,
                                     my=fgf.my,by=fgf.by,
                                     mx=fgf.mx,bx=fgf.bx)
         nc.set_fgf(fgf, 1)
@@ -525,18 +530,18 @@ def hsd2nc(scene, filename='test.nc', include_rad=True,
     nc.close()
 
 
-def test_quicklook_radiance(filename='test.nc', var='radiance'):
-    from cspp_quicklooks.core import ql_common as qlc
-    from netCDF4 import Dataset
-    hime = Dataset(filename, 'r') 
-    data = hime.variables[var][:,:]
-    lat = hime.variables['latitude'][:,:]
-    lon = hime.variables['longitude'][:,:]
-    try: 
-        qlc.map_image('test.png', swath=data, lat=lat, lon=lon)
-    except:
-        print("quicklook error")
-    return {'data': data, 'lat': lat, 'lon':lon}
+# def test_quicklook_radiance(filename='test.nc', var='radiance'):
+#     from cspp_quicklooks.core import ql_common as qlc
+#     from netCDF4 import Dataset
+#     hime = Dataset(filename, 'r')
+#     data = hime.variables[var][:,:]
+#     lat = hime.variables['latitude'][:,:]
+#     lon = hime.variables['longitude'][:,:]
+#     try:
+#         qlc.map_image('test.png', swath=data, lat=lat, lon=lon)
+#     except:
+#         print("quicklook error")
+#     return {'data': data, 'lat': lat, 'lon':lon}
 
 
 def attach_nav(candidates=None):

himawari/scmi.py

@@ -229,7 +229,7 @@ import os, sys
 import logging, unittest, argparse
 import numpy as np
 from datetime import datetime, time
-import HimawariScene as himawari
+from himawari.HimawariScene import HimawariScene
 import re
 
 LOG = logging.getLogger(__name__)
@@ -340,7 +340,7 @@ SCMI_FGF_ATT=dict(
     grid_mapping_name='geostationary',
     semi_major=None,  # 6378137.0,
     # semi_major_axis=6378137.0,  # CF
-    sweep_angle_axis="x",
+    sweep_angle_axis="y",
 )
 
 def _ahi_bit_depth(channel):
@@ -411,7 +411,8 @@ def scmi_filename(
         tile=None,           # 001..### upper left to lower right
         satellite='H8',   # 16, 17, ... H8
         scene_time=None,  # datetime object
-        creation_time=None): # now, datetime object
+        creation_time=None, # now, datetime object
+        **kwargs):
     scene_time = scene_time.strftime('%Y%j%H%M%S')
     if creation_time is None:
         creation_time = datetime.utcnow()
@@ -480,18 +481,20 @@ class AttributeHelper(object):
     """
     helper object which wraps around a HimawariScene to provide SCMI attributes
     """
+    additional_naming_info = {}  
     tile_count = (0,0)  # ny, nx
     hsd = None
     offset = (0,0)  # ty, tx tile number
     tile_shape = (0,0)  # wy, wx height and width of tile in pixels
     scene_shape = (0,0)  # sy, sx height and width of scene in pixels
 
-    def __init__(self, hsd, offset, tile_count, scene_shape):
+    def __init__(self, hsd, offset, tile_count, scene_shape, **naming_info):
         self.hsd = hsd
         self.offset = offset
         self.tile_count = tile_count
         self.scene_shape = scene_shape
-        self.tile_shape = (scene_shape[0] / tile_count[0], scene_shape[1] / tile_count[1])
+        self.tile_shape = (int(scene_shape[0] / tile_count[0]), int(scene_shape[1] / tile_count[1]))
+        self.additional_naming_info = naming_info
         if (scene_shape[0] % tile_count[0] != 0) or (scene_shape[1] % tile_count[1] != 0):
             raise ValueError('tile shape %r does not fit evenly into scene shape %r' % (self.tile_shape, scene_shape))
 
@@ -536,10 +539,10 @@ class AttributeHelper(object):
         return self.offset[0] * self.tile_count[1] + self.offset[1] + 1
 
     def _filename(self):
-        return scmi_filename(channel=self.hsd.metadata.band, scene_time=self._scene_time(), tile=self._tile_number())
+        return scmi_filename(channel=self.hsd.metadata.band, scene_time=self._scene_time(), tile=self._tile_number(), **self.additional_naming_info)
 
     def _product_name(self):
-        return scmi_product(channel=self.hsd.metadata.band, scene_time=self._scene_time())
+        return scmi_product(channel=self.hsd.metadata.band, scene_time=self._scene_time(), **self.additional_naming_info)
 
     def _global_product_tile_height(self):  # = None, # 1100,
         return self.tile_shape[0]
@@ -561,8 +564,8 @@ class AttributeHelper(object):
     def _tile_center(self): # = None, # 88.0022078322,
         # calculate center longitude of tile
         # FIXME: resolve whether we need half-pixel offset
-        row = self._global_tile_row_offset() + self.tile_shape[0]/2
-        col = self._global_tile_column_offset() + self.tile_shape[1]/2
+        row = self._global_tile_row_offset() + int(self.tile_shape[0] / 2)
+        col = self._global_tile_column_offset() + int(self.tile_shape[1] / 2)
         # if we're not getting proper nav from the file, substitute nominal values
         args = dict(HCAST_DEFAULT_NAV_CENTER) if self._file_nav_is_incomplete else {}
         nav = self.hsd.geo(line_offset=row, column_offset=col, lines=1, columns=1, **args)
@@ -580,10 +583,10 @@ class AttributeHelper(object):
         return self.scene_shape[1]
 
     def _global_tile_row_offset(self):
-        return self.offset[0] * self.tile_shape[0]
+        return int(self.offset[0] * self.tile_shape[0])
 
     def _global_tile_column_offset(self):
-        return self.offset[1] * self.tile_shape[1]
+        return int(self.offset[1] * self.tile_shape[1])
 
     def _global_tile_center_latitude(self): # = None, # 88.0022078322,
         nav = self._tile_center()
@@ -898,16 +901,33 @@ def default_filename(scene):
     raise NotImplementedError("default_filename is not yet implemented")
 
 
+def workaround_awips16_17(ncpath):
+    """
+    patch from DHoese for AWIPS2 16.4.1 and 17.1.1 NetCDF reader giving Error 72
+    :param ncpath: NetCDF file to patch
+    :return: None
+    """
+    # hack to get files created by new NetCDF library
+    # versions to be read by AWIPS buggy java version
+    # of NetCDF
+    LOG.info("Modifying SCMI NetCDF file to work with AWIPS")
+    import h5py
+    h = h5py.File(ncpath, 'a')
+    if '_NCProperties' in h.attrs:
+        del h.attrs['_NCProperties']
+    h.close()
+
 
 def hsd2scmi(scene, filename=None, include_rad=False,
              include_geo=False, include_fgf=True,
              downsample_factor=0, downsample_to_band=0,
              stride=0, nav_scene=None,
-             offset=None, shape=None, tile_count=(1,1)):
+             offset=None, shape=None, tile_count=(1,1), 
+             **naming_info):
     """convey content of scene to writer object
     """
     if isinstance(scene, str):
-        scene = himawari.HimawariScene(scene, downsample_to_band=downsample_to_band)
+        scene = HimawariScene(scene, downsample_to_band=downsample_to_band)
     # filename = default_filename(scene) if filename is None else filename
 
     if offset is None:
@@ -920,7 +940,7 @@ def hsd2scmi(scene, filename=None, include_rad=False,
 
     scene_kind = scene.kind
     start = meta.start_time
-    print "start_time is %04d-%02d-%02dT%02d:%02d:%02d = %d" % (start.year, start.month, start.day, start.hour, start.minute, start.second, start.unix_time)
+    print("start_time is %04d-%02d-%02dT%02d:%02d:%02d = %d" % (start.year, start.month, start.day, start.hour, start.minute, start.second, start.unix_time))
 
     downsample_factor = 1 if downsample_factor<=0 else int(downsample_factor)
     stride = 1 if stride<=0 else stride
@@ -935,7 +955,7 @@ def hsd2scmi(scene, filename=None, include_rad=False,
         reduction_factor = 1
 
     # create file, dimensions and variables before proceeding to attributes, then values
-    shape = tuple(q/reduction_factor for q in shape)
+    shape = tuple(int(q / reduction_factor) for q in shape)
     resolution = RESOLUTION_FROM_WIDTH[scene.extents[1]]
 
     tiles_total = tile_count[0] * tile_count[1]
@@ -947,14 +967,14 @@ def hsd2scmi(scene, filename=None, include_rad=False,
         ts = time.time()
         zult = himawari.downsample(np.ma.fix_invalid(q, copy=False, fill_value=np.NAN), factor=downsample_factor)
         te = time.time()
-        print "%.3f seconds elapsed during downsampling" % (te-ts)
+        print("%.3f seconds elapsed during downsampling" % (te-ts))
         return np.ma.masked_array(zult, np.isnan(zult))
 
     for ty in range(tile_count[0]):
         for tx in range(tile_count[1]):
             LOG.info('building tile ({},{})'.format(ty,tx))
             tile_offset = (ty, tx)
-            helper = AttributeHelper(scene, tile_offset, tile_count, shape)
+            helper = AttributeHelper(scene, tile_offset, tile_count, shape, **naming_info)
             tile_shape = helper.tile_shape
             filename = filename if (filename and tile_count==1) else helper._filename()
             nc = SCMI_writer(filename, resolution, offset, tile_shape, scene.kind, scene.band,
@@ -1018,8 +1038,9 @@ def hsd2scmi(scene, filename=None, include_rad=False,
                                             my=fgf.my,by=fgf.by,
                                             mx=fgf.mx,bx=fgf.bx)
                 if reduction_factor>1:
-                    fgf = himawari.fgf_yxmb(fgf.y[reduction_factor/2::reduction_factor],
-                                            fgf.x[reduction_factor/2::reduction_factor],
+                    rfo2 = int(reduction_factor / 2)
+                    fgf = himawari.fgf_yxmb(fgf.y[rfo2::reduction_factor],
+                                            fgf.x[rfo2::reduction_factor],
                                             my=fgf.my,by=fgf.by,
                                             mx=fgf.mx,bx=fgf.bx)
                 y = fgf.y[ty*tile_shape[0]:(ty+1)*tile_shape[0]]
@@ -1066,7 +1087,7 @@ def test_quicklook_radiance(filename='test.nc', var='radiance'):
     try:
         qlc.map_image('test.png', swath=data, lat=lat, lon=lon)
     except:
-        print "quicklook error"
+        print("quicklook error")
     return {'data': data, 'lat': lat, 'lon':lon}
 
 
@@ -1082,7 +1103,7 @@ def attach_nav(candidates=None):
             pass
 
 def _parse_tile_count(s):
-    vals = map(int, re.findall(r'\d+', s))
+    vals = list(map(int, re.findall(r'\d+', s)))
     if len(vals)==1:
         return vals[0], vals[0]
     elif len(vals)==2:
@@ -1090,6 +1111,16 @@ def _parse_tile_count(s):
     else:
         raise ValueError('cannot get tile count out of %r' % s)
 
+def _debug(type, value, tb):
+    "enable with sys.excepthook = debug"
+    if not sys.stdin.isatty():
+        sys.__excepthook__(type, value, tb)
+    else:
+        import traceback, pdb
+        traceback.print_exception(type, value, tb)
+        # …then start the debugger in post-mortem mode.
+        pdb.post_mortem(tb)  # more “modern”
+
 def main():
     parser = argparse.ArgumentParser(
         description="PURPOSE",
@@ -1109,6 +1140,13 @@ def main():
                         help='factor to stride by, typically 1, 2 or 4')
     parser.add_argument('-t', '--tiles', dest='tiles', default='1',
                         help='number of tiles to create in each dimension, or y,x tile count pair')
+    parser.add_argument('-R', '--region', help='region tag to use in filename, e.g. HFD', default='HFD')
+    parser.add_argument('-E', '--environment', default='DT',
+                        help='environment and data type to use in filename, e.g. DT or OR')
+    parser.add_argument('--fix', default=False, action='store_true',
+                        help='apply netCDF "HDF Error 72" workaround for AWIPS2 16.4.1 and 17.1.1')
+    parser.add_argument('--debug', dest='debug', action='store_true',
+                        help="enable interactive PDB debugger on exception")
     # FUTURE: include a resolution tag for 2km/1km/0p5km
 
     # http://docs.python.org/2.7/library/argparse.html#nargs
@@ -1118,6 +1156,9 @@ def main():
                         help="positional arguments don't have the '-' prefix")
     args = parser.parse_args()
 
+    if args.debug:
+        sys.excepthook = _debug
+
     levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
     logging.basicConfig(level=levels[min(3, args.verbosity)])
 
@@ -1125,10 +1166,21 @@ def main():
     #     unittest.main()
     #     return 0
 
-    fns = list(hsd2scmi(*args.pos_args, downsample_factor=args.factor, stride=args.stride,
-                         include_geo=args.geolocation, include_fgf=True, include_rad=False,
-                         tile_count=_parse_tile_count(args.tiles), nav_scene=attach_nav(args.nav_donor)))
-    print('==> ' + ', '.join(fns))
+    try:
+        environment, data_type = args.environment
+    except ValueError:
+        LOG.error('environment parameter requires two characters')
+        return 1
+
+    if args.pos_args:
+        fns = list(hsd2scmi(*args.pos_args, downsample_factor=args.factor, stride=args.stride,
+                             include_geo=args.geolocation, include_fgf=True, include_rad=False,
+                             tile_count=_parse_tile_count(args.tiles), nav_scene=attach_nav(args.nav_donor),
+                             environment=environment, data_type=data_type, region=args.region))
+        for fn in fns:
+            if args.fix:
+                workaround_awips16_17(fn)
+            print("> " + fn)
 
     return 0
 

include/HimawariTypes.h

@@ -105,6 +105,9 @@ struct HimawariMetadata {
     int resolution;
     int32_t wmo_sat_id;    /* http://www.wmo-sat.info/oscar/satellites */
     char satellite_name[17];
+    char processing_center_name[17];
+    char observation_area[5];
+    char format_version[33];
 };
 
 

include/aHimawariFile.hxx

@@ -26,6 +26,7 @@