From abeef363a757d44a7174b5ff7c4c756a65e3dd63 Mon Sep 17 00:00:00 2001
From: David Hoese <david.hoese@ssec.wisc.edu>
Date: Mon, 21 Oct 2019 19:21:07 -0500
Subject: [PATCH] Add initial almost working package scripts
---
buildbucket/package.sh | 28 ++-
gridded_glm/bin/_make_glm_grids.py | 325 +++++++++++++++++++++++++++++
gridded_glm/bin/env.sh | 31 +++
gridded_glm/bin/make_glm_grids.sh | 32 +++
gridded_glm/bin/run.sh | 4 -
5 files changed, 411 insertions(+), 9 deletions(-)
create mode 100644 gridded_glm/bin/_make_glm_grids.py
create mode 100644 gridded_glm/bin/env.sh
create mode 100755 gridded_glm/bin/make_glm_grids.sh
delete mode 100644 gridded_glm/bin/run.sh
diff --git a/buildbucket/package.sh b/buildbucket/package.sh
index 0a7132a..3d9f1ea 100755
--- a/buildbucket/package.sh
+++ b/buildbucket/package.sh
@@ -19,6 +19,24 @@ else
exit 1
fi
+pkg_name=cspp_geo_gridded_glm-${version}
+DIST=${DIST:-"/dock"}
+
+make_dockerfile() {
+ cat >$DIST/Dockerfile <<EOF
+# docker build . -f Dockerfile -t gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-gridded-glm/run_package:${version}
+# docker run -it -v /path/to/data:/work --rm gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-gridded-glm/run_package:${version} make_glm_grids.sh OR_GLM-L2-LCFA_*.nc
+FROM centos:7
+RUN mkdir -p /opt/ssec && mkdir -p /work
+ADD ${pkg_name}.tar.xz /opt/ssec
+WORKDIR /work
+ENV CSPP_GEO_GGLM_HOME=/opt/ssec/${pkg_name}
+ENV PATH=\$PATH:/opt/ssec/${pkg_name}/bin
+EOF
+
+}
+
+
conda activate build
# Debug Info
@@ -41,11 +59,11 @@ pip install --no-deps git+https://github.com/deeplycloudy/stormdrain.git
pip install --no-deps git+https://github.com/deeplycloudy/glmtools.git@${glmtools_ref}
# Build a tarball version of the current conda environment
+# TODO: Add conda cleanup commands similar to what Polar2Grid uses to save space
conda_tb=conda_lmatools-${lmatools_ref}_glmtools-${glmtools_ref}.tar.gz
-conda pack -n build -o ${conda_tb}
+conda pack --n-threads $(nproc) -n build -o ${conda_tb}
# Build up our package directory
-pkg_name=cspp_geo_gridded_glm-${version}
mkdir -p ${pkg_name}
cd ${pkg_name}
mkdir -p bin opt/conda
@@ -58,6 +76,6 @@ tar -xz -C ./opt/conda -f ../${conda_tb}
cd ..
# Create tarball of package directory
-tar -czf ${pkg_name}.tar.gz ${pkg_name}
-mv ${pkg_name}.tar.gz /dock/
-
+XZ_DEFAULTS="--threads=$(nproc)" tar -Jcf --uid 0 --gid 0 ${pkg_name}.tar.xz ${pkg_name}
+mv ${pkg_name}.tar.xz ${DIST}/
+make_dockerfile
diff --git a/gridded_glm/bin/_make_glm_grids.py b/gridded_glm/bin/_make_glm_grids.py
new file mode 100644
index 0000000..3987597
--- /dev/null
+++ b/gridded_glm/bin/_make_glm_grids.py
@@ -0,0 +1,325 @@
+import argparse
+
+parse_desc = """Grid GLM flash data. The start and end times can be specified
+independently, or if not provided they will be inferred from the filenames.
+
+Grid spacing is regular in latitude and longitude with the grid box
+being sized to match the requested dx, dy at the center of the grid.
+
+Within the output directory, a year/month/day directory will be created,
+e.g., 2017/Jul/04/, and within that directory the grid files will be created.
+
+Therefore, this script can be used to process multiple days and they will
+be written to a standardized directory structure.
+"""
+
+
+def create_parser():
+ parser = argparse.ArgumentParser(description=parse_desc)
+ parser.add_argument(dest='filenames', metavar='filename', nargs='*')
+ parser.add_argument('-o', '--output_dir', metavar='directory',
+ required=True, dest='outdir', action='store', )
+ parser.add_argument('--ctr_lat', metavar='latitude', required=False,
+ dest='ctr_lat', action='store', type=float,
+ help='center latitude')
+ parser.add_argument('--ctr_lon', metavar='longitude', required=False,
+ dest='ctr_lon', action='store', type=float,
+ help='center longitude')
+ parser.add_argument('--start', metavar='yyyy-mm-ddThh:mm:ss',
+ dest='start', action='store',
+ help='UTC start time, e.g., 2017-07-04T08:00:00')
+ parser.add_argument('--end', metavar='yyyy-mm-ddThh:mm:ss',
+ dest='end', action='store',
+ help='UTC end time, e.g., 2017-07-04T09:00:00')
+ parser.add_argument('--dx', metavar='km',
+ dest='dx', action='store', default=10.0, type=float,
+ help='approximate east-west grid spacing')
+ parser.add_argument('--dy', metavar='km',
+ dest='dy', action='store', default=10.0, type=float,
+ help='approximate north-south grid spacing')
+ parser.add_argument('--dt', metavar='seconds',
+ dest='dt', action='store', default=60.0, type=float,
+ help='frame duration')
+ parser.add_argument('--width', metavar='distance in km',
+ dest='width', action='store', default=400.0,
+ type=float, help='total width of the grid')
+ parser.add_argument('--height', metavar='distance in km',
+ dest='height', action='store', default=400.0,
+ type=float, help='total height of the grid')
+ parser.add_argument('--nevents', metavar='minimum events per flash',
+ type=int, dest='min_events', action='store', default=1,
+ help='minimum number of events per flash')
+ parser.add_argument('--ngroups', metavar='minimum groups per flash',
+ type=int, dest='min_groups', action='store', default=1,
+ help='minimum number of groups per flash')
+ parser.add_argument('--fixed_grid',
+ action='store_true', dest='fixed_grid',
+ help='grid to the geostationary fixed grid')
+ parser.add_argument('--subdivide_grid', metavar='sqrt(number of subgrids)',
+ action='store', dest='subdivide_grid',
+ type=int, default=1,
+ help=("subdivide the grid this many times along "
+ "each dimension"))
+ parser.add_argument('--goes_position', default='none',
+ action='store', dest='goes_position',
+ help=("One of [east|west|test]. "
+ "Also requires goes_sector."))
+ parser.add_argument('--goes_sector', default='none',
+ action='store', dest='goes_sector',
+ help=("One of [full|conus|meso]. "
+ "Also requires goes_position. If sector is "
+ "meso, ctr_lon and ctr_lat are interpreted as "
+ "the ctr_x and ctr_y of the fixed grid"))
+ parser.add_argument('--corner_points', metavar='filename.pickle',
+ action='store', dest='corner_points',
+ help=("name of file containing a pickled "
+ "corner point lookup table"))
+ parser.add_argument('--split_events', dest='split_events',
+ action='store_true',
+ help='Split GLM event polygons when gridding')
+ parser.add_argument('--ellipse', dest='ellipse_rev', default=-1,
+ action='store', type=int,
+ help='Lightning ellipse revision. -1 (default)=infer'
+ ' from date in each GLM file, 0=value at launch,'
+ ' 1=late 2018 revision')
+ parser.add_argument('--float_output', dest='output_scale_and_offset',
+ default=True,
+ action='store_false',
+ help='write all output variables as floating point')
+ parser.add_argument('--lma', dest='is_lma',
+ action='store_true',
+ help='grid LMA h5 files instead of GLM data')
+ # parser.add_argument('-v', dest='verbose', action='store_true',
+ # help='verbose mode')
+ return parser
+
+
+##### END PARSING #####
+
+import numpy as np
+import subprocess, glob
+from datetime import datetime
+import os
+from functools import partial
+
+import logging
+
+
+class MyFormatter(logging.Formatter):
+ """ Custom class to allow logging of microseconds"""
+ converter = datetime.fromtimestamp
+
+ def formatTime(self, record, datefmt=None):
+ ct = self.converter(record.created)
+ if datefmt:
+ s = ct.strftime(datefmt)
+ else:
+ t = ct.strftime("%Y-%m-%d %H:%M:%S")
+ s = "%s,%03d" % (t, record.msecs)
+ return s
+
+
+logoutfile = logging.FileHandler("make_GLM_grid.log")
+formatter = MyFormatter(fmt='%(levelname)s %(asctime)s %(message)s',
+ datefmt='%Y-%m-%dT%H:%M:%S.%f')
+logoutfile.setFormatter(formatter)
+logging.basicConfig(handlers=[logoutfile],
+ level=logging.DEBUG)
+
+# Separate from log setup - actually log soemthign specific to this module.
+log = logging.getLogger(__name__)
+log.info("Starting GLM Gridding")
+
+
+def nearest_resolution(args):
+ """ Uses args.dx to find the closest resolution specified by the
+ GOES-R PUG. Returns something like "10.0km" that can be used as the
+ resolution argument to get_GOESR_grid.
+ """
+ goes_resln_options = np.asarray([0.5, 1.0, 2.0, 4.0, 8.0, 10.0])
+ resln_idx = np.argmin(np.abs(goes_resln_options - args.dx))
+ closest_resln = goes_resln_options[resln_idx]
+ resln = '{0:4.1f}km'.format(closest_resln).replace(' ', '')
+ return resln
+
+
+def grid_setup(args):
+ from lmatools.grid.make_grids import write_cf_netcdf_latlon, write_cf_netcdf_noproj, write_cf_netcdf_fixedgrid
+ from lmatools.grid.make_grids import dlonlat_at_grid_center, grid_h5flashfiles
+ from glmtools.grid.make_grids import grid_GLM_flashes
+ from glmtools.io.glm import parse_glm_filename
+ from lmatools.io.LMA_h5_file import parse_lma_h5_filename
+ from lmatools.grid.fixed import get_GOESR_grid, get_GOESR_coordsys
+
+ # When passed None for the minimum event or group counts, the gridder will skip
+ # the check, saving a bit of time.
+ min_events = int(args.min_events)
+ if min_events <= 1:
+ min_events = None
+ min_groups = int(args.min_groups)
+ if min_groups <= 1:
+ min_groups = None
+
+ if args.is_lma:
+ filename_parser = parse_lma_h5_filename
+ start_idx = 0
+ end_idx = 1
+ else:
+ filename_parser = parse_glm_filename
+ start_idx = 3
+ end_idx = 4
+
+ glm_filenames = args.filenames
+ base_filenames = [os.path.basename(p) for p in glm_filenames]
+ try:
+ filename_infos = [filename_parser(f) for f in base_filenames]
+ # opsenv, algorithm, platform, start, end, created = parse_glm_filename(f)
+ filename_starts = [info[start_idx] for info in filename_infos]
+ filename_ends = [info[end_idx] for info in filename_infos]
+ except ValueError:
+ log.error("One or more GLM files has a non-standard filename.")
+ log.error("Assuming that --start and --end have been passed directly.")
+
+ from glmtools.io.glm import parse_glm_filename
+ if args.start is not None:
+ start_time = datetime.strptime(args.start[:19], '%Y-%m-%dT%H:%M:%S')
+ else:
+ start_time = min(filename_starts)
+ if args.end is not None:
+ end_time = datetime.strptime(args.end[:19], '%Y-%m-%dT%H:%M:%S')
+ else:
+ end_time = max(filename_ends)
+
+ date = datetime(start_time.year, start_time.month, start_time.day)
+ # grid_dir = os.path.join('/data/LCFA-production/', 'grid_test')
+ # outpath = grid_dir+'/20%s' %(date.strftime('%y/%b/%d'))
+ outpath = os.path.join(args.outdir, '20%s' % (date.strftime('%y/%b/%d')))
+ if os.path.exists(outpath) == False:
+ os.makedirs(outpath)
+ # subprocess.call(['chmod', 'a+w', outpath, grid_dir+'/20%s' %(date.strftime('%y/%b')), grid_dir+'/20%s' %(date.strftime('%y'))])
+
+ if args.fixed_grid:
+ proj_name = 'geos'
+
+ if (args.goes_position != 'none') & (args.goes_sector != 'none'):
+ resln = nearest_resolution(args)
+ view = get_GOESR_grid(position=args.goes_position,
+ view=args.goes_sector,
+ resolution=resln)
+ nadir_lon = view['nadir_lon']
+ dx = dy = view['resolution']
+ nx, ny = view['pixelsEW'], view['pixelsNS']
+ geofixcs, grs80lla = get_GOESR_coordsys(sat_lon_nadir=nadir_lon)
+
+ if 'centerEW' in view:
+ x_ctr, y_ctr = view['centerEW'], view['centerNS']
+ elif args.goes_sector == 'meso':
+ # use ctr_lon, ctr_lat to get the center of the mesoscale FOV
+ x_ctr, y_ctr, z_ctr = geofixcs.fromECEF(
+ *grs80lla.toECEF(args.ctr_lon, args.ctr_lat, 0.0))
+ elif (args.goes_position != 'none') & (args.goes_sector == 'none'):
+ # Requires goes_position, a center, and a width. Fully flexible
+ # in resolution, i.e., doesn't slave it to one of the GOES-R specs
+ view = get_GOESR_grid(position=args.goes_position,
+ view='full',
+ resolution='1.0km')
+ nadir_lon = view['nadir_lon']
+ dx1km = dy1km = view['resolution']
+ geofixcs, grs80lla = get_GOESR_coordsys(sat_lon_nadir=nadir_lon)
+ x_ctr, y_ctr, z_ctr = geofixcs.fromECEF(
+ *grs80lla.toECEF(args.ctr_lon, args.ctr_lat, 0.0))
+
+ # Convert the specified resolution in km given by args.dx to
+ # a delta in fixed grid coordinates using the 1 km delta from the
+ # GOES-R PUG.
+ dx, dy = args.dx * dx1km, args.dy * dy1km
+ nx, ny = int(args.width / args.dx), int(args.height / args.dy)
+ else:
+ raise ValueError("Gridding on the fixed grid requires "
+ "goes_position and dx. For goes_sector='meso', also specify "
+ "ctr_lon and ctr_lat. Without goes_sector, also include width "
+ "and height.")
+ # Need to use +1 here to convert to xedge, yedge expected by gridder
+ # instead of the pixel centroids that will result in the final image
+ nx += 1
+ ny += 1
+ x_bnd = (np.arange(nx, dtype='float') - (nx) / 2.0) * dx + x_ctr + 0.5 * dx
+ y_bnd = (np.arange(ny, dtype='float') - (ny) / 2.0) * dy + y_ctr + 0.5 * dy
+ log.debug(("initial x,y_ctr", x_ctr, y_ctr))
+ log.debug(("initial x,y_bnd", x_bnd.shape, y_bnd.shape))
+ x_bnd = np.asarray([x_bnd.min(), x_bnd.max()])
+ y_bnd = np.asarray([y_bnd.min(), y_bnd.max()])
+
+ geofixcs, grs80lla = get_GOESR_coordsys(sat_lon_nadir=nadir_lon)
+ ctr_lon, ctr_lat, ctr_alt = grs80lla.fromECEF(
+ *geofixcs.toECEF(x_ctr, y_ctr, 0.0))
+ fixed_grid = geofixcs
+ log.debug((x_bnd, y_bnd, dx, dy, nx, ny))
+
+ output_writer = partial(write_cf_netcdf_fixedgrid, nadir_lon=nadir_lon)
+ else:
+ # Default
+ proj_name = 'latlong'
+ output_writer = write_cf_netcdf_latlon
+ ctr_lat = float(args.ctr_lat)
+ ctr_lon = float(args.ctr_lon)
+ dx_km = float(args.dx) * 1.0e3
+ dy_km = float(args.dy) * 1.0e3
+ width, height = 1000.0 * float(args.width), 1000.0 * float(args.height)
+ x_bnd_km = (-width / 2.0, width / 2.0)
+ y_bnd_km = (-height / 2.0, height / 2.0)
+ dx, dy, x_bnd, y_bnd = dlonlat_at_grid_center(ctr_lat, ctr_lon,
+ dx=dx_km, dy=dy_km,
+ x_bnd=x_bnd_km, y_bnd=y_bnd_km)
+
+ # tuples of the corners
+ corners = np.vstack([(x_bnd[0], y_bnd[0]), (x_bnd[0], y_bnd[1]),
+ (x_bnd[1], y_bnd[1]), (x_bnd[1], y_bnd[0])])
+ # print(x_bnd, y_bnd)
+
+ if args.is_lma:
+ gridder = grid_h5flashfiles
+ output_filename_prefix = 'LMA'
+ else:
+ gridder = grid_GLM_flashes
+ output_filename_prefix = 'GLM'
+
+ grid_kwargs = dict(proj_name=proj_name,
+ base_date=date, do_3d=False,
+ dx=dx, dy=dy, frame_interval=float(args.dt),
+ x_bnd=x_bnd, y_bnd=y_bnd,
+ ctr_lat=ctr_lat, ctr_lon=ctr_lon, outpath=outpath,
+ min_points_per_flash=min_events,
+ output_writer=output_writer, subdivide=args.subdivide_grid,
+ output_filename_prefix=output_filename_prefix,
+ output_kwargs={'scale_and_offset': args.output_scale_and_offset},
+ spatial_scale_factor=1.0)
+
+ if args.fixed_grid:
+ grid_kwargs['fixed_grid'] = True
+ grid_kwargs['nadir_lon'] = nadir_lon
+ if args.split_events:
+ grid_kwargs['clip_events'] = True
+ if min_groups is not None:
+ grid_kwargs['min_groups_per_flash'] = min_groups
+ if args.is_lma:
+ grid_kwargs['energy_grids'] = True
+ else:
+ grid_kwargs['energy_grids'] = ('total_energy',)
+ if (proj_name == 'pixel_grid') or (proj_name == 'geos'):
+ grid_kwargs['pixel_coords'] = fixed_grid
+ grid_kwargs['ellipse_rev'] = args.ellipse_rev
+ # if args.corner_points:
+ # grid_kwargs['corner_pickle'] = args.corner_points
+ return gridder, glm_filenames, start_time, end_time, grid_kwargs
+
+
+if __name__ == '__main__':
+ parser = create_parser()
+ args = parser.parse_args()
+
+ from multiprocessing import freeze_support
+
+ freeze_support()
+ gridder, glm_filenames, start_time, end_time, grid_kwargs = grid_setup(args)
+ gridder(glm_filenames, start_time, end_time, **grid_kwargs)
diff --git a/gridded_glm/bin/env.sh b/gridded_glm/bin/env.sh
new file mode 100644
index 0000000..0bcdaa0
--- /dev/null
+++ b/gridded_glm/bin/env.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# encoding: utf-8
+# Copyright (C) 2019 Space Science and Engineering Center (SSEC),
+# University of Wisconsin-Madison.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# This file is part of the CSPP Geo Gridded GLM software package. CSPP Geo
+# Gridded GLM takes GOES GLM Level 2 LCFA files and grids them to the ABI
+# fixed grid. It does this using the open source glmtools python package by
+# Eric Bruning.
+
+if [ -z "$CSPP_GEO_GGLM_HOME" ]; then
+ export CSPP_GEO_GGLM_HOME="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
+fi
+
+unset PYTHONPATH
+unset LD_LIBRARY_PATH
+
+export PATH=$PATH:$CSPP_GEO_GGLM_HOME/bin:$CSPP_GEO_GGLM_HOME/opt/conda/bin
\ No newline at end of file
diff --git a/gridded_glm/bin/make_glm_grids.sh b/gridded_glm/bin/make_glm_grids.sh
new file mode 100755
index 0000000..d844d91
--- /dev/null
+++ b/gridded_glm/bin/make_glm_grids.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# encoding: utf-8
+# Copyright (C) 2019 Space Science and Engineering Center (SSEC),
+# University of Wisconsin-Madison.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# This file is part of the CSPP Geo Gridded GLM software package. CSPP Geo
+# Gridded GLM takes GOES GLM Level 2 LCFA files and grids them to the ABI
+# fixed grid. It does this using the open source glmtools python package by
+# Eric Bruning.
+
+if [ -z "$CSPP_GEO_GGLM_HOME" ]; then
+ export CSPP_GEO_GGLM_HOME="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
+fi
+
+# Setup necessary environments
+source $CSPP_GEO_GGLM_HOME/bin/env.sh
+
+# Call the python module to do the processing, passing all arguments
+python3 $CSPP_GEO_GGLM_HOME/bin/_make_glm_grids.py "$@"
diff --git a/gridded_glm/bin/run.sh b/gridded_glm/bin/run.sh
deleted file mode 100644
index 5850002..0000000
--- a/gridded_glm/bin/run.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-echo "TODO"
-exit 1
\ No newline at end of file
--
GitLab