From abeef363a757d44a7174b5ff7c4c756a65e3dd63 Mon Sep 17 00:00:00 2001 From: David Hoese <david.hoese@ssec.wisc.edu> Date: Mon, 21 Oct 2019 19:21:07 -0500 Subject: [PATCH] Add initial almost working package scripts --- buildbucket/package.sh | 28 ++- gridded_glm/bin/_make_glm_grids.py | 325 +++++++++++++++++++++++++++++ gridded_glm/bin/env.sh | 31 +++ gridded_glm/bin/make_glm_grids.sh | 32 +++ gridded_glm/bin/run.sh | 4 - 5 files changed, 411 insertions(+), 9 deletions(-) create mode 100644 gridded_glm/bin/_make_glm_grids.py create mode 100644 gridded_glm/bin/env.sh create mode 100755 gridded_glm/bin/make_glm_grids.sh delete mode 100644 gridded_glm/bin/run.sh diff --git a/buildbucket/package.sh b/buildbucket/package.sh index 0a7132a..3d9f1ea 100755 --- a/buildbucket/package.sh +++ b/buildbucket/package.sh @@ -19,6 +19,24 @@ else exit 1 fi +pkg_name=cspp_geo_gridded_glm-${version} +DIST=${DIST:-"/dock"} + +make_dockerfile() { + cat >$DIST/Dockerfile <<EOF +# docker build . -f Dockerfile -t gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-gridded-glm/run_package:${version} +# docker run -it -v /path/to/data:/work --rm gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-gridded-glm/run_package:${version} make_glm_grids.sh OR_GLM-L2-LCFA_*.nc +FROM centos:7 +RUN mkdir -p /opt/ssec && mkdir -p /work +ADD ${pkg_name}.tar.xz /opt/ssec +WORKDIR /work +ENV CSPP_GEO_GGLM_HOME=/opt/ssec/${pkg_name} +ENV PATH=\$PATH:/opt/ssec/${pkg_name}/bin +EOF + +} + + conda activate build # Debug Info @@ -41,11 +59,11 @@ pip install --no-deps git+https://github.com/deeplycloudy/stormdrain.git pip install --no-deps git+https://github.com/deeplycloudy/glmtools.git@${glmtools_ref} # Build a tarball version of the current conda environment +# TODO: Add conda cleanup commands similar to what Polar2Grid uses to save space conda_tb=conda_lmatools-${lmatools_ref}_glmtools-${glmtools_ref}.tar.gz -conda pack -n build -o ${conda_tb} +conda pack --n-threads $(nproc) -n build -o ${conda_tb} # Build up our package directory -pkg_name=cspp_geo_gridded_glm-${version} mkdir -p ${pkg_name} cd ${pkg_name} mkdir -p bin opt/conda @@ -58,6 +76,6 @@ tar -xz -C ./opt/conda -f ../${conda_tb} cd .. # Create tarball of package directory -tar -czf ${pkg_name}.tar.gz ${pkg_name} -mv ${pkg_name}.tar.gz /dock/ - +XZ_DEFAULTS="--threads=$(nproc)" tar -Jcf --uid 0 --gid 0 ${pkg_name}.tar.xz ${pkg_name} +mv ${pkg_name}.tar.xz ${DIST}/ +make_dockerfile diff --git a/gridded_glm/bin/_make_glm_grids.py b/gridded_glm/bin/_make_glm_grids.py new file mode 100644 index 0000000..3987597 --- /dev/null +++ b/gridded_glm/bin/_make_glm_grids.py @@ -0,0 +1,325 @@ +import argparse + +parse_desc = """Grid GLM flash data. The start and end times can be specified +independently, or if not provided they will be inferred from the filenames. + +Grid spacing is regular in latitude and longitude with the grid box +being sized to match the requested dx, dy at the center of the grid. + +Within the output directory, a year/month/day directory will be created, +e.g., 2017/Jul/04/, and within that directory the grid files will be created. + +Therefore, this script can be used to process multiple days and they will +be written to a standardized directory structure. +""" + + +def create_parser(): + parser = argparse.ArgumentParser(description=parse_desc) + parser.add_argument(dest='filenames', metavar='filename', nargs='*') + parser.add_argument('-o', '--output_dir', metavar='directory', + required=True, dest='outdir', action='store', ) + parser.add_argument('--ctr_lat', metavar='latitude', required=False, + dest='ctr_lat', action='store', type=float, + help='center latitude') + parser.add_argument('--ctr_lon', metavar='longitude', required=False, + dest='ctr_lon', action='store', type=float, + help='center longitude') + parser.add_argument('--start', metavar='yyyy-mm-ddThh:mm:ss', + dest='start', action='store', + help='UTC start time, e.g., 2017-07-04T08:00:00') + parser.add_argument('--end', metavar='yyyy-mm-ddThh:mm:ss', + dest='end', action='store', + help='UTC end time, e.g., 2017-07-04T09:00:00') + parser.add_argument('--dx', metavar='km', + dest='dx', action='store', default=10.0, type=float, + help='approximate east-west grid spacing') + parser.add_argument('--dy', metavar='km', + dest='dy', action='store', default=10.0, type=float, + help='approximate north-south grid spacing') + parser.add_argument('--dt', metavar='seconds', + dest='dt', action='store', default=60.0, type=float, + help='frame duration') + parser.add_argument('--width', metavar='distance in km', + dest='width', action='store', default=400.0, + type=float, help='total width of the grid') + parser.add_argument('--height', metavar='distance in km', + dest='height', action='store', default=400.0, + type=float, help='total height of the grid') + parser.add_argument('--nevents', metavar='minimum events per flash', + type=int, dest='min_events', action='store', default=1, + help='minimum number of events per flash') + parser.add_argument('--ngroups', metavar='minimum groups per flash', + type=int, dest='min_groups', action='store', default=1, + help='minimum number of groups per flash') + parser.add_argument('--fixed_grid', + action='store_true', dest='fixed_grid', + help='grid to the geostationary fixed grid') + parser.add_argument('--subdivide_grid', metavar='sqrt(number of subgrids)', + action='store', dest='subdivide_grid', + type=int, default=1, + help=("subdivide the grid this many times along " + "each dimension")) + parser.add_argument('--goes_position', default='none', + action='store', dest='goes_position', + help=("One of [east|west|test]. " + "Also requires goes_sector.")) + parser.add_argument('--goes_sector', default='none', + action='store', dest='goes_sector', + help=("One of [full|conus|meso]. " + "Also requires goes_position. If sector is " + "meso, ctr_lon and ctr_lat are interpreted as " + "the ctr_x and ctr_y of the fixed grid")) + parser.add_argument('--corner_points', metavar='filename.pickle', + action='store', dest='corner_points', + help=("name of file containing a pickled " + "corner point lookup table")) + parser.add_argument('--split_events', dest='split_events', + action='store_true', + help='Split GLM event polygons when gridding') + parser.add_argument('--ellipse', dest='ellipse_rev', default=-1, + action='store', type=int, + help='Lightning ellipse revision. -1 (default)=infer' + ' from date in each GLM file, 0=value at launch,' + ' 1=late 2018 revision') + parser.add_argument('--float_output', dest='output_scale_and_offset', + default=True, + action='store_false', + help='write all output variables as floating point') + parser.add_argument('--lma', dest='is_lma', + action='store_true', + help='grid LMA h5 files instead of GLM data') + # parser.add_argument('-v', dest='verbose', action='store_true', + # help='verbose mode') + return parser + + +##### END PARSING ##### + +import numpy as np +import subprocess, glob +from datetime import datetime +import os +from functools import partial + +import logging + + +class MyFormatter(logging.Formatter): + """ Custom class to allow logging of microseconds""" + converter = datetime.fromtimestamp + + def formatTime(self, record, datefmt=None): + ct = self.converter(record.created) + if datefmt: + s = ct.strftime(datefmt) + else: + t = ct.strftime("%Y-%m-%d %H:%M:%S") + s = "%s,%03d" % (t, record.msecs) + return s + + +logoutfile = logging.FileHandler("make_GLM_grid.log") +formatter = MyFormatter(fmt='%(levelname)s %(asctime)s %(message)s', + datefmt='%Y-%m-%dT%H:%M:%S.%f') +logoutfile.setFormatter(formatter) +logging.basicConfig(handlers=[logoutfile], + level=logging.DEBUG) + +# Separate from log setup - actually log soemthign specific to this module. +log = logging.getLogger(__name__) +log.info("Starting GLM Gridding") + + +def nearest_resolution(args): + """ Uses args.dx to find the closest resolution specified by the + GOES-R PUG. Returns something like "10.0km" that can be used as the + resolution argument to get_GOESR_grid. + """ + goes_resln_options = np.asarray([0.5, 1.0, 2.0, 4.0, 8.0, 10.0]) + resln_idx = np.argmin(np.abs(goes_resln_options - args.dx)) + closest_resln = goes_resln_options[resln_idx] + resln = '{0:4.1f}km'.format(closest_resln).replace(' ', '') + return resln + + +def grid_setup(args): + from lmatools.grid.make_grids import write_cf_netcdf_latlon, write_cf_netcdf_noproj, write_cf_netcdf_fixedgrid + from lmatools.grid.make_grids import dlonlat_at_grid_center, grid_h5flashfiles + from glmtools.grid.make_grids import grid_GLM_flashes + from glmtools.io.glm import parse_glm_filename + from lmatools.io.LMA_h5_file import parse_lma_h5_filename + from lmatools.grid.fixed import get_GOESR_grid, get_GOESR_coordsys + + # When passed None for the minimum event or group counts, the gridder will skip + # the check, saving a bit of time. + min_events = int(args.min_events) + if min_events <= 1: + min_events = None + min_groups = int(args.min_groups) + if min_groups <= 1: + min_groups = None + + if args.is_lma: + filename_parser = parse_lma_h5_filename + start_idx = 0 + end_idx = 1 + else: + filename_parser = parse_glm_filename + start_idx = 3 + end_idx = 4 + + glm_filenames = args.filenames + base_filenames = [os.path.basename(p) for p in glm_filenames] + try: + filename_infos = [filename_parser(f) for f in base_filenames] + # opsenv, algorithm, platform, start, end, created = parse_glm_filename(f) + filename_starts = [info[start_idx] for info in filename_infos] + filename_ends = [info[end_idx] for info in filename_infos] + except ValueError: + log.error("One or more GLM files has a non-standard filename.") + log.error("Assuming that --start and --end have been passed directly.") + + from glmtools.io.glm import parse_glm_filename + if args.start is not None: + start_time = datetime.strptime(args.start[:19], '%Y-%m-%dT%H:%M:%S') + else: + start_time = min(filename_starts) + if args.end is not None: + end_time = datetime.strptime(args.end[:19], '%Y-%m-%dT%H:%M:%S') + else: + end_time = max(filename_ends) + + date = datetime(start_time.year, start_time.month, start_time.day) + # grid_dir = os.path.join('/data/LCFA-production/', 'grid_test') + # outpath = grid_dir+'/20%s' %(date.strftime('%y/%b/%d')) + outpath = os.path.join(args.outdir, '20%s' % (date.strftime('%y/%b/%d'))) + if os.path.exists(outpath) == False: + os.makedirs(outpath) + # subprocess.call(['chmod', 'a+w', outpath, grid_dir+'/20%s' %(date.strftime('%y/%b')), grid_dir+'/20%s' %(date.strftime('%y'))]) + + if args.fixed_grid: + proj_name = 'geos' + + if (args.goes_position != 'none') & (args.goes_sector != 'none'): + resln = nearest_resolution(args) + view = get_GOESR_grid(position=args.goes_position, + view=args.goes_sector, + resolution=resln) + nadir_lon = view['nadir_lon'] + dx = dy = view['resolution'] + nx, ny = view['pixelsEW'], view['pixelsNS'] + geofixcs, grs80lla = get_GOESR_coordsys(sat_lon_nadir=nadir_lon) + + if 'centerEW' in view: + x_ctr, y_ctr = view['centerEW'], view['centerNS'] + elif args.goes_sector == 'meso': + # use ctr_lon, ctr_lat to get the center of the mesoscale FOV + x_ctr, y_ctr, z_ctr = geofixcs.fromECEF( + *grs80lla.toECEF(args.ctr_lon, args.ctr_lat, 0.0)) + elif (args.goes_position != 'none') & (args.goes_sector == 'none'): + # Requires goes_position, a center, and a width. Fully flexible + # in resolution, i.e., doesn't slave it to one of the GOES-R specs + view = get_GOESR_grid(position=args.goes_position, + view='full', + resolution='1.0km') + nadir_lon = view['nadir_lon'] + dx1km = dy1km = view['resolution'] + geofixcs, grs80lla = get_GOESR_coordsys(sat_lon_nadir=nadir_lon) + x_ctr, y_ctr, z_ctr = geofixcs.fromECEF( + *grs80lla.toECEF(args.ctr_lon, args.ctr_lat, 0.0)) + + # Convert the specified resolution in km given by args.dx to + # a delta in fixed grid coordinates using the 1 km delta from the + # GOES-R PUG. + dx, dy = args.dx * dx1km, args.dy * dy1km + nx, ny = int(args.width / args.dx), int(args.height / args.dy) + else: + raise ValueError("Gridding on the fixed grid requires " + "goes_position and dx. For goes_sector='meso', also specify " + "ctr_lon and ctr_lat. Without goes_sector, also include width " + "and height.") + # Need to use +1 here to convert to xedge, yedge expected by gridder + # instead of the pixel centroids that will result in the final image + nx += 1 + ny += 1 + x_bnd = (np.arange(nx, dtype='float') - (nx) / 2.0) * dx + x_ctr + 0.5 * dx + y_bnd = (np.arange(ny, dtype='float') - (ny) / 2.0) * dy + y_ctr + 0.5 * dy + log.debug(("initial x,y_ctr", x_ctr, y_ctr)) + log.debug(("initial x,y_bnd", x_bnd.shape, y_bnd.shape)) + x_bnd = np.asarray([x_bnd.min(), x_bnd.max()]) + y_bnd = np.asarray([y_bnd.min(), y_bnd.max()]) + + geofixcs, grs80lla = get_GOESR_coordsys(sat_lon_nadir=nadir_lon) + ctr_lon, ctr_lat, ctr_alt = grs80lla.fromECEF( + *geofixcs.toECEF(x_ctr, y_ctr, 0.0)) + fixed_grid = geofixcs + log.debug((x_bnd, y_bnd, dx, dy, nx, ny)) + + output_writer = partial(write_cf_netcdf_fixedgrid, nadir_lon=nadir_lon) + else: + # Default + proj_name = 'latlong' + output_writer = write_cf_netcdf_latlon + ctr_lat = float(args.ctr_lat) + ctr_lon = float(args.ctr_lon) + dx_km = float(args.dx) * 1.0e3 + dy_km = float(args.dy) * 1.0e3 + width, height = 1000.0 * float(args.width), 1000.0 * float(args.height) + x_bnd_km = (-width / 2.0, width / 2.0) + y_bnd_km = (-height / 2.0, height / 2.0) + dx, dy, x_bnd, y_bnd = dlonlat_at_grid_center(ctr_lat, ctr_lon, + dx=dx_km, dy=dy_km, + x_bnd=x_bnd_km, y_bnd=y_bnd_km) + + # tuples of the corners + corners = np.vstack([(x_bnd[0], y_bnd[0]), (x_bnd[0], y_bnd[1]), + (x_bnd[1], y_bnd[1]), (x_bnd[1], y_bnd[0])]) + # print(x_bnd, y_bnd) + + if args.is_lma: + gridder = grid_h5flashfiles + output_filename_prefix = 'LMA' + else: + gridder = grid_GLM_flashes + output_filename_prefix = 'GLM' + + grid_kwargs = dict(proj_name=proj_name, + base_date=date, do_3d=False, + dx=dx, dy=dy, frame_interval=float(args.dt), + x_bnd=x_bnd, y_bnd=y_bnd, + ctr_lat=ctr_lat, ctr_lon=ctr_lon, outpath=outpath, + min_points_per_flash=min_events, + output_writer=output_writer, subdivide=args.subdivide_grid, + output_filename_prefix=output_filename_prefix, + output_kwargs={'scale_and_offset': args.output_scale_and_offset}, + spatial_scale_factor=1.0) + + if args.fixed_grid: + grid_kwargs['fixed_grid'] = True + grid_kwargs['nadir_lon'] = nadir_lon + if args.split_events: + grid_kwargs['clip_events'] = True + if min_groups is not None: + grid_kwargs['min_groups_per_flash'] = min_groups + if args.is_lma: + grid_kwargs['energy_grids'] = True + else: + grid_kwargs['energy_grids'] = ('total_energy',) + if (proj_name == 'pixel_grid') or (proj_name == 'geos'): + grid_kwargs['pixel_coords'] = fixed_grid + grid_kwargs['ellipse_rev'] = args.ellipse_rev + # if args.corner_points: + # grid_kwargs['corner_pickle'] = args.corner_points + return gridder, glm_filenames, start_time, end_time, grid_kwargs + + +if __name__ == '__main__': + parser = create_parser() + args = parser.parse_args() + + from multiprocessing import freeze_support + + freeze_support() + gridder, glm_filenames, start_time, end_time, grid_kwargs = grid_setup(args) + gridder(glm_filenames, start_time, end_time, **grid_kwargs) diff --git a/gridded_glm/bin/env.sh b/gridded_glm/bin/env.sh new file mode 100644 index 0000000..0bcdaa0 --- /dev/null +++ b/gridded_glm/bin/env.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# encoding: utf-8 +# Copyright (C) 2019 Space Science and Engineering Center (SSEC), +# University of Wisconsin-Madison. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# This file is part of the CSPP Geo Gridded GLM software package. CSPP Geo +# Gridded GLM takes GOES GLM Level 2 LCFA files and grids them to the ABI +# fixed grid. It does this using the open source glmtools python package by +# Eric Bruning. + +if [ -z "$CSPP_GEO_GGLM_HOME" ]; then + export CSPP_GEO_GGLM_HOME="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" +fi + +unset PYTHONPATH +unset LD_LIBRARY_PATH + +export PATH=$PATH:$CSPP_GEO_GGLM_HOME/bin:$CSPP_GEO_GGLM_HOME/opt/conda/bin \ No newline at end of file diff --git a/gridded_glm/bin/make_glm_grids.sh b/gridded_glm/bin/make_glm_grids.sh new file mode 100755 index 0000000..d844d91 --- /dev/null +++ b/gridded_glm/bin/make_glm_grids.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# encoding: utf-8 +# Copyright (C) 2019 Space Science and Engineering Center (SSEC), +# University of Wisconsin-Madison. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# This file is part of the CSPP Geo Gridded GLM software package. CSPP Geo +# Gridded GLM takes GOES GLM Level 2 LCFA files and grids them to the ABI +# fixed grid. It does this using the open source glmtools python package by +# Eric Bruning. + +if [ -z "$CSPP_GEO_GGLM_HOME" ]; then + export CSPP_GEO_GGLM_HOME="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" +fi + +# Setup necessary environments +source $CSPP_GEO_GGLM_HOME/bin/env.sh + +# Call the python module to do the processing, passing all arguments +python3 $CSPP_GEO_GGLM_HOME/bin/_make_glm_grids.py "$@" diff --git a/gridded_glm/bin/run.sh b/gridded_glm/bin/run.sh deleted file mode 100644 index 5850002..0000000 --- a/gridded_glm/bin/run.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -echo "TODO" -exit 1 \ No newline at end of file -- GitLab