diff --git a/buildbucket/README.md b/buildbucket/README.md
index d93b9597ee49adaa5a1388d224aa200523378108..c5ebc01ffc03d7d2df1f808ca25ace9973e87e99 100644
--- a/buildbucket/README.md
+++ b/buildbucket/README.md
@@ -26,7 +26,7 @@ docker push gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-gridded-glm/buildbucket:
To create the Gridded GLM package with the version number 1.0.0 in your current directory:
```bash
-docker run --rm -v "${PWD}":/dock gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-gridded-glm/buildbucket:latest package.sh 1.0.0
+docker run --rm -v "${PWD}":/dock gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-gridded-glm/buildbucket:latest ./package.sh 1.0.0
```
If some things need to be customized you can specify various environment
diff --git a/gridded_glm/bin/_make_glm_grids.py b/gridded_glm/bin/_make_glm_grids.py
index dc09f471c22d740f54e16757870e597ccf667aa9..5766989c789328777564e9f1f9a78b93aec2edfb 100644
--- a/gridded_glm/bin/_make_glm_grids.py
+++ b/gridded_glm/bin/_make_glm_grids.py
@@ -1,131 +1,97 @@
-import argparse
+#!/usr/bin/env python3
-parse_desc = """Grid GLM flash data. The start and end times can be specified
+import numpy as np
+from datetime import datetime
+import os
+from functools import partial
+from lmatools.grid.make_grids import write_cf_netcdf_latlon, write_cf_netcdf_noproj, write_cf_netcdf_fixedgrid
+from lmatools.grid.make_grids import dlonlat_at_grid_center, grid_h5flashfiles
+from glmtools.grid.make_grids import grid_GLM_flashes
+from glmtools.io.glm import parse_glm_filename
+from lmatools.grid.fixed import get_GOESR_grid, get_GOESR_coordsys
+
+import logging
+
+log = logging.getLogger(__name__)
+
+parse_desc = """Grid GLM flash data.
+
+The start and end times can be specified
independently, or if not provided they will be inferred from the filenames.
Grid spacing is regular in latitude and longitude with the grid box
being sized to match the requested dx, dy at the center of the grid.
-Within the output directory, a year/month/day directory will be created,
-e.g., 2017/Jul/04/, and within that directory the grid files will be created.
-
Therefore, this script can be used to process multiple days and they will
be written to a standardized directory structure.
"""
def create_parser():
+ import argparse
parser = argparse.ArgumentParser(description=parse_desc)
- parser.add_argument(dest='filenames', metavar='filename', nargs='+')
+ parser.add_argument('-v', '--verbose', dest='verbosity', action="count", default=0,
+ help='each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG (default INFO)')
+ parser.add_argument('-l', '--log', dest="log_fn", default=None,
+ help="specify the log filename")
parser.add_argument('-o', '--output-dir', metavar='directory',
- default='.', action='store')
- parser.add_argument('--ctr-lat', metavar='latitude', required=False,
- action='store', type=float, help='center latitude')
- parser.add_argument('--ctr-lon', metavar='longitude', required=False,
- action='store', type=float, help='center longitude')
+ default='.')
+ parser.add_argument('--ctr-lat', metavar='latitude',
+ type=float, help='center latitude')
+ parser.add_argument('--ctr-lon', metavar='longitude',
+ type=float, help='center longitude')
parser.add_argument('--start', metavar='yyyy-mm-ddThh:mm:ss',
- action='store',
help='UTC start time, e.g., 2017-07-04T08:00:00')
parser.add_argument('--end', metavar='yyyy-mm-ddThh:mm:ss',
- action='store',
help='UTC end time, e.g., 2017-07-04T09:00:00')
parser.add_argument('--dx', metavar='km',
- action='store', default=10.0, type=float,
+ default=10.0, type=float,
help='approximate east-west grid spacing')
parser.add_argument('--dy', metavar='km',
- action='store', default=10.0, type=float,
+ default=10.0, type=float,
help='approximate north-south grid spacing')
parser.add_argument('--dt', metavar='seconds',
- action='store', default=60.0, type=float,
+ default=60.0, type=float,
help='frame duration')
parser.add_argument('--width', metavar='distance in km',
- action='store', default=400.0,
+ default=400.0,
type=float, help='total width of the grid')
parser.add_argument('--height', metavar='distance in km',
- action='store', default=400.0,
+ default=400.0,
type=float, help='total height of the grid')
parser.add_argument('--nevents', metavar='minimum events per flash',
- type=int, dest='min_events', action='store', default=1,
+ type=int, dest='min_events', default=1,
help='minimum number of events per flash')
parser.add_argument('--ngroups', metavar='minimum groups per flash',
- type=int, dest='min_groups', action='store', default=1,
+ type=int, dest='min_groups', default=1,
help='minimum number of groups per flash')
- parser.add_argument('--fixed-grid', action='store_true',
- help='grid to the geostationary fixed grid')
parser.add_argument('--subdivide-grid', metavar='sqrt(number of subgrids)',
- action='store', type=int, default=1,
- help=("subdivide the grid this many times along "
- "each dimension"))
- parser.add_argument('--goes-position', default='none', action='store',
- help=("One of [east|west|test]. "
- "Also requires goes_sector."))
- parser.add_argument('--goes-sector', default='none',
- action='store',
- help=("One of [full|conus|meso]. "
- "Also requires goes_position. If sector is "
- "meso, ctr_lon and ctr_lat are interpreted as "
- "the ctr_x and ctr_y of the fixed grid"))
- parser.add_argument('--corner-points', metavar='filename.pickle',
- action='store',
- help=("name of file containing a pickled "
- "corner point lookup table"))
- parser.add_argument('--split-events',
- action='store_true',
- help='Split GLM event polygons when gridding')
+ type=int, default=1,
+ help="subdivide the grid this many times along "
+ "each dimension")
+ parser.add_argument('--goes-position',
+ help="One of [east|west|test]. "
+ "Requires '--goes-sector'.")
+ parser.add_argument('--goes-sector',
+ help="One of [full|conus|meso]. "
+ "Requires goes_position. If sector is "
+ "meso, ctr_lon and ctr_lat are interpreted as "
+ "the ctr_x and ctr_y of the fixed grid")
+ # parser.add_argument('--split-events',
+ # action='store_true',
+ # help='Split GLM event polygons when gridding')
parser.add_argument('--ellipse', dest='ellipse_rev', default=-1,
- action='store', type=int,
+ type=int,
help='Lightning ellipse revision. -1 (default)=infer'
' from date in each GLM file, 0=value at launch,'
' 1=late 2018 revision')
parser.add_argument('--float-output', dest='output_scale_and_offset',
- default=True,
action='store_false',
help='write all output variables as floating point')
- parser.add_argument('--lma', dest='is_lma',
- action='store_true',
- help='grid LMA h5 files instead of GLM data')
- # parser.add_argument('-v', dest='verbose', action='store_true',
- # help='verbose mode')
+ parser.add_argument(dest='filenames', metavar='filename', nargs='+')
return parser
-##### END PARSING #####
-
-import numpy as np
-import subprocess, glob
-from datetime import datetime
-import os
-from functools import partial
-
-import logging
-
-
-class MyFormatter(logging.Formatter):
- """ Custom class to allow logging of microseconds"""
- converter = datetime.fromtimestamp
-
- def formatTime(self, record, datefmt=None):
- ct = self.converter(record.created)
- if datefmt:
- s = ct.strftime(datefmt)
- else:
- t = ct.strftime("%Y-%m-%d %H:%M:%S")
- s = "%s,%03d" % (t, record.msecs)
- return s
-
-
-logoutfile = logging.FileHandler("make_GLM_grid.log")
-formatter = MyFormatter(fmt='%(levelname)s %(asctime)s %(message)s',
- datefmt='%Y-%m-%dT%H:%M:%S.%f')
-logoutfile.setFormatter(formatter)
-logging.basicConfig(handlers=[logoutfile],
- level=logging.DEBUG)
-
-# Separate from log setup - actually log soemthign specific to this module.
-log = logging.getLogger(__name__)
-log.info("Starting GLM Gridding")
-
-
def nearest_resolution(args):
""" Uses args.dx to find the closest resolution specified by the
GOES-R PUG. Returns something like "10.0km" that can be used as the
@@ -138,14 +104,35 @@ def nearest_resolution(args):
return resln
-def grid_setup(args):
- from lmatools.grid.make_grids import write_cf_netcdf_latlon, write_cf_netcdf_noproj, write_cf_netcdf_fixedgrid
- from lmatools.grid.make_grids import dlonlat_at_grid_center, grid_h5flashfiles
- from glmtools.grid.make_grids import grid_GLM_flashes
- from glmtools.io.glm import parse_glm_filename
- from lmatools.io.LMA_h5_file import parse_lma_h5_filename
- from lmatools.grid.fixed import get_GOESR_grid, get_GOESR_coordsys
+def get_start_end(filenames, start_time=None, end_time=None):
+ """Compute start and end time of data based on filenames."""
+ base_filenames = [os.path.basename(p) for p in filenames]
+ try:
+ filename_infos = [parse_glm_filename(f) for f in base_filenames]
+ # opsenv, algorithm, platform, start, end, created = parse_glm_filename(f)
+ filename_starts = [info[3] for info in filename_infos]
+ filename_ends = [info[4] for info in filename_infos]
+ except ValueError:
+ filename_starts = None
+ filename_ends = None
+
+ if args.start is not None:
+ start_time = datetime.strptime(args.start, '%Y-%m-%dT%H:%M:%S')
+ elif filename_starts is not None:
+ start_time = min(filename_starts)
+
+ if args.end is not None:
+ end_time = datetime.strptime(args.end, '%Y-%m-%dT%H:%M:%S')
+ elif filename_ends is not None:
+ end_time = max(filename_ends)
+
+ if start_time is None or end_time is None:
+ raise ValueError("Could not determine start/end time")
+ return start_time, end_time
+
+
+def grid_setup(args):
# When passed None for the minimum event or group counts, the gridder will skip
# the check, saving a bit of time.
min_events = int(args.min_events)
@@ -155,127 +142,75 @@ def grid_setup(args):
if min_groups <= 1:
min_groups = None
- if args.is_lma:
- filename_parser = parse_lma_h5_filename
- start_idx = 0
- end_idx = 1
- else:
- filename_parser = parse_glm_filename
- start_idx = 3
- end_idx = 4
-
- glm_filenames = args.filenames
- base_filenames = [os.path.basename(p) for p in glm_filenames]
try:
- filename_infos = [filename_parser(f) for f in base_filenames]
- # opsenv, algorithm, platform, start, end, created = parse_glm_filename(f)
- filename_starts = [info[start_idx] for info in filename_infos]
- filename_ends = [info[end_idx] for info in filename_infos]
+ start_time, end_time = get_start_end(args.filenames, args.start, args.end)
except ValueError:
- log.error("One or more GLM files has a non-standard filename.")
- log.error("Assuming that --start and --end have been passed directly.")
-
- from glmtools.io.glm import parse_glm_filename
- if args.start is not None:
- start_time = datetime.strptime(args.start[:19], '%Y-%m-%dT%H:%M:%S')
- else:
- start_time = min(filename_starts)
- if args.end is not None:
- end_time = datetime.strptime(args.end[:19], '%Y-%m-%dT%H:%M:%S')
- else:
- end_time = max(filename_ends)
+ log.error("Non-standard filenames provided, use --start and --end to specify data times.")
+ raise
date = datetime(start_time.year, start_time.month, start_time.day)
- # grid_dir = os.path.join('/data/LCFA-production/', 'grid_test')
- # outpath = grid_dir+'/20%s' %(date.strftime('%y/%b/%d'))
os.makedirs(args.output_dir, exist_ok=True)
+ proj_name = 'geos'
+
+ if args.goes_position is not None and args.goes_sector is not None:
+ resln = nearest_resolution(args)
+ view = get_GOESR_grid(position=args.goes_position,
+ view=args.goes_sector,
+ resolution=resln)
+ nadir_lon = view['nadir_lon']
+ dx = dy = view['resolution']
+ nx, ny = view['pixelsEW'], view['pixelsNS']
+ geofixcs, grs80lla = get_GOESR_coordsys(sat_lon_nadir=nadir_lon)
- if args.fixed_grid:
- proj_name = 'geos'
-
- if (args.goes_position != 'none') & (args.goes_sector != 'none'):
- resln = nearest_resolution(args)
- view = get_GOESR_grid(position=args.goes_position,
- view=args.goes_sector,
- resolution=resln)
- nadir_lon = view['nadir_lon']
- dx = dy = view['resolution']
- nx, ny = view['pixelsEW'], view['pixelsNS']
- geofixcs, grs80lla = get_GOESR_coordsys(sat_lon_nadir=nadir_lon)
-
- if 'centerEW' in view:
- x_ctr, y_ctr = view['centerEW'], view['centerNS']
- elif args.goes_sector == 'meso':
- # use ctr_lon, ctr_lat to get the center of the mesoscale FOV
- x_ctr, y_ctr, z_ctr = geofixcs.fromECEF(
- *grs80lla.toECEF(args.ctr_lon, args.ctr_lat, 0.0))
- elif (args.goes_position != 'none') & (args.goes_sector == 'none'):
- # Requires goes_position, a center, and a width. Fully flexible
- # in resolution, i.e., doesn't slave it to one of the GOES-R specs
- view = get_GOESR_grid(position=args.goes_position,
- view='full',
- resolution='1.0km')
- nadir_lon = view['nadir_lon']
- dx1km = dy1km = view['resolution']
- geofixcs, grs80lla = get_GOESR_coordsys(sat_lon_nadir=nadir_lon)
+ if 'centerEW' in view:
+ x_ctr, y_ctr = view['centerEW'], view['centerNS']
+ elif args.goes_sector == 'meso':
+ # use ctr_lon, ctr_lat to get the center of the mesoscale FOV
x_ctr, y_ctr, z_ctr = geofixcs.fromECEF(
*grs80lla.toECEF(args.ctr_lon, args.ctr_lat, 0.0))
-
- # Convert the specified resolution in km given by args.dx to
- # a delta in fixed grid coordinates using the 1 km delta from the
- # GOES-R PUG.
- dx, dy = args.dx * dx1km, args.dy * dy1km
- nx, ny = int(args.width / args.dx), int(args.height / args.dy)
- else:
- raise ValueError("Gridding on the fixed grid requires "
- "goes_position and dx. For goes_sector='meso', also specify "
- "ctr_lon and ctr_lat. Without goes_sector, also include width "
- "and height.")
- # Need to use +1 here to convert to xedge, yedge expected by gridder
- # instead of the pixel centroids that will result in the final image
- nx += 1
- ny += 1
- x_bnd = (np.arange(nx, dtype='float') - (nx) / 2.0) * dx + x_ctr + 0.5 * dx
- y_bnd = (np.arange(ny, dtype='float') - (ny) / 2.0) * dy + y_ctr + 0.5 * dy
- log.debug(("initial x,y_ctr", x_ctr, y_ctr))
- log.debug(("initial x,y_bnd", x_bnd.shape, y_bnd.shape))
- x_bnd = np.asarray([x_bnd.min(), x_bnd.max()])
- y_bnd = np.asarray([y_bnd.min(), y_bnd.max()])
-
+ elif args.goes_position is not None and args.goes_sector is None:
+ # Requires goes_position, a center, and a width. Fully flexible
+ # in resolution, i.e., doesn't slave it to one of the GOES-R specs
+ view = get_GOESR_grid(position=args.goes_position,
+ view='full',
+ resolution='1.0km')
+ nadir_lon = view['nadir_lon']
+ dx1km = dy1km = view['resolution']
geofixcs, grs80lla = get_GOESR_coordsys(sat_lon_nadir=nadir_lon)
- ctr_lon, ctr_lat, ctr_alt = grs80lla.fromECEF(
- *geofixcs.toECEF(x_ctr, y_ctr, 0.0))
- fixed_grid = geofixcs
- log.debug((x_bnd, y_bnd, dx, dy, nx, ny))
-
- output_writer = partial(write_cf_netcdf_fixedgrid, nadir_lon=nadir_lon)
+ x_ctr, y_ctr, z_ctr = geofixcs.fromECEF(
+ *grs80lla.toECEF(args.ctr_lon, args.ctr_lat, 0.0))
+
+ # Convert the specified resolution in km given by args.dx to
+ # a delta in fixed grid coordinates using the 1 km delta from the
+ # GOES-R PUG.
+ dx, dy = args.dx * dx1km, args.dy * dy1km
+ nx, ny = int(args.width / args.dx), int(args.height / args.dy)
else:
- # Default
- proj_name = 'latlong'
- output_writer = write_cf_netcdf_latlon
- ctr_lat = float(args.ctr_lat)
- ctr_lon = float(args.ctr_lon)
- dx_km = float(args.dx) * 1.0e3
- dy_km = float(args.dy) * 1.0e3
- width, height = 1000.0 * float(args.width), 1000.0 * float(args.height)
- x_bnd_km = (-width / 2.0, width / 2.0)
- y_bnd_km = (-height / 2.0, height / 2.0)
- dx, dy, x_bnd, y_bnd = dlonlat_at_grid_center(ctr_lat, ctr_lon,
- dx=dx_km, dy=dy_km,
- x_bnd=x_bnd_km, y_bnd=y_bnd_km)
-
- # tuples of the corners
- corners = np.vstack([(x_bnd[0], y_bnd[0]), (x_bnd[0], y_bnd[1]),
- (x_bnd[1], y_bnd[1]), (x_bnd[1], y_bnd[0])])
- # print(x_bnd, y_bnd)
-
- if args.is_lma:
- gridder = grid_h5flashfiles
- output_filename_prefix = 'LMA'
- else:
- gridder = grid_GLM_flashes
- output_filename_prefix = 'GLM'
-
+ raise ValueError("Gridding on the fixed grid requires "
+ "goes_position and dx. For goes_sector='meso', also specify "
+ "ctr_lon and ctr_lat. Without goes_sector, also include width "
+ "and height.")
+ # Need to use +1 here to convert to xedge, yedge expected by gridder
+ # instead of the pixel centroids that will result in the final image
+ nx += 1
+ ny += 1
+ x_bnd = (np.arange(nx, dtype='float') - (nx) / 2.0) * dx + x_ctr + 0.5 * dx
+ y_bnd = (np.arange(ny, dtype='float') - (ny) / 2.0) * dy + y_ctr + 0.5 * dy
+ log.debug(("initial x,y_ctr", x_ctr, y_ctr))
+ log.debug(("initial x,y_bnd", x_bnd.shape, y_bnd.shape))
+ x_bnd = np.asarray([x_bnd.min(), x_bnd.max()])
+ y_bnd = np.asarray([y_bnd.min(), y_bnd.max()])
+
+ geofixcs, grs80lla = get_GOESR_coordsys(sat_lon_nadir=nadir_lon)
+ ctr_lon, ctr_lat, ctr_alt = grs80lla.fromECEF(
+ *geofixcs.toECEF(x_ctr, y_ctr, 0.0))
+ fixed_grid = geofixcs
+ log.debug((x_bnd, y_bnd, dx, dy, nx, ny))
+
+ output_writer = partial(write_cf_netcdf_fixedgrid, nadir_lon=nadir_lon)
+
+ gridder = grid_GLM_flashes
+ output_filename_prefix = 'GLM'
grid_kwargs = dict(proj_name=proj_name,
base_date=date, do_3d=False,
dx=dx, dy=dy, frame_interval=float(args.dt),
@@ -290,28 +225,32 @@ def grid_setup(args):
if args.fixed_grid:
grid_kwargs['fixed_grid'] = True
grid_kwargs['nadir_lon'] = nadir_lon
- if args.split_events:
- grid_kwargs['clip_events'] = True
+ # if args.split_events:
+ grid_kwargs['clip_events'] = True
if min_groups is not None:
grid_kwargs['min_groups_per_flash'] = min_groups
- if args.is_lma:
- grid_kwargs['energy_grids'] = True
- else:
- grid_kwargs['energy_grids'] = ('total_energy',)
+ grid_kwargs['energy_grids'] = ('total_energy',)
if (proj_name == 'pixel_grid') or (proj_name == 'geos'):
grid_kwargs['pixel_coords'] = fixed_grid
grid_kwargs['ellipse_rev'] = args.ellipse_rev
- # if args.corner_points:
- # grid_kwargs['corner_pickle'] = args.corner_points
- return gridder, glm_filenames, start_time, end_time, grid_kwargs
+ return gridder, args.filenames, start_time, end_time, grid_kwargs
if __name__ == '__main__':
+ import sys
parser = create_parser()
args = parser.parse_args()
- from multiprocessing import freeze_support
+ # Configure logging
+ levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
+ logging.basicConfig(level=levels[min(3, args.verbosity)], filename=args.log_fn)
+ if levels[min(3, args.verbosity)] > logging.DEBUG:
+ import warnings
+ warnings.filterwarnings("ignore")
+ log.info("Starting GLM Gridding")
+ log.debug("Starting script with: %s", sys.argv)
+ from multiprocessing import freeze_support
freeze_support()
gridder, glm_filenames, start_time, end_time, grid_kwargs = grid_setup(args)
gridder(glm_filenames, start_time, end_time, **grid_kwargs)