Merge branch '2020-02-10-beta1-high-priority' into 'master'

2020 02 10 beta1 high priority

Closes #14 and #9

See merge request !5

buildbucket/buildbucket_environment.yaml

@@ -27,4 +27,4 @@ dependencies:
     - git+https://github.com/deeplycloudy/lmatools.git@minvaluegrids
     - git+https://github.com/deeplycloudy/stormdrain.git
     - git+https://github.com/deeplycloudy/glmtools.git@master
-    - git+https://github.com/pytroll/satpy.git@master
\ No newline at end of file
+    - git+https://github.com/pytroll/satpy.git@7ae7b274a01e5e170bd2cf0f18ccf53b0bd33ada
\ No newline at end of file

gridded_glm/libexec/_glm_trio_picker.py

@@ -39,13 +39,12 @@ if __name__ == '__main__':
 
     # and run gridded glm a'la:
     #    minute_gridder.sh \
-    #     --goes-position auto --goes-sector full \
+    #     --goes-sector full \
     #     --create-tiles \
     #     ./2020-11-05/CLASS/OR_GLM-L2-LCFA_G16_s20203101529000_e20203101529205_c20203101529215.nc \
     #     ./2020-11-05/CLASS/OR_GLM-L2-LCFA_G16_s20203101529200_e20203101529405_c20203101529430.nc \
     #     ./2020-11-05/CLASS/OR_GLM-L2-LCFA_G16_s20203101529400_e20203101530004_c20203101530021.nc
-    subprocess.run(["cspp-geo-gglm-minute-gridder.sh",
-                    "--goes-position", "auto",
+    subprocess.run(["cspp-geo-gglm.sh",
                     "--goes-sector", "full",
                     "--create-tiles",
                     fileglob[0], fileglob[1], fileglob[2]]

gridded_glm/libexec/_minute_gridder.py

 #!/usr/bin/env python3
 # Based on https://github.com/deeplycloudy/glmtools/blob/master/examples/grid/make_GLM_grids.py
 
-parse_desc = """Grid the past X minutes of GLM flash data, given a single input file.
+parse_desc = """Create one minute NetCDF4 grids (and, optionally, AWIPS-compatible tiles) from GLM flash data.
+
+Example usage:\n
+    cspp-geo-gglm.sh \\
+        --goes-sector conus \\
+        --create-tiles \\
+        -vv \\
+        OR_GLM-L2-LCFA_G17_s20182750032000_e20182750032200_c20182750032225.nc \\
+        OR_GLM-L2-LCFA_G17_s20182750032200_e20182750032400_c20182750032426.nc \\
+        OR_GLM-L2-LCFA_G17_s20182750032400_e20182750033000_c20182750033025.nc
 """
 
 import numpy as np
@@ -26,24 +35,21 @@ log = logging.getLogger(__name__)
 
 def create_parser():
     import argparse
-    parser = argparse.ArgumentParser(description=parse_desc)
+    parser = argparse.ArgumentParser(description=parse_desc, formatter_class=argparse.RawTextHelpFormatter) # RawTextHelpFormatter preserves our newlines in the example usage message
     parser.add_argument('-v', '--verbose', dest='verbosity', action="count", default=0,
-                        help='each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG (default INFO)')
+                        help="each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG\n"
+                             "(default: ERROR)")
     parser.add_argument('-l', '--log', dest="log_fn", default=None,
-                        help="specify the log filename")
-    # from Requirements: "Output is Gridded GLM in the native glmtools NetCDF4 format, with a user option to produce AWIPS-compatible NetCDF tiles as described below"
-    parser.add_argument('-o', '--output-dir', metavar='output directory',
-                        default=os.getcwd())
-    parser.add_argument('--goes-sector', default="full",
-                        help="One of [full|conus|meso]. "
-                             "Requires goes_position. If sector is "
-                             "meso, ctr_lon and ctr_lat are interpreted as "
-                             "the ctr_x and ctr_y of the fixed grid")
-    parser.add_argument('--goes-position', default="auto",
-                        help="One of [east|west|test|auto]. "
-                             "Requires '--goes-sector'.")
+                        help="specify a log filename.\n"
+                             "(default: print to screen).")
+    parser.add_argument('-o', '--output-dir', metavar='OUTPUT_DIR',
+                        default=os.getcwd(), help="output directory (default: use current directory)")
+    parser.add_argument('--goes-sector', default="full", choices=['full', 'conus', 'meso'],
+                        help="If sector is meso, ctr_lon and ctr_lat \n"
+                             "are interpreted as the ctr_x and ctr_y of the fixed grid.\n"
+                             "(default: full)")
     parser.add_argument("-t", "--create-tiles", default=False, action='store_true',
-                        help="create AWIPS-compatible tiles") # FIXME: improve this help text
+                        help="create AWIPS-compatible tiles (default: off)")
     parser.add_argument('--ctr-lat', metavar='latitude',
                         type=float, help='center latitude (required for meso)')
     parser.add_argument('--ctr-lon', metavar='longitude',
@@ -52,38 +58,6 @@ def create_parser():
     parser.add_argument(dest='filenames', metavar='filename', nargs='+')
     return parser
 
-"""
-old arguments for reference
-
-FIXME: remove this whole comment once everything is working
-
-    parser.add_argument('--dx', metavar='km',
-                        default=10.0, type=float,
-                        help='approximate east-west grid spacing')
-    parser.add_argument('--dy', metavar='km',
-                        default=10.0, type=float,
-                        help='approximate north-south grid spacing')
-    parser.add_argument('--dt', metavar='seconds',
-                        default=60.0, type=float,
-                        help='frame duration')
-    parser.add_argument('--width', metavar='distance in km',
-                        default=400.0,
-                        type=float, help='total width of the grid')
-    parser.add_argument('--height', metavar='distance in km',
-                        default=400.0,
-                        type=float, help='total height of the grid')
-    parser.add_argument('--nevents', metavar='minimum events per flash',
-                        type=int, dest='min_events', default=1,
-                        help='minimum number of events per flash')
-    parser.add_argument('--ngroups', metavar='minimum groups per flash',
-                        type=int, dest='min_groups', default=1,
-                        help='minimum number of groups per flash')
-    parser.add_argument('--subdivide-grid', metavar='sqrt(number of subgrids)',
-                        type=int, default=1,
-                        help="subdivide the grid this many times along "
-                             "each dimension")
-"""
-
 
 def get_resolution(args):
     closest_resln = 2.0 # hardcoding resolution to 2.0 for now. see nearest_resolution in make_glm_grids for how we could expose this if we change our minds.
@@ -99,7 +73,7 @@ def get_goes_position(filenames):
         return "west"
 
     # we require that all files are from the same sensor and raise an exception if not
-    raise ValueError("position 'auto' but could not determine position - did you provide a mix of satellites?")
+    raise ValueError("could not determine GOES position - did you provide a mix of satellites?")
 
 
 def get_start_end(filenames, start_time=None, end_time=None):
@@ -145,10 +119,7 @@ def grid_setup(args, work_dir=os.getcwd()):
 
     outputpath = os.path.join(work_dir, "{dataset_name}") # GLMTools expects a template in addition to the path
 
-    if args.goes_position == "auto":
-        goes_position = get_goes_position(args.filenames)
-    else:
-        goes_position = args.goes_position
+    goes_position = get_goes_position(args.filenames)
 
     resln = get_resolution(args)
     view = get_GOESR_grid(position=goes_position,
@@ -222,7 +193,8 @@ if __name__ == '__main__':
 
     # Configure logging
     levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
-    logging.basicConfig(level=levels[min(3, args.verbosity)], filename=args.log_fn)
+    clamp = lambda n, minn, maxn: max(min(maxn, n), minn) # used below to keep us from going off the end of the logging levels
+    logging.basicConfig(level=levels[clamp(args.verbosity, 0, len(levels)-1)], filename=args.log_fn)
     if levels[min(3, args.verbosity)] > logging.DEBUG:
         import warnings
         warnings.filterwarnings("ignore")
@@ -269,8 +241,8 @@ if __name__ == '__main__':
                               source_name="", # You could probably make source_name an empty string. I think it is required by the writer for legacy reasons but isn't actually used for the glm output
                               base_dir=tempdir_path, # base_dir is the output directory. I think blank is the same as current directory.
                               tile_size=(506, 904), # tile_size is set to the size of the GLMF sample tiles we were given and should match the full disk ABI tiles which is what they wanted
-                              check_categories=False) # check_categories is there because of that issue I mentioned where DQF is all valid all the time so there is no way to detect empty tiles unless we ignore the "category" products
-
+                              check_categories=False, # check_categories is there because of that issue I mentioned where DQF is all valid all the time so there is no way to detect empty tiles unless we ignore the "category" products
+                              compress=True)
 
     # pick up output files from the tempdir
     # output looks like: OR_GLM-L2-GLMC-M3_G17_T03_20200925160040.nc