set up warn missing to work in config file settings

examples/config-files/exconfig.py

@@ -33,6 +33,14 @@ settings[constants.DO_MAKE_FORKS_KEY] = False
 # by default each data set will be plotted in it's own range, if you set this
 # value to True, then the maximum of the two ranges will be used to plot both
 settings[constants.USE_SHARED_ORIG_RANGE_KEY] = False
+# should we test for pass/fail when comparing data sets?
+# by default pass/fail testing will be turned on
+settings[constants.DO_TEST_PASSFAIL_KEY] = False
+# should the user be warned about variables they requested that could not be compared?
+# by default this is turned off, if you turn it on missing variables will be reported in
+# warnings on the command line and when doing pass/fail testing they will be considered
+# failing variables and will be reported on any report pages that you generate
+settings[constants.WARN_MISSING_KEY] = True
 
 # the names of the latitude and longitude variables that will be used
 lat_lon_info = {}

pyglance/glance/compare.py

@@ -230,7 +230,7 @@ def collocate_to_file_library_call(a_path, b_path, var_list=None,
                                                               defaultValues,
                                                               requestedNames,
                                                               usedConfigFile,
-                                                              warnIfRequestedVarsUnavailable=runInfo[OPTIONS_WARN_MISSING_KEY],)
+                                                              warnIfRequestedVarsUnavailable=runInfo[WARN_MISSING_KEY], )
 
     # return for lon_lat_data variables will be in the form 
     #{LON_KEY: longitude_data,      LAT_KEY: latitude_data,      INVALID_MASK_KEY: spaciallyInvalidMaskData}
@@ -563,13 +563,16 @@ def report_one_input_library_call (a_path, var_list=None,
         LOG.error("Unable to continue with examination because file (" + pathsTemp[A_FILE_KEY] + ") could not be opened.")
         sys.exit(1)
 
+    # hang onto the warn missing behavior for later
+    warn_missing_vars = runInfo[WARN_MISSING_KEY]
+
     # get information about the names the user requested
     nameStats = {}
     finalNames, nameStats[POSSIBLE_NAMES_KEY], missingNames = config_organizer.resolve_names_one_file(aFile.file_object,
-                                                                                                      defaultValues, # TODO, might need a different default set
+                                                                                                      defaultValues,
                                                                                                       requestedNames,
                                                                                                       usedConfigFile,
-                                                                                                      warnIfRequestedVarsUnavailable=runInfo[OPTIONS_WARN_MISSING_KEY],)
+                                                                                                      warnIfRequestedVarsUnavailable=warn_missing_vars, )
 
     # get info on the global attributes
     globalAttrInfo = {}
@@ -597,7 +600,7 @@ def report_one_input_library_call (a_path, var_list=None,
     # store info about our variables for our main report page
     variableInspections = {}
     # if we're warning about missing variables, add those into our list now
-    if runInfo[OPTIONS_WARN_MISSING_KEY]:
+    if warn_missing_vars :
         for varNameTemp in missingNames:
             variableInspections[varNameTemp] =  _make_error_var_entry(missingNames[varNameTemp])
 
@@ -616,7 +619,7 @@ def report_one_input_library_call (a_path, var_list=None,
             tmpMsg = displayName + " is of a type that cannot be loaded using current file handling libraries " \
                                    "included with Glance. Skipping " + displayName + "."
             LOG.warning(tmpMsg)
-            if runInfo[OPTIONS_WARN_MISSING_KEY]:
+            if warn_missing_vars :
                 variableInspections[explanationName] = _make_error_var_entry(tmpMsg)
             continue
         
@@ -634,7 +637,7 @@ def report_one_input_library_call (a_path, var_list=None,
             tmpMsg = displayName + " data could not be loaded. This variable will not be included in the output report. " + \
                      "The following error was encountered while trying to load this variable:\n" + str(ex)
             LOG.warning(tmpMsg)
-            if runInfo[OPTIONS_WARN_MISSING_KEY]:
+            if warn_missing_vars :
                 variableInspections[explanationName] = _make_error_var_entry(tmpMsg)
             continue
 
@@ -775,7 +778,7 @@ def report_one_input_library_call (a_path, var_list=None,
                      + 'longitude ' + str(good_shape_from_lon_lat) + ' and '
                      + 'latitude '  + str(good_shape_from_lon_lat) + ' variables.')
             LOG.warning(message)
-            if runInfo[OPTIONS_WARN_MISSING_KEY]:
+            if warn_missing_vars :
                 variableInspections[explanationName] = _make_error_var_entry(message)
         
     # the end of the loop to examine all the variables
@@ -839,7 +842,7 @@ def report_two_inputs_library_call (a_path, b_path, var_list=None,
                                                                                                                 options_set,
                                                                                                                 requestedVars = var_list)
     do_pass_fail = runInfo[DO_TEST_PASSFAIL_KEY]
-    warn_on_missing = runInfo[OPTIONS_WARN_MISSING_KEY] if OPTIONS_WARN_MISSING_KEY in runInfo else False
+    warn_on_missing = runInfo[WARN_MISSING_KEY]
 
     # note some of this information for debugging purposes
     LOG.debug('paths: ' +           str(pathsTemp))
@@ -1276,7 +1279,7 @@ def stats_two_inputs_library_call(afn, bfn, var_list=None,
     default_min_r_squared_coeff = defaultValues[MIN_OK_R_SQUARED_COEFF_KEY] if MIN_OK_R_SQUARED_COEFF_KEY in defaultValues else None
     do_pass_fail = options_set[DO_TEST_PASSFAIL_KEY] # TODO, once stats respects config files # do_pass_fail = runInfo[DO_TEST_PASSFAIL_KEY]
     has_failed = False # for keeping track of if each variable comparison failed
-    warn_missing = options_set[OPTIONS_WARN_MISSING_KEY] if OPTIONS_WARN_MISSING_KEY in options_set else False
+    warn_missing = options_set[WARN_MISSING_KEY] if WARN_MISSING_KEY in options_set else False # TODO, once stats respect config, get this from the runInfo
     
     # open the files
     a_file_path = pathsTemp[A_FILE_KEY]
@@ -1493,7 +1496,7 @@ def stats_one_input_library_call (afn, var_list=None, options_set=None, do_docum
         print("Configuration file: " + options_set[OPTIONS_CONFIG_FILE_KEY], file=output_channel, )
 
     # figure out the variable names and their individual settings
-    warn_missing = options_set[OPTIONS_WARN_MISSING_KEY] if OPTIONS_WARN_MISSING_KEY in options_set else False
+    warn_missing = options_set[WARN_MISSING_KEY] if WARN_MISSING_KEY in options_set else False
     available_variable_names = filesInfo[COMMON_VAR_NAMES_KEY]
     default_missing_val = defaultValues[FILL_VALUE_KEY] if FILL_VALUE_KEY in defaultValues else None
 

pyglance/glance/config_organizer.py

@@ -30,6 +30,7 @@ glance_setting_defaults = {
                                DETAIL_DPI_KEY:             150,
                                THUMBNAIL_DPI_KEY:          50,
                                DO_TEST_PASSFAIL_KEY:       True,
+                               WARN_MISSING_KEY:           False,
                           }
 
 # these are the built in longitude/latitude defaults
@@ -395,6 +396,8 @@ _STOMPABLE_OPTIONS = {
                         OPTIONS_IMAGES_ON_FAIL_KEY: False,
                         DO_MAKE_FORKS_KEY:          False,
                         DO_TEST_PASSFAIL_KEY:       True,
+                        WARN_MISSING_KEY:           False,
+                        # TODO add nolonlat to this
                      }
 def _warn_for_stomped_commandline_options(optionSet) :
     """
@@ -485,7 +488,6 @@ def load_config_or_options(aPath, bPath, optionsSet, requestedVars = [ ]) :
             # line derived value that we set here)
             runInfo[DO_MAKE_REPORT_KEY]         = not optionsSet[OPTIONS_NO_REPORT_KEY]      if OPTIONS_NO_REPORT_KEY      in optionsSet else False
             runInfo[USE_NO_LON_OR_LAT_VARS_KEY] =     optionsSet[USE_NO_LON_OR_LAT_VARS_KEY] if USE_NO_LON_OR_LAT_VARS_KEY in optionsSet else False
-            runInfo[OPTIONS_WARN_MISSING_KEY]   =     optionsSet[OPTIONS_WARN_MISSING_KEY]   if OPTIONS_WARN_MISSING_KEY   in optionsSet else False
             
             # get everything from the config file
             runInfo.update(glanceRunConfig.settings)
@@ -510,7 +512,7 @@ def load_config_or_options(aPath, bPath, optionsSet, requestedVars = [ ]) :
         runInfo[DO_MAKE_IMAGES_KEY]         = not optionsSet[OPTIONS_NO_IMAGES_KEY]
         runInfo[DO_IMAGES_ONLY_ON_FAIL_KEY] = optionsSet[OPTIONS_IMAGES_ON_FAIL_KEY]
         runInfo[DO_MAKE_FORKS_KEY]          = optionsSet[DO_MAKE_FORKS_KEY]
-        runInfo[OPTIONS_WARN_MISSING_KEY]   = optionsSet[OPTIONS_WARN_MISSING_KEY]
+        runInfo[WARN_MISSING_KEY]           = optionsSet[WARN_MISSING_KEY]
         runInfo[DO_TEST_PASSFAIL_KEY]       = optionsSet[DO_TEST_PASSFAIL_KEY]
         
         # only record these if we are using lon/lat
@@ -604,7 +606,7 @@ def parse_arguments (version_string, commands_list, commands_help_text, ) :
                         help="enable logging of debug output (warning: this will generate far more messages)")
     parser.add_argument('-n', '--version', version=version_string, action="version",
                         help="print the Glance version")
-    parser.add_argument('--warnmissingvars', dest=OPTIONS_WARN_MISSING_KEY,
+    parser.add_argument('--warnmissingvars', dest=WARN_MISSING_KEY,
                         action="store_true", default=False,
                         help="emit warnings if variables you request by name are missing from your input files; "
                              "will also cause missing variables to appear in reports and affect pass/fail testing")
@@ -699,7 +701,7 @@ def convert_options_to_dict (options) :
     tempOptions = { }
 
     # warning defaults
-    tempOptions[OPTIONS_WARN_MISSING_KEY]   = options.warnmissing
+    tempOptions[WARN_MISSING_KEY]   = options.warnmissing
 
     # variable defaults
     tempOptions[EPSILON_KEY]                = options.epsilon

pyglance/glance/constants.py

@@ -47,6 +47,7 @@ USE_NO_LON_OR_LAT_VARS_KEY = 'noLonLatVars'
 SHORT_CIRCUIT_DIFFS_KEY    = 'short_circuit_diffs'
 USE_CUSTOM_PROJ_KEY        = 'use_custom_projection'
 PARSABLE_OUTPUT_KEY        = 'parsable_output'
+WARN_MISSING_KEY           = 'warnmissing'
 
 # high level pass fail settings
 EPSILON_FAIL_TOLERANCE_KEY = 'epsilon_failure_tolerance'
@@ -231,7 +232,6 @@ OPTIONS_LAT_VAR_NAME_KEY   = 'latitudeVar'
 OPTIONS_LON_VAR_NAME_KEY   = 'longitudeVar'
 OPTIONS_LONLAT_EPSILON_KEY = 'lonlatepsilon'
 OPTIONS_RE_TO_STRIP_KEY    = 'regularExpressionsToStrip'
-OPTIONS_WARN_MISSING_KEY   = 'warnmissing'
 
 # values used by the reports