Commit e539b47b authored by Eva Schiffer's avatar Eva Schiffer
Browse files

stats calls with one input file will now respect config files

parent bd4120bf
...@@ -1177,13 +1177,30 @@ def stats_two_inputs_library_call(afn, bfn, var_list=None, ...@@ -1177,13 +1177,30 @@ def stats_two_inputs_library_call(afn, bfn, var_list=None,
can also be used as a library routine, simply pass in an output channel can also be used as a library routine, simply pass in an output channel
and/or use the returned dictionary of statistics for your own form of and/or use the returned dictionary of statistics for your own form of
display. display.
TODO, should this move to a different file?
""" """
# set some values for defaults # set some values for defaults
var_list = [] if var_list is None else var_list var_list = ['.*', ] if var_list is None else var_list
options_set = {} if options_set is None else options_set options_set = {} if options_set is None else options_set
"""
# load the user settings from either the command line or a user defined config file
if len(var_list) <= 0:
var_list = ['.*', ]
pathsTemp, runInfo, defaultValues, requestedNames, usedConfigFile = \
config_organizer.load_config_or_options(afn,
bfn,
options_set,
requestedVars=var_list)
print("pathsTemp: " + str(pathsTemp))
print("runInfo: " + str(runInfo))
print("defaultValues: " + str(defaultValues))
print("requestedNames: " + str(requestedNames))
print("usedConfigFile: " + str(usedConfigFile))
print("options_set: " + str(options_set))
"""
# unpack some options # unpack some options
epsilon_val = options_set[EPSILON_KEY] epsilon_val = options_set[EPSILON_KEY]
missing_val = options_set[OPTIONS_FILL_VALUE_KEY] missing_val = options_set[OPTIONS_FILL_VALUE_KEY]
...@@ -1273,55 +1290,105 @@ def stats_two_inputs_library_call(afn, bfn, var_list=None, ...@@ -1273,55 +1290,105 @@ def stats_two_inputs_library_call(afn, bfn, var_list=None,
return status_code return status_code
# note: if we aren't doing pass/fail, stats will not return anything # note: if we aren't doing pass/fail, stats will not return anything
def stats_one_input_library_call (afn, var_list=None, options_set=None, do_document=False, output_channel=sys.stdout): def stats_one_input_library_call (afn, var_list=None, options_set=None, do_document=False,
output_channel=sys.stdout):
""" """
this method handles the actual work of generating stats for a single input file and this method handles the actual work of generating stats for a single input file and
can also be used as a library routine, simply pass in an output channel can also be used as a library routine, simply pass in an output channel
and/or use the returned dictionary of statistics for your own form of and/or use the returned dictionary of statistics for your own form of
display. display.
TODO, should this move to a different file?
""" """
# set some values for defaults # set some values for defaults
var_list = [ ] if var_list is None else var_list var_list = ['.*',] if var_list is None else var_list
options_set = { } if options_set is None else options_set options_set = { } if options_set is None else options_set
# unpack some options # load the user settings from either the command line or a user defined config file
missing_val = options_set[OPTIONS_FILL_VALUE_KEY] if len(var_list) <= 0 :
var_list = ['.*',]
LOG.debug ("file a: " + afn) pathsTemp, runInfo, defaultValues, requestedNames, usedConfigFile = \
config_organizer.load_config_or_options(afn,
None, # there is no B path
options_set,
requestedVars=var_list)
# open the file # open the file
filesInfo = open_and_process_files([afn]) aFilePath = pathsTemp[A_FILE_KEY]
aFile = filesInfo[afn][FILE_OBJECT_KEY] LOG.debug("Opening file A: " + aFilePath)
filesInfo = open_and_process_files([aFilePath])
aFile = filesInfo[aFilePath][FILE_OBJECT_KEY]
# give info on the file and if we used a config file
print('-' * 32, file=output_channel, )
print("File: " + aFilePath, file=output_channel, )
if usedConfigFile :
print("Configuration file: " + options_set[OPTIONS_CONFIG_FILE_KEY], file=output_channel, )
# figure out the variable names and their individual settings # figure out the variable names and their individual settings
if len(var_list) <= 0 :
var_list = ['.*']
warn_missing = options_set[OPTIONS_WARN_MISSING_KEY] if OPTIONS_WARN_MISSING_KEY in options_set else False warn_missing = options_set[OPTIONS_WARN_MISSING_KEY] if OPTIONS_WARN_MISSING_KEY in options_set else False
names = config_organizer.parse_varnames( filesInfo[COMMON_VAR_NAMES_KEY], var_list, available_variable_names = filesInfo[COMMON_VAR_NAMES_KEY]
epsilon=None, missing=missing_val, default_missing_val = defaultValues[FILL_VALUE_KEY] if FILL_VALUE_KEY in defaultValues else None
warn_unfound=warn_missing, )
LOG.debug(str(names)) # if we have command line input, we still need to parse the variable names
doc_each = do_document and len(names)==1 if not usedConfigFile :
doc_atend = do_document and len(names)!=1 temp_names = config_organizer.parse_varnames(available_variable_names, requestedNames,
epsilon=None, missing=default_missing_val,
warn_unfound=warn_missing, )
requestedNames = { }
for name, epsilon, missing in sorted(temp_names, key=lambda X: X[0]) :
requestedNames[name] = {
VARIABLE_TECH_NAME_KEY: name,
# note, since this is an inspection report, the epsilon is not needed
FILL_VALUE_KEY: missing,
}
num_requested_vars = len(requestedNames)
LOG.debug("Trying to analyze variable names: " + str(requestedNames))
for name, epsilon, missing in sorted(names, key=lambda X:X[0]): # are we going to put documentation of what the stats mean after each stat or only at the end?
doc_each = do_document and num_requested_vars == 1
doc_atend = do_document and num_requested_vars != 1
# if we have no variables, something has gone wrong
if len(requestedNames) <= 0 and warn_missing :
LOG.warning("Unable to find any selected variables to analyze.")
for display_name in sorted(requestedNames) :
if VARIABLE_TECH_NAME_KEY not in requestedNames[display_name] :
LOG.warning("Variable with display name of \"" + display_name + "\" has no technical name given. "
"This variable cannot be processed without a techincal name.")
continue
tech_name = requestedNames[display_name][VARIABLE_TECH_NAME_KEY]
explain_name = display_name + "(" + tech_name + ")" if display_name != tech_name else display_name
if tech_name not in available_variable_names :
if warn_missing :
LOG.warning("Requested variable \"" + explain_name + "\" is not available in the file. "
"Unable to process this variable.")
continue
# make sure that it's possible to load this variable # make sure that it's possible to load this variable
if not(aFile.is_loadable_type(name)) : if not(aFile.is_loadable_type(tech_name)) :
LOG.warning(name + " is of a type that cannot be loaded using current file handling libraries included with Glance." + LOG.warning("\"" + explain_name + "\" is of a type that cannot be loaded using current file handling "
" Skipping " + name + ".") "libraries included with Glance." +
" Skipping " + explain_name + ".")
continue continue
aData = aFile[name] # load the variable data, filtering as needed
aData = load_variable_data(aFile, tech_name,
amiss = missing dataFilter=requestedNames[display_name][FILTER_FUNCTION_A_KEY] if FILTER_FUNCTION_A_KEY in requestedNames[display_name] else None,
if missing is None: variableToFilterOn=requestedNames[display_name][VAR_FILTER_NAME_A_KEY] if VAR_FILTER_NAME_A_KEY in requestedNames[display_name] else None,
amiss = aFile.missing_value(name) variableBasedFilter=requestedNames[display_name][VAR_FILTER_FUNCTION_A_KEY] if VAR_FILTER_FUNCTION_A_KEY in requestedNames[display_name] else None,
LOG.debug('analyzing %s with missing data value %s' % (name,amiss)) altVariableFileObject=dataobj.FileInfo(requestedNames[display_name][VAR_FILTER_ALT_FILE_A_KEY]).file_object if VAR_FILTER_ALT_FILE_A_KEY in requestedNames[display_name] else None,
fileDescriptionForDisplay="file A")
amiss = requestedNames[display_name][FILL_VALUE_KEY] if FILL_VALUE_KEY in requestedNames[display_name] else default_missing_val
# if we still don't have a fill value, try to get it from the file
if amiss is None:
amiss = aFile.missing_value(tech_name)
LOG.debug('analyzing %s with missing data value %s' % (explain_name, amiss))
print('-' * 32, file=output_channel,) print('-' * 32, file=output_channel,)
print(name, file=output_channel,) print(explain_name, file=output_channel,)
print('', file=output_channel,) print('', file=output_channel,)
variable_stats = statistics.StatisticalInspectionAnalysis.withSimpleData(aData, amiss) variable_stats = statistics.StatisticalInspectionAnalysis.withSimpleData(aData, amiss)
lal = list(variable_stats.dictionary_form().items()) lal = list(variable_stats.dictionary_form().items())
...@@ -1678,6 +1745,12 @@ def main(): ...@@ -1678,6 +1745,12 @@ def main():
do_doc = (options.verbose or options.debug) do_doc = (options.verbose or options.debug)
to_return = 0 to_return = 0
# if there are requested variables and a config file, warn the user that we're going to ignore their request
if len(variables) > 0 and tempOptions[OPTIONS_CONFIG_FILE_KEY] is not None:
LOG.warning("User requested specific variables (" + str(variables) + ") on the command line that will be "
"overridden by the requested configuration file. Requested variables from the command line will be ignored. "
"Please list your desired variables in your configuration file.")
# if we were given an output path use that to create the stats # if we were given an output path use that to create the stats
toPrintTo = sys.stdout toPrintTo = sys.stdout
outpath = clean_path(options.outputpath) outpath = clean_path(options.outputpath)
...@@ -1714,6 +1787,7 @@ def main(): ...@@ -1714,6 +1787,7 @@ def main():
options_set=tempOptions, options_set=tempOptions,
do_document=do_doc, do_document=do_doc,
output_channel=toPrintTo) output_channel=toPrintTo)
# TODO handle the return code differently for pass/fail testing
# close our text output file if needed # close our text output file if needed
if fileForOutput is not None: if fileForOutput is not None:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment