Skip to content
Snippets Groups Projects
Commit 68a8b2e0 authored by (no author)'s avatar (no author)
Browse files

changed so that variable input from config file and some internal storage uses... 

changed so that variable input from config file and some internal storage uses display name (or a more abstract variable key) in order to allow multiple virtual data sets from the same variable to be present in a single configuration file

git-svn-id: https://svn.ssec.wisc.edu/repos/glance/trunk@55 8a9318a1-56ba-4d59-b755-99d26321be01
parent ff4e7485
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
pyglance/glance/compare.py
+ 68
− 59
View file @ 68a8b2e0
......@@ -163,26 +163,36 @@ def _resolve_names(fileAObject, fileBObject, defaultValues,
# check each of the names the user asked for to see if it is either in the list of common names
# or, if the user asked for an alternate name mapping in file B, if the two mapped names are in
# files A and B respectively
for name in requestedNames : # TODO rekey on display name?
for dispName in requestedNames :
name_b = name
if ('alternate_name_in_B' in requestedNames[name]) :
name_b = requestedNames[name_b]['alternate_name_in_B']
# hang on to info on the current variable
currNameInfo = requestedNames[dispName]
if (name in fileCommonNames) | \
(requestedNames[name].has_key('alternate_name_in_B') and
(name in nameComparison['uniqueToAVars']) and
(name_b in nameComparison['uniqueToBVars'])) :
finalNames[name] = defaultValues.copy()
finalNames[name]['variable_name'] = name
finalNames[name].update(requestedNames[name])
# get the variable name
if 'variable_name' in currNameInfo :
name = currNameInfo['variable_name']
name_b = name
# load the missing value if it was not provided
if finalNames[name]['missing_value'] is None :
finalNames[name]['missing_value'] = fileAObject.missing_value(name)
if not('missing_value_alt_in_b' in finalNames[name]) or \
(finalNames[name]['missing_value_alt_in_b'] is None) :
finalNames[name]['missing_value_alt_in_b'] = fileBObject.missing_value(name_b)
if ('alternate_name_in_B' in currNameInfo) :
name_b = currNameInfo['alternate_name_in_B']
if (name in fileCommonNames) | \
(currNameInfo.has_key('alternate_name_in_B') and
(name in nameComparison['uniqueToAVars']) and
(name_b in nameComparison['uniqueToBVars'])) :
finalNames[dispName] = defaultValues.copy()
finalNames[dispName]['display_name'] = dispName
finalNames[dispName].update(currNameInfo)
# load the missing value if it was not provided
if finalNames[dispName]['missing_value'] is None :
finalNames[dispName]['missing_value'] = fileAObject.missing_value(name)
if not('missing_value_alt_in_b' in finalNames[dispName]) or \
(finalNames[dispName]['missing_value_alt_in_b'] is None) :
finalNames[dispName]['missing_value_alt_in_b'] = fileBObject.missing_value(name_b)
else :
LOG.warn('No technical variable name was given for the entry described as "' + dispName + '". ' +
'Skipping this variable.')
else:
# format command line input similarly to the stuff from the config file
print (requestedNames)
......@@ -883,30 +893,28 @@ python -m glance
# go through each of the possible variables in our files
# and make a report section with images for whichever ones we can
for name in finalNames:
for varKey in finalNames:
# pull out the information for this variable analysis run
varRunInfo = finalNames[name].copy()
varRunInfo = finalNames[varKey].copy()
# make some local copies of our name info for display and labeling
displayName = name
if (varRunInfo.has_key('display_name')) :
technicalName = varRunInfo['variable_name']
displayName = technicalName
if 'display_name' in varRunInfo :
displayName = varRunInfo['display_name']
explanationName = name
if (varRunInfo.has_key('alternate_name_in_B')) :
explanationName = explanationName + " / " + varRunInfo['alternate_name_in_B']
explanationName = displayName + '(' + explanationName + ')'
print('analyzing: ' + explanationName + ')')
explanationName = technicalName
# if B has an alternate variable name, figure that out
has_alt_B_variable = False
b_variable = varRunInfo['variable_name']
if (varRunInfo.has_key('alternate_name_in_B')) :
has_alt_B_variable = True
b_variable = technicalName
if 'alternate_name_in_B' in varRunInfo :
b_variable = varRunInfo['alternate_name_in_B']
explanationName = explanationName + " / " + b_variable
explanationName = displayName + ' (' + explanationName + ')'
print('analyzing: ' + explanationName + ')')
# get the data for the variable
aData = aFile[varRunInfo['variable_name']]
aData = aFile[technicalName]
bData = bFile[b_variable]
# apply any data filter functions we may have
......@@ -923,23 +931,23 @@ python -m glance
(bData.shape == longitudeCommon.shape)) :
# build a dictionary of information on the variable
variableAnalysisInfo[varRunInfo['variable_name']] = {}
variableAnalysisInfo[varRunInfo['variable_name']]['data'] = {'A': aData,
'B': bData}
variableAnalysisInfo[varRunInfo['variable_name']]['var_stats'] = delta.summarize(aData, bData,
varRunInfo['epsilon'],
(varRunInfo['missing_value'],
varRunInfo['missing_value_alt_in_b']),
spaciallyInvalidMaskA, spaciallyInvalidMaskB)
variableAnalysisInfo[varKey] = {}
variableAnalysisInfo[varKey]['data'] = {'A': aData,
'B': bData}
variableAnalysisInfo[varKey]['var_stats'] = delta.summarize(aData, bData,
varRunInfo['epsilon'],
(varRunInfo['missing_value'],
varRunInfo['missing_value_alt_in_b']),
spaciallyInvalidMaskA, spaciallyInvalidMaskB)
# add a little additional info to our variable run info before we squirrel it away
varRunInfo['time'] = datetime.datetime.ctime(datetime.datetime.now())
passedFraction = (1.0 - variableAnalysisInfo[name]['var_stats']
passedFraction = (1.0 - variableAnalysisInfo[varKey]['var_stats']
['Numerical Comparison Statistics']['diff_outside_epsilon_fraction'])
varRunInfo['did_pass'] = _check_pass_or_fail(varRunInfo,
variableAnalysisInfo[name]['var_stats'],
variableAnalysisInfo[varKey]['var_stats'],
defaultValues)
variableAnalysisInfo[varRunInfo['variable_name']]['run_info'] = varRunInfo
variableAnalysisInfo[varRunInfo['variable_name']]['exp_name'] = explanationName
variableAnalysisInfo[varKey]['run_info'] = varRunInfo
variableAnalysisInfo[varKey]['exp_name'] = explanationName
# if we can't compare the variable, we should tell the user
else :
......@@ -957,12 +965,12 @@ python -m glance
# loop to create the images for all our variables
if (runInfo['shouldIncludeImages']) :
for name in variableAnalysisInfo :
for varKey in variableAnalysisInfo :
# create the images comparing that variable
print("\tcreating figures for: " + variableAnalysisInfo[name]['exp_name'])
plot.plot_and_save_figure_comparison(variableAnalysisInfo[name]['data']['A'],
variableAnalysisInfo[name]['data']['B'],
variableAnalysisInfo[name]['run_info'],
print("\tcreating figures for: " + variableAnalysisInfo[varKey]['exp_name'])
plot.plot_and_save_figure_comparison(variableAnalysisInfo[varKey]['data']['A'],
variableAnalysisInfo[varKey]['data']['B'],
variableAnalysisInfo[varKey]['run_info'],
files['file A']['path'],
files['file B']['path'],
latitudeA, longitudeA,
......@@ -972,32 +980,33 @@ python -m glance
spaciallyInvalidMaskB,
outputPath, True,
doFork=runInfo['doFork'])
print("\tfinished creating figures for: " + variableAnalysisInfo[name]['exp_name'])
print("\tfinished creating figures for: " + variableAnalysisInfo[varKey]['exp_name'])
# reports are fast, so the parent thread will just do this
# generate our general report pages once we've looked at all the variables
if (runInfo['shouldIncludeReport']) :
# this is going to be in the form
# [var_name] = {"passEpsilonPercent": percent ok with epsilon, "epsilon": epsilon)
# [varKey] = {"passEpsilonPercent": percent ok with epsilon, "epsilon": epsilon)
variableComparisons = {}
# generate the variable reports
for name in variableAnalysisInfo :
for varKey in variableAnalysisInfo :
# hang on to our good % and other info to describe our comparison
passedPercent = (1.0 - variableAnalysisInfo[name]['var_stats']
passedPercent = (1.0 - variableAnalysisInfo[varKey]['var_stats']
['Numerical Comparison Statistics']['diff_outside_epsilon_fraction']) * 100.0
variableComparisons[name] = {'pass_epsilon_percent': passedPercent,
'variable_run_info': variableAnalysisInfo[name]['run_info']
}
variableComparisons[varKey] = {'pass_epsilon_percent': passedPercent,
'variable_run_info': variableAnalysisInfo[varKey]['run_info']
}
print ('\tgenerating report for: ' + variableAnalysisInfo[name]['exp_name'])
print ('\tgenerating report for: ' + variableAnalysisInfo[varKey]['exp_name'])
report.generate_and_save_variable_report(files,
variableAnalysisInfo[name]['run_info'], runInfo,
variableAnalysisInfo[name]['var_stats'],
variableAnalysisInfo[varKey]['run_info'], runInfo,
variableAnalysisInfo[varKey]['var_stats'],
spatialInfo,
outputPath, name + ".html")
outputPath,
variableAnalysisInfo[varKey]['run_info']['variable_name'] + ".html")
print ('generating summary report')
# get the current time
......
pyglance/glance/exconfig.py
+ 31
− 32
View file @ 68a8b2e0
......@@ -43,7 +43,7 @@ lat_lon_info ['latitude_alt_name_in_b'] = 'resampled_latitude' # the alternate
# for differening data types (like ints/floats or float32/float64) or to handle slicing out only a subset of the data for analysis
# note: these two filters will only be applied to the longitude and latitude data in file B
lat_lon_info['data_filter_function_lon_in_b'] = (insert lambda function here)
lat_lon_info['data_filter_function_lon_in_b'] = (insert lambda function here)
lat_lon_info['data_filter_function_lat_in_b'] = (insert lambda function here)
"""
# this value can be used to control how similar the longitude and latitude must be to be considered matching
# if all of your longitude and latitude do not match under this epsilon, most of the comparison report will
......@@ -74,8 +74,12 @@ defaultValues = {'epsilon': 0.0, # the acceptable differe
'nonfinite_data_tolerance': None # the allowed fraction of non-finite data
# None indicates that variables should not be tested
# on amount of non-finite data
#'data_filter_function_a': (insert lambda function here),
#'data_filter_function_b': (insert lambda function here)
# the following two functions can be defined in order to filter the variable data,
# for example, these could be used to compensate
# for differening data types (like ints/floats or float32/float64)
# or to handle slicing out only a subset of the data for analysis
# 'data_filter_function_a': (insert lambda function here), # note: will only be applied to file A data
# 'data_filter_function_b': (insert lambda function here) # note: will only be applied to file B data
}
# a list of all the variables to analyze, all of the details are optional,
......@@ -91,11 +95,10 @@ defaultValues = {'epsilon': 0.0, # the acceptable differe
# leave it as an empty dictionary)
setOfVariables = {}
setOfVariables['imager_prof_retr_abi_total_totals_index'] = { # this should match the variable name in your files
setOfVariables['Total Totals'] = { # the key of your variable entry will be used as the display name for your variable
'display_name': 'Total Totals', # this entry is totally optional, it's used to label
# the report and some of the plots, if ommitted
# the variable name will be used in those places instead
'variable_name': 'imager_prof_retr_abi_total_totals_index',
# this should match the variable name in your file
'epsilon': 1.0, # the acceptable difference between file A and file B
......@@ -130,16 +133,12 @@ setOfVariables['imager_prof_retr_abi_total_totals_index'] = { # this s
# let you label your ranges (ie, the space between two
# boundaries) or your boundaries.
# the following two functions can be defined in order to filter the variable data,
# for example, these could be used to compensate
# for differening data types (like ints/floats or float32/float64)
# or to handle slicing out only a subset of the data for analysis
# data filters can be defined/overridden on a variable by variable basis
# 'data_filter_function_a': (insert lambda function here), # note: will only be applied to file A data
# 'data_filter_function_b': (insert lambda function here) # note: will only be applied to file B data
}
setOfVariables['imager_prof_retr_abi_total_precipitable_water_high'] = {
'display_name': 'Total Precipitable Water, High',
setOfVariables['Total Precipitable Water, High'] = {
'variable_name': 'imager_prof_retr_abi_total_precipitable_water_high',
'epsilon': 3.0,
'alternate_name_in_B': 'imager_prof_retr_abi_total_precipitable_water_low',
......@@ -149,42 +148,42 @@ setOfVariables['imager_prof_retr_abi_total_precipitable_water_high'] = {
# to appear only in file A)
}
setOfVariables['imager_prof_retr_abi_total_precipitable_water'] = {
'display_name': 'Total Precipitable Water',
setOfVariables['Total Precipitable Water'] = {
'variable_name': 'imager_prof_retr_abi_total_precipitable_water',
'epsilon': 3.0
# example:
# because missing, and the two tolerances are not defined here,
# this variable would use the defaultValues for those
}
setOfVariables['imager_prof_retr_abi_total_precipitable_water_low'] = {
'display_name': 'Total Precipitable Water, Low',
setOfVariables['Total Precipitable Water, Low'] = {
'variable_name': 'imager_prof_retr_abi_total_precipitable_water_low',
'epsilon': 3.0
}
setOfVariables['imager_prof_retr_abi_total_precipitable_water_mid'] = {
'display_name': 'Total Precipitable Water, Mid',
setOfVariables['Total Precipitable Water, Mid'] = {
'variable_name': 'imager_prof_retr_abi_total_precipitable_water_mid',
'epsilon': 3.0
}
setOfVariables['imager_prof_retr_abi_land_surface_temperature'] = {
'display_name': 'Land Surface Temperature',
setOfVariables['Land Surface Temperature'] = {
'variable_name': 'imager_prof_retr_abi_land_surface_temperature',
'epsilon': 5.0
}
setOfVariables['imager_prof_retr_abi_k_index'] = {
'display_name': 'K-Index',
setOfVariables['K-Index'] = {
'variable_name': 'imager_prof_retr_abi_k_index',
'epsilon': 2.0
}
setOfVariables['imager_prof_retr_abi_tprof'] = {
'display_name': 'Temperature Profile',
setOfVariables['Temperature Profile'] = {
'variable_name': 'imager_prof_retr_abi_tprof',
'epsilon': 0.1
}
setOfVariables['imager_prof_retr_abi_cape'] = {
'display_name': 'CAPE',
setOfVariables['CAPE'] = {
'variable_name': 'imager_prof_retr_abi_cape',
'epsilon': 1000
}
setOfVariables['imager_prof_retr_abi_lifted_index'] = {
'display_name': 'Lifted Index',
setOfVariables['Lifted Index'] = {
'variable_name': 'imager_prof_retr_abi_lifted_index',
'epsilon': 2.0
}
setOfVariables['imager_prof_retr_abi_showalter_index'] = {
'display_name': 'Showalter Index',
setOfVariables['Showalter Index'] = {
'variable_name': 'imager_prof_retr_abi_showalter_index',
'epsilon': 2.0
}
pyglance/glance/mainreport.txt
+ 10
− 10
View file @ 68a8b2e0
......@@ -31,7 +31,7 @@ Copyright (c) 2009 University of Wisconsin SSEC. All rights reserved.
${tempFileInfo['displayName']}
% endif
<blockquote>
file path: ${tempFileInfo['path']} <br>
path: ${tempFileInfo['path']} <br>
md5sum for ${fileKey}: ${tempFileInfo['md5sum']} <br>
last modified: ${tempFileInfo['lastModifiedTime']}
</blockquote>
......@@ -148,15 +148,15 @@ Copyright (c) 2009 University of Wisconsin SSEC. All rights reserved.
## report on all the variables that were compared and give a basic stat idea of how well they did
<blockquote>
<p>
## TODO replace the table with floating boxes in some form
## TODO replace the table with floating boxes at some point
<table>
% for variableName in sorted(list(variables)) :
% for variableKey in sorted(list(variables)) :
<%
tempVariableInfo = variables[variableName]
varDisplayName = variableName
if (tempVariableInfo['variable_run_info'].has_key('display_name')) :
tempVariableInfo = variables[variableKey]
technicalName = tempVariableInfo['variable_run_info']['variable_name']
varDisplayName = technicalName
if 'display_name' in tempVariableInfo['variable_run_info'] :
varDisplayName = tempVariableInfo['variable_run_info']['display_name']
passPercent = tempVariableInfo['pass_epsilon_percent']
didPass = tempVariableInfo['variable_run_info']['did_pass']
......@@ -172,7 +172,7 @@ Copyright (c) 2009 University of Wisconsin SSEC. All rights reserved.
% endif
</td>
<td>
Variable: <a href="${variableName | u}.html">${varDisplayName}</a> <br>
Variable: <a href="${technicalName | u}.html">${varDisplayName}</a> <br>
Epsilon used: ${tempVariableInfo['variable_run_info']['epsilon']} <br>
Finite values within one epsilon of difference:
${report.make_formatted_display_string(passPercent)}%
......@@ -207,7 +207,7 @@ Copyright (c) 2009 University of Wisconsin SSEC. All rights reserved.
% if (len(uniqueToAVars) > 0) or (len(uniqueToBVars) > 0) :
<h3>Unique Variables</h3>
% if (len(uniqueToAVars) != 0) :
% if (len(uniqueToAVars) > 0) :
<p>
Variables only found in file A:
<blockquote>
......@@ -218,7 +218,7 @@ Copyright (c) 2009 University of Wisconsin SSEC. All rights reserved.
</p>
% endif
% if (len(uniqueToBVars) != 0) :
% if (len(uniqueToBVars) > 0) :
<p>
Variables only found in file B:
<blockquote>
......
pyglance/glance/variablereport.txt
+ 2
− 2
View file @ 68a8b2e0
......@@ -37,11 +37,11 @@ Copyright (c) 2009 University of Wisconsin SSEC. All rights reserved.
<% tempFileInfo = files[fileKey] %>
<p>
${fileKey}:
% if tempFileInfo.has_key('displayName') :
% if 'displayName' in tempFileInfo :
${tempFileInfo['displayName']}
% endif
<blockquote>
file path: ${tempFileInfo['path']} <br>
path: ${tempFileInfo['path']} <br>
md5sum for ${fileKey}: ${tempFileInfo['md5sum']} <br>
last modified: ${tempFileInfo['lastModifiedTime']}
</blockquote>
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment