Skip to content
Snippets Groups Projects
Commit 71c6a26c authored by (no author)'s avatar (no author)
Browse files

replaced trouble with mismatch; added units to most plots; corrected off by... 

replaced trouble with mismatch; added units to most plots; corrected off by one error in histogram and changed 'bin' to interval in the displayed stats

git-svn-id: https://svn.ssec.wisc.edu/repos/glance/trunk@141 8a9318a1-56ba-4d59-b755-99d26321be01
parent 778f5918
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
pyglance/glance/compare.py
+ 12
− 10
View file @ 71c6a26c
......@@ -477,7 +477,7 @@ def _check_lon_lat_equality(longitudeADataObject, latitudeADataObject,
latitudeDiffInfo = dataobj.DiffInfoObject(latitudeADataObject, latitudeBDataObject, epsilonValue=llepsilon)
# how much difference is there between the two sets?
lon_lat_not_equal_mask = longitudeDiffInfo.diff_data_object.masks.trouble_mask | latitudeDiffInfo.diff_data_object.masks.trouble_mask
lon_lat_not_equal_mask = longitudeDiffInfo.diff_data_object.masks.mismatch_mask | latitudeDiffInfo.diff_data_object.masks.mismatch_mask
lon_lat_not_equal_points_count = sum(lon_lat_not_equal_mask)
lon_lat_not_equal_points_percent = (float(lon_lat_not_equal_points_count) / float(lon_lat_not_equal_mask.size)) * 100.0
......@@ -491,23 +491,23 @@ def _check_lon_lat_equality(longitudeADataObject, latitudeADataObject,
if ((len(longitudeADataObject.data[~longitudeADataObject.masks.ignore_mask]) > 0) and
(len( latitudeADataObject.data[~ latitudeADataObject.masks.ignore_mask]) > 0)) :
plot.plot_and_save_spacial_trouble(longitudeADataObject, latitudeADataObject,
plot.plot_and_save_spacial_mismatch(longitudeADataObject, latitudeADataObject,
lon_lat_not_equal_mask,
"A", "Lon./Lat. Points Mismatched between A and B\n" +
"(Shown in A)",
"LonLatMismatch",
outputPath, True,
fullDPI=fullDPI, thumbDPI=thumbDPI)
fullDPI=fullDPI, thumbDPI=thumbDPI, units="degrees")
if ((len(longitudeBDataObject.data[~longitudeBDataObject.masks.ignore_mask]) > 0) and
(len( latitudeBDataObject.data[~ latitudeBDataObject.masks.ignore_mask]) > 0)) :
plot.plot_and_save_spacial_trouble(longitudeBDataObject, latitudeBDataObject,
plot.plot_and_save_spacial_mismatch(longitudeBDataObject, latitudeBDataObject,
lon_lat_not_equal_mask,
"B", "Lon./Lat. Points Mismatched between A and B\n" +
"(Shown in B)",
"LonLatMismatch",
outputPath, True,
fullDPI=fullDPI, thumbDPI=thumbDPI)
fullDPI=fullDPI, thumbDPI=thumbDPI, units="degrees")
# setup our return data
returnInfo = {}
......@@ -560,22 +560,22 @@ def _compare_spatial_invalidity(longitude_a_object, longitude_b_object,
if ((spatial_info['file A']['numInvPts'] > 0) and (do_include_images) and
(len(longitude_a_object.data[~invalid_in_a_mask]) > 0) and
(len( latitude_a_object.data[~invalid_in_a_mask]) > 0)) :
plot.plot_and_save_spacial_trouble(longitude_a_object, latitude_a_object,
plot.plot_and_save_spacial_mismatch(longitude_a_object, latitude_a_object,
valid_only_in_mask_a,
"A", "Points only valid in\nFile A\'s longitude & latitude",
"SpatialMismatch",
output_path, True,
fullDPI=fullDPI, thumbDPI=thumbDPI)
fullDPI=fullDPI, thumbDPI=thumbDPI, units="degrees")
if ((spatial_info['file B']['numInvPts'] > 0) and (do_include_images) and
(len(longitude_b_object.data[~invalid_in_b_mask]) > 0) and
(len( latitude_b_object.data[~invalid_in_b_mask]) > 0)
) :
plot.plot_and_save_spacial_trouble(longitude_b_object, latitude_b_object,
plot.plot_and_save_spacial_mismatch(longitude_b_object, latitude_b_object,
valid_only_in_mask_b,
"B", "Points only valid in\nFile B\'s longitude & latitude",
"SpatialMismatch",
output_path, True,
fullDPI=fullDPI, thumbDPI=thumbDPI)
fullDPI=fullDPI, thumbDPI=thumbDPI, units="degrees")
return invalid_in_common_mask, spatial_info, longitude_common, latitude_common
......@@ -1471,7 +1471,9 @@ def reportGen_library_call (a_path, b_path, var_list=[ ],
tupleName= varRunInfo['tupleName'] if 'tupleName' in varRunInfo else 'tuple',
epsilonPercent=varRunInfo['epsilon_percent'] if 'epsilon_percent' in varRunInfo else None,
fullDPI= runInfo['detail_DPI'],
thumbDPI= runInfo['thumb_DPI'])
thumbDPI= runInfo['thumb_DPI'],
units_a= varRunInfo['units_a'] if 'units_a' in varRunInfo else None,
units_b= varRunInfo['units_b'] if 'units_b' in varRunInfo else None)
print("\tfinished creating figures for: " + explanationName)
......
pyglance/glance/data.py
+ 7
− 7
View file @ 71c6a26c
......@@ -61,15 +61,15 @@ class DiffMaskSetObject (BasicMaskSetObject) :
masks are used to capture information about where data should be
ignored/valid/non-finite in the compared data.
Additionally, masks describing trouble points and points outside
Additionally, masks describing mismatch points and points outside
of the epsilon analysis tolerances are included.
trouble_mask - a mask of data points which may indicate issues in the data
mismatch_mask - a mask of data points which may indicate issues in the data
outside_epsilon_mask - a mask of points which did not pass the epsilon
tolerance testing
"""
def __init__(self, ignoreMask, validInBothMask, troubleMask, epsilonMask) :
def __init__(self, ignoreMask, validInBothMask, mismatchMask, epsilonMask) :
"""
create a more complex mask, including additional difference information
"""
......@@ -77,7 +77,7 @@ class DiffMaskSetObject (BasicMaskSetObject) :
self.ignore_mask = ignoreMask
self.valid_mask = validInBothMask
self.trouble_mask = troubleMask
self.mismatch_mask = mismatchMask
self.outside_epsilon_mask = epsilonMask
class DataObject (object) :
......@@ -265,15 +265,15 @@ class DiffInfoObject (object) :
if (epsilonPercent is not None) :
outside_epsilon_mask |= (abs(raw_diff) > abs(aDataObject.data * (float(epsilonPercent) / 100.0))) & valid_in_both
# trouble points = mismatched nans, mismatched missing-values, differences that are too large
trouble_pt_mask = ( (aDataObject.masks.non_finite_mask ^ bDataObject.masks.non_finite_mask) |
# mismatch points = mismatched nans, mismatched missing-values, differences that are too large
mismatch_pt_mask = ( (aDataObject.masks.non_finite_mask ^ bDataObject.masks.non_finite_mask) |
(aDataObject.masks.missing_mask ^ bDataObject.masks.missing_mask) |
outside_epsilon_mask )
# make our diff data object
diff_data_object = DataObject(raw_diff, fillValue=fill_data_value)
diff_data_object.masks = DiffMaskSetObject(ignore_in_both, valid_in_both,
trouble_pt_mask, outside_epsilon_mask)
mismatch_pt_mask, outside_epsilon_mask)
return diff_data_object
......
pyglance/glance/exconfig.py
+ 2
− 2
View file @ 71c6a26c
......@@ -207,8 +207,8 @@ setOfVariables['Total Precipitable Water, Low'] = {
'do_plot_scatter': True,
# should the histogram be plotted?
'do_plot_histogram': True,
# should the trouble plot be plotted?
'do_plot_trouble': True
# should the mismatch plot be plotted?
'do_plot_mismatch': True
}
setOfVariables['Total Precipitable Water, Mid'] = {
'variable_name': 'imager_prof_retr_abi_total_precipitable_water_mid',
......
pyglance/glance/figures.py
+ 122
− 74
View file @ 71c6a26c
......@@ -80,21 +80,21 @@ def _plot_tag_data_simple(tagData) :
-1 if no valid tagData was given.
"""
numTroublePoints = -1
numMismatchPoints = -1
# if there are "tag" masks, plot them over the existing map
if not (tagData is None) :
numTroublePoints = sum(tagData)
numMismatchPoints = sum(tagData)
# if we have trouble points, we need to show them
if numTroublePoints > 0:
# if we have mismatch points, we need to show them
if numMismatchPoints > 0:
# figure out how many bad points there are
totalNumPoints = tagData.size # the number of points
percentBad = (float(numTroublePoints) / float(totalNumPoints)) * 100.0
LOG.debug('\t\tnumber of trouble points: ' + str(numTroublePoints))
LOG.debug('\t\tpercent of trouble points: ' + str(percentBad))
percentBad = (float(numMismatchPoints) / float(totalNumPoints)) * 100.0
LOG.debug('\t\tnumber of mismatch points: ' + str(numMismatchPoints))
LOG.debug('\t\tpercent of mismatch points: ' + str(percentBad))
new_kwargs = {}
new_kwargs['cmap'] = greenColorMap
......@@ -102,14 +102,14 @@ def _plot_tag_data_simple(tagData) :
p = contourf(cleanTagData, **new_kwargs)
# TODO, need to incorporate plot for small numbers of pts
# display the number of trouble points on the report if we were passed a set of tag data
troublePtString = '\n\nShowing ' + str(numTroublePoints) + ' Trouble Points'
# display the number of mismatch points on the report if we were passed a set of tag data
mismatchPtString = '\n\nShowing ' + str(numMismatchPoints) + ' Mismatch Points'
# if our plot is more complex, add clarification
if numTroublePoints > 0 :
troublePtString = troublePtString + ' in Green'
plt.xlabel(troublePtString)
if numMismatchPoints > 0 :
mismatchPtString = mismatchPtString + ' in Green'
plt.xlabel(mismatchPtString)
return numTroublePoints
return numMismatchPoints
def _plot_tag_data_mapped(bMap, tagData, x, y, addExplinationLabel=True) :
"""
......@@ -122,19 +122,19 @@ def _plot_tag_data_mapped(bMap, tagData, x, y, addExplinationLabel=True) :
The return will be the number of points plotted or
-1 if no valid tagData was given.
numTroublePoints = _plot_tag_data_mapped(bMap, tagData, x, y)
numMismatchPoints = _plot_tag_data_mapped(bMap, tagData, x, y)
"""
numTroublePoints = -1
numMismatchPoints = -1
# if there are "tag" masks, plot them over the existing map
if (tagData is not None) and (tagData.size > 0) :
# look at how many trouble points we have
numTroublePoints = sum(tagData)
# look at how many mismatch points we have
numMismatchPoints = sum(tagData)
neededHighlighting = False
if numTroublePoints > 0 :
if numMismatchPoints > 0 :
# pick out the cooridinates of the points we want to plot
newX = np.array(x[tagData])
......@@ -142,9 +142,9 @@ def _plot_tag_data_mapped(bMap, tagData, x, y, addExplinationLabel=True) :
# figure out how many bad points there are
totalNumPoints = x.size # the number of points
percentBad = (float(numTroublePoints) / float(totalNumPoints)) * 100.0
LOG.debug('\t\tnumber of trouble points: ' + str(numTroublePoints))
LOG.debug('\t\tpercent of trouble points: ' + str(percentBad))
percentBad = (float(numMismatchPoints) / float(totalNumPoints)) * 100.0
LOG.debug('\t\tnumber of mismatch points: ' + str(numMismatchPoints))
LOG.debug('\t\tpercent of mismatch points: ' + str(percentBad))
# if there are very few points, make them easier to notice
# by plotting some colored circles underneath them
......@@ -155,8 +155,8 @@ def _plot_tag_data_mapped(bMap, tagData, x, y, addExplinationLabel=True) :
neededHighlighting = True
p = bMap.plot(newX, newY, 'o', color='#993399', markersize=3)
# if there are way too many trouble points, we can't use plot for this
if (numTroublePoints > 1000000) :
# if there are way too many mismatch points, we can't use plot for this
if (numMismatchPoints > 1000000) :
new_kwargs = {}
new_kwargs['cmap'] = greenColorMap
p = maps.show_x_y_data(x, y, bMap, data=tagData, **new_kwargs)
......@@ -165,21 +165,21 @@ def _plot_tag_data_mapped(bMap, tagData, x, y, addExplinationLabel=True) :
p = bMap.plot(newX, newY, '.', color='#00FF00', markersize=1)
if addExplinationLabel :
# display the number of trouble points on the report if we were passed a set of tag data
# display the number of mismatch points on the report if we were passed a set of tag data
# I'm not thrilled with this solution for getting it below the labels drawn by the basemap
# but I don't think there's a better one at the moment given matplotlib's workings
troublePtString = '\n\nShowing ' + str(numTroublePoints) + ' Trouble Points'
mismatchPtString = '\n\nShowing ' + str(numMismatchPoints) + ' Mismatch Points'
# if our plot is more complex, add clarification
if numTroublePoints > 0 :
troublePtString = troublePtString + ' in Green'
if numMismatchPoints > 0 :
mismatchPtString = mismatchPtString + ' in Green'
if neededHighlighting :
troublePtString = troublePtString + '\nwith Purple Circles for Visual Clarity'
plt.xlabel(troublePtString)
mismatchPtString = mismatchPtString + '\nwith Purple Circles for Visual Clarity'
plt.xlabel(mismatchPtString)
return numTroublePoints
return numMismatchPoints
# build a scatter plot of the x,y points
def create_scatter_plot(dataX, dataY, title, xLabel, yLabel, badMask=None, epsilon=None) :
def create_scatter_plot(dataX, dataY, title, xLabel, yLabel, badMask=None, epsilon=None, units_x=None, units_y=None) :
"""
build a scatter plot of the data
if a bad mask is given the points selected by that mask will be plotted in a different color
......@@ -191,9 +191,9 @@ def create_scatter_plot(dataX, dataY, title, xLabel, yLabel, badMask=None, epsil
return create_complex_scatter_plot ([(dataX, dataY, badMask,
'b', 'r',
'within\nepsilon', 'outside\nepsilon')],
title, xLabel, yLabel, epsilon=epsilon)
title, xLabel, yLabel, epsilon=epsilon, units_x=units_x, units_y=units_y)
def create_complex_scatter_plot(dataList, title, xLabel, yLabel, epsilon=None) :
def create_complex_scatter_plot(dataList, title, xLabel, yLabel, epsilon=None, units_x=None, units_y=None) :
"""
build a scatter plot with multiple data sets in different colors
the dataList parameter should be in the form:
......@@ -231,11 +231,11 @@ def create_complex_scatter_plot(dataList, title, xLabel, yLabel, epsilon=None) :
axes.plot(dataX, dataY, ',', color=goodColor, label=goodLabel)
# plot the bad data
numTroublePts = 0
numMismatchPts = 0
if (badX is not None) and (badY is not None) and (badMask is not None) :
numTroublePts = badX.size
LOG.debug('\t\tplotting ' + str(numTroublePts) + ' trouble points in scatter plot.' )
if numTroublePts > 0 :
numMismatchPts = badX.size
LOG.debug('\t\tplotting ' + str(numMismatchPts) + ' mismatch points in scatter plot.' )
if numMismatchPts > 0 :
axes.plot(badX, badY, ',', color=badColor, label=badLabel)
# draw some extra informational lines
......@@ -245,10 +245,18 @@ def create_complex_scatter_plot(dataList, title, xLabel, yLabel, epsilon=None) :
# as long as things have been plotted with proper labels they should show up here
axes.legend(loc=0, markerscale=3.0) # Note: at the moment markerscale doesn't seem to work
# add the units to the x and y labels
tempXLabel = xLabel
tempYLabel = yLabel
if str.lower(str(units_x)) != "none" :
tempXLabel = tempXLabel + " in " + units_x
if str.lower(str(units_y)) != "none" :
tempYLabel = tempYLabel + " in " + units_y
# and some informational stuff
axes.set_title(title)
plt.xlabel(xLabel)
plt.ylabel(yLabel)
plt.xlabel(tempXLabel)
plt.ylabel(tempYLabel)
# format our axes so they display gracefully
yFormatter = FormatStrFormatter("%4.4g")
......@@ -259,12 +267,17 @@ def create_complex_scatter_plot(dataList, title, xLabel, yLabel, epsilon=None) :
return figure
# build a hexbin plot of the x,y points and show the density of the point distribution
def create_hexbin_plot(dataX, dataY, title, xLabel, yLabel, epsilon=None) :
def create_hexbin_plot(dataX, dataY, title, xLabel, yLabel, epsilon=None, units_x=None, units_y=None) :
# make the figure
figure = plt.figure()
axes = figure.add_subplot(111)
# for some reason, if you give the hexplot a data set that's all the same number it dies horribly
if ( ((dataX is None) or (len(dataX) <= 0)) or ((dataY is None) or (len(dataY) <= 0)) or
((dataX.max() == dataX.min()) and (dataY.max() == dataY.min())) ):
return figure
# the hexbin plot of the good data
plt.hexbin(dataX, dataY, bins='log', cmap=cm.jet)
plt.axis([dataX.min(), dataX.max(), dataY.min(), dataY.max()])
......@@ -276,10 +289,18 @@ def create_hexbin_plot(dataX, dataY, title, xLabel, yLabel, epsilon=None) :
# draw some extra informational lines
_draw_x_equals_y_line(axes, color='w', epsilon=epsilon, epsilonColor='k')
# add the units to the x and y labels
tempXLabel = xLabel
tempYLabel = yLabel
if str.lower(str(units_x)) != "none" :
tempXLabel = tempXLabel + " in " + units_x
if str.lower(str(units_y)) != "none" :
tempYLabel = tempYLabel + " in " + units_y
# and some informational stuff
axes.set_title(title)
plt.xlabel(xLabel)
plt.ylabel(yLabel)
plt.xlabel(tempXLabel)
plt.ylabel(tempYLabel)
# format our axes so they display gracefully
yFormatter = FormatStrFormatter("%4.4g")
......@@ -320,7 +341,7 @@ def _draw_x_equals_y_line(axes, color='k', style='--', epsilon=None, epsilonColo
axes.set_ybound(ybounds)
# build a histogram figure of the given data with the given title and number of bins
def create_histogram(data, bins, title, xLabel, yLabel, displayStats=False) :
def create_histogram(data, bins, title, xLabel, yLabel, displayStats=False, units=None) :
# make the figure
figure = plt.figure()
......@@ -330,7 +351,7 @@ def create_histogram(data, bins, title, xLabel, yLabel, displayStats=False) :
return figure
# the histogram of the data
n, bins, patches = plt.hist(data, bins)
n, outBins, patches = plt.hist(data, bins)
# format our axes so they display gracefully
yFormatter = FormatStrFormatter("%3.3g")
......@@ -338,9 +359,14 @@ def create_histogram(data, bins, title, xLabel, yLabel, displayStats=False) :
xFormatter = FormatStrFormatter("%.4g")
axes.xaxis.set_major_formatter(xFormatter)
# add the units to the x and y labels
tempXLabel = xLabel
if str.lower(str(units)) != "none" :
tempXLabel = tempXLabel + " in " + units
# and some informational stuff
axes.set_title(title)
plt.xlabel(xLabel)
plt.xlabel(tempXLabel)
plt.ylabel(yLabel)
# if stats were passed in, put some of the information on the graph
......@@ -356,7 +382,7 @@ def create_histogram(data, bins, title, xLabel, yLabel, displayStats=False) :
# info on the display of our statistics
xbounds = axes.get_xbound()
numBinsToUse = len(bins)
numBinsToUse = bins
xrange = xbounds[1] - xbounds[0]
binSize = xrange / float(numBinsToUse)
......@@ -365,8 +391,8 @@ def create_histogram(data, bins, title, xLabel, yLabel, displayStats=False) :
statText = statText + '\n' + 'mean: ' + report.make_formatted_display_string(meanVal)
statText = statText + '\n' + 'median: ' + report.make_formatted_display_string(medianVal)
statText = statText + '\n' + 'std: ' + report.make_formatted_display_string(stdVal)
statText = statText + '\n\n' + 'bins: ' + report.make_formatted_display_string(numBinsToUse)
statText = statText + '\n' + 'bin size ' + report.make_formatted_display_string(binSize)
statText = statText + '\n\n' + 'intervals: ' + report.make_formatted_display_string(numBinsToUse)
statText = statText + '\n' + 'interval size ' + report.make_formatted_display_string(binSize)
# figure out where to place the text and put it on the figure
centerOfDisplay = xbounds[0] + (float(xrange) / 2.0)
......@@ -384,7 +410,7 @@ def create_histogram(data, bins, title, xLabel, yLabel, displayStats=False) :
# set on the existing image
def create_mapped_figure(data, latitude, longitude, baseMapInstance, boundingAxes, title,
invalidMask=None, colorMap=None, tagData=None,
dataRanges=None, dataRangeNames=None, dataRangeColors=None, **kwargs) :
dataRanges=None, dataRangeNames=None, dataRangeColors=None, units=None, **kwargs) :
# make a clean version of our lon/lat
latitudeClean = ma.array(latitude, mask=~invalidMask)
......@@ -422,7 +448,7 @@ def create_mapped_figure(data, latitude, longitude, baseMapInstance, boundingAxe
axes.set_title(title)
# show a generic color bar
doLabelRanges = False
if not (data is None) :
if data is not None :
cbar = colorbar(format='%.3g')
# if there are specific requested labels, add them
if not (dataRangeNames is None) :
......@@ -434,12 +460,16 @@ def create_mapped_figure(data, latitude, longitude, baseMapInstance, boundingAxe
else : # we will want to label the ranges themselves
cbar.ax.set_yticklabels(dataRangeNames) # todo, this line is temporary
doLabelRanges = True
else :
# add the units to the colorbar
if str.lower(str(units)) != "none" :
cbar.set_label(units)
numTroublePoints = _plot_tag_data_mapped(bMap, tagData, x, y)
numMismatchPoints = _plot_tag_data_mapped(bMap, tagData, x, y)
print ('number of trouble points: ' + str(numTroublePoints))
LOG.debug ('number of mismatch points: ' + str(numMismatchPoints))
# if we still need to label the ranges, do it now that our fake axis won't mess the trouble points up
# if we still need to label the ranges, do it now that our fake axis won't mess the mismatch points up
if doLabelRanges :
""" TODO get this working properly
fakeAx = plt.axes ([0.77, 0.05, 0.2, 0.9], frameon=False)
......@@ -457,8 +487,9 @@ def create_mapped_figure(data, latitude, longitude, baseMapInstance, boundingAxe
# if any masks are passed in the tagData list they will be plotted as an overlays
# set on the existing image
# TODO, this method has not been throughly tested
# TODO, this method needs an input colormap so the mismatch plot can be the right color
def create_quiver_mapped_figure(data, latitude, longitude, baseMapInstance, boundingAxes, title,
invalidMask=None, tagData=None, uData=None, vData=None, **kwargs) :
invalidMask=None, tagData=None, uData=None, vData=None, units=None, **kwargs) :
# make a clean version of our lon/lat/data
latitudeClean = latitude[~invalidMask]
......@@ -486,14 +517,23 @@ def create_quiver_mapped_figure(data, latitude, longitude, baseMapInstance, boun
# show the title
axes.set_title(title)
numTroublePoints = _plot_tag_data_mapped(bMap, tagDataClean, x, y)
# make a color bar if we have color data
""" todo, currently this crashes glance because of an error in matplot lib
if colorData is not None :
cbar = plt.colorbar(format='%.3g')
# add the units to the colorbar
if str.lower(str(units)) != "none" :
cbar.set_label(units)
"""
numMismatchPoints = _plot_tag_data_mapped(bMap, tagDataClean, x, y)
return figure
def create_simple_figure(data, figureTitle, invalidMask=None, tagData=None, colorMap=None, colorbarLimits=None) :
def create_simple_figure(data, figureTitle, invalidMask=None, tagData=None, colorMap=None, colorbarLimits=None, units=None) :
"""
create a simple figure showing the data given masked by the invalid mask
any tagData passed in will be interpreted as trouble points on the image and plotted as a
any tagData passed in will be interpreted as mismatch points on the image and plotted as a
filled contour overlay in green on the image
if a colorMap is given it will be used to plot the data,
if not the default colorMap for imshow will be used
......@@ -521,11 +561,14 @@ def create_simple_figure(data, figureTitle, invalidMask=None, tagData=None, colo
clim(vmin=colorbarLimits[0], vmax=colorbarLimits[-1])
# make a color bar
cbar = colorbar(format='%.3g')
# add the units to the colorbar
if str.lower(str(units)) != "none" :
cbar.set_label(units)
# and some informational stuff
axes.set_title(figureTitle)
numTroublePoints = _plot_tag_data_simple(tagData)
numMismatchPoints = _plot_tag_data_simple(tagData)
return figure
......@@ -535,7 +578,7 @@ def create_line_plot_figure(dataList, figureTitle) :
if tagData is given, under-label those points with green circles
Each entry in the dataList should be a tupple containing:
(data, invalidMask, colorString, labelName, tagData)
(data, invalidMask, colorString, labelName, tagData, units)
The color string describes a color for plotting in matplotlib.
The label names will be used for the legend, which will be shown if there is
......@@ -544,6 +587,8 @@ def create_line_plot_figure(dataList, figureTitle) :
will be masked and a default label of "data#" (where # is an arbitrary
unique counter) will be used.
tagData may also be passed as None if tagging is not desired in the output.
units describes the units used to measure the data (such as mm or degrees)
and will be used to label the plot. units may be passed as None.
"""
# build the plot
......@@ -555,7 +600,7 @@ def create_line_plot_figure(dataList, figureTitle) :
minTagPts = -1
maxTagPts = -1
plottedTagData = False
for dataSet, invalidMask, colorString, labelName, tagData in dataList :
for dataSet, invalidMask, colorString, labelName, tagData, units in dataList :
# if we don't have these, set them to defaults
if invalidMask is None :
......@@ -579,34 +624,37 @@ def create_line_plot_figure(dataList, figureTitle) :
if tagData is not None :
plottedTagData = True
numTroublePoints = sum(tagData)
LOG.debug('\t\tnumber of trouble points: ' + str(numTroublePoints))
if numTroublePoints < minTagPts:
minTagPts = numTroublePoints
if numTroublePoints > maxTagPts :
maxTagPts = numTroublePoints
numMismatchPoints = sum(tagData)
LOG.debug('\t\tnumber of mismatch points: ' + str(numMismatchPoints))
if numMismatchPoints < minTagPts:
minTagPts = numMismatchPoints
if numMismatchPoints > maxTagPts :
maxTagPts = numMismatchPoints
# if we have trouble points, we need to show them
if numTroublePoints > 0:
# if we have mismatch points, we need to show them
if numMismatchPoints > 0:
cleanTagData = ma.array(dataSet, mask=~tagData | invalidMask)
axes.plot(indexData, cleanTagData, 'yo', label='trouble point')
axes.plot(indexData, cleanTagData, 'yo', label='mismatch point')
if str.lower(str(units)) !="none" :
labelName = labelName + " in " + units
axes.plot(indexData, cleanData, '-' + colorString, label=labelName)
# display the number of trouble points on the report if we were passed
# display the number of mismatch points on the report if we were passed
# a set of tag data and we were able to compare it to some actual data
if (plottedTagData and (minTagPts >= 0) and (maxTagPts >=0)) :
troublePtString = '\nMarking '
mismatchPtString = '\nMarking '
if (minTagPts == maxTagPts) :
troublePtString = troublePtString + str(minTagPts) + ' Trouble Points with Yellow Circles'
mismatchPtString = mismatchPtString + str(minTagPts) + ' Mismatch Points with Yellow Circles'
else :
troublePtString = (troublePtString + 'between ' + str(minTagPts) + ' and ' + str(maxTagPts) + ' Trouble Points'
mismatchPtString = (mismatchPtString + 'between ' + str(minTagPts) + ' and ' + str(maxTagPts) + ' Mismatch Points'
+ '\non the various data sets (using Yellow Circles)')
plt.xlabel(troublePtString)
plt.xlabel(mismatchPtString)
if (len(dataList) > 1) or plottedTagData :
# make a key to explain our plot
......
pyglance/glance/mainreport.txt
+ 1
− 1
View file @ 71c6a26c
......@@ -9,7 +9,7 @@ Copyright (c) 2009 University of Wisconsin SSEC. All rights reserved.
import glance.report as report
%>
<title>Spacial File Comparison</title>
<title>File Comparison</title>
</head>
<body>
......
pyglance/glance/plot.py
+ 27
− 22
View file @ 71c6a26c
......@@ -106,11 +106,11 @@ def _log_spawn_and_wait_if_needed (imageDescription, childPids,
return
def plot_and_save_spacial_trouble(longitudeObject, latitudeObject, spacialTroubleMask,
def plot_and_save_spacial_mismatch(longitudeObject, latitudeObject, spacialMismatchMask,
fileNameDiscriminator, title, fileBaseName, outputPath, makeSmall=False,
fullDPI=fullSizeDPI, thumbDPI=thumbSizeDPI) :
fullDPI=fullSizeDPI, thumbDPI=thumbSizeDPI, units=None) :
"""
given information on spatially placed trouble points in A and B, plot only those points in a very obvious way
given information on spatially placed mismatch points in A and B, plot only those points in a very obvious way
on top of a background plot of a's data shown in grayscale, save this plot to the output path given
if makeSmall is passed as true a smaller version of the image will also be saved
"""
......@@ -118,27 +118,28 @@ def plot_and_save_spacial_trouble(longitudeObject, latitudeObject, spacialTroubl
# get the bounding axis and make a basemap
boundingAxes = plotfns.get_visible_axes(longitudeObject.data, latitudeObject.data, ~spaciallyInvalidMask)
LOG.debug("Visible axes for lon/lat trouble figure are: " + str(boundingAxes))
LOG.debug("Visible axes for lon/lat mismatch figure are: " + str(boundingAxes))
baseMapInstance, boundingAxes = maps.create_basemap(longitudeObject.data, latitudeObject.data,
boundingAxes, plotfns.select_projection(boundingAxes))
# make the figure
LOG.info("Creating spatial trouble image")
spatialTroubleFig = figures.create_mapped_figure(None, latitudeObject.data, longitudeObject.data, baseMapInstance, boundingAxes,
title, invalidMask=spaciallyInvalidMask, tagData=spacialTroubleMask)
LOG.info("Creating spatial mismatch image")
spatialMismatchFig = figures.create_mapped_figure(None, latitudeObject.data, longitudeObject.data, baseMapInstance,
boundingAxes, title, invalidMask=spaciallyInvalidMask,
tagData=spacialMismatchMask, units=units)
# save the figure
LOG.info("Saving spatial trouble image")
spatialTroubleFig.savefig(outputPath + "/" + fileBaseName + "." + fileNameDiscriminator + ".png", dpi=fullDPI)
LOG.info("Saving spatial mismatch image")
spatialMismatchFig.savefig(outputPath + "/" + fileBaseName + "." + fileNameDiscriminator + ".png", dpi=fullDPI)
# we may also save a smaller versions of the figure
if (makeSmall) :
spatialTroubleFig.savefig(outputPath + "/" + fileBaseName + "." + fileNameDiscriminator + ".small.png", dpi=thumbDPI)
spatialMismatchFig.savefig(outputPath + "/" + fileBaseName + "." + fileNameDiscriminator + ".small.png", dpi=thumbDPI)
# get rid of the figure
spatialTroubleFig.clf()
plt.close(spatialTroubleFig)
del(spatialTroubleFig)
spatialMismatchFig.clf()
plt.close(spatialMismatchFig)
del(spatialMismatchFig)
return
......@@ -164,7 +165,8 @@ def plot_and_save_comparison_figures (aData, bData,
binIndex=None, tupleIndex=None,
binName='bin', tupleName='tuple',
epsilonPercent=None,
fullDPI=None, thumbDPI=None) :
fullDPI=None, thumbDPI=None,
units_a=None, units_b=None) :
"""
Plot images for a set of figures based on the data sets and settings
passed in. The images will be saved to disk according to the settings.
......@@ -239,16 +241,16 @@ def plot_and_save_comparison_figures (aData, bData,
spaciallyInvalidMaskA = lonLatDataDict['a']['inv_mask']
spaciallyInvalidMaskB = lonLatDataDict['b']['inv_mask']
# compare the two data sets to get our difference data and trouble info
# compare the two data sets to get our difference data and mismatch info
aDataObject = dataobj.DataObject(aData, fillValue=missingValue, ignoreMask=spaciallyInvalidMaskA)
bDataObject = dataobj.DataObject(bData, fillValue=missingValueAltInB, ignoreMask=spaciallyInvalidMaskB)
diffInfo = dataobj.DiffInfoObject(aDataObject, bDataObject, epsilonValue=epsilon, epsilonPercent=epsilonPercent)
#TODO, for the moment, unpack these values into local variables
rawDiffData = diffInfo.diff_data_object.data
goodMask = diffInfo.diff_data_object.masks.valid_mask
goodInAMask = diffInfo.a_data_object.masks.valid_mask
goodInBMask = diffInfo.b_data_object.masks.valid_mask
troubleMask = diffInfo.diff_data_object.masks.trouble_mask
rawDiffData = diffInfo.diff_data_object.data
goodMask = diffInfo.diff_data_object.masks.valid_mask
goodInAMask = diffInfo.a_data_object.masks.valid_mask
goodInBMask = diffInfo.b_data_object.masks.valid_mask
mismatchMask = diffInfo.diff_data_object.masks.mismatch_mask
outsideEpsilonMask = diffInfo.diff_data_object.masks.outside_epsilon_mask
aNotFiniteMask = diffInfo.a_data_object.masks.non_finite_mask
bNotFiniteMask = diffInfo.b_data_object.masks.non_finite_mask
......@@ -292,7 +294,7 @@ def plot_and_save_comparison_figures (aData, bData,
# point by point
absDiffData=absDiffData, rawDiffData=rawDiffData,
goodInBothMask=goodMask,
troubleMask=troubleMask, outsideEpsilonMask=outsideEpsilonMask,
mismatchMask=mismatchMask, outsideEpsilonMask=outsideEpsilonMask,
# only used for plotting quiver data
aUData=aUData, aVData=aVData,
......@@ -301,7 +303,10 @@ def plot_and_save_comparison_figures (aData, bData,
# only used for line plots
binIndex=binIndex, tupleIndex=tupleIndex,
binName=binName, tupleName=tupleName,
epsilonPercent=epsilonPercent
epsilonPercent=epsilonPercent,
# used for display in several types of plots
units_a=units_a, units_b=units_b
)
plottingFunctions.update(moreFunctions)
......
pyglance/glance/plotcreatefns.py
+ 102
− 74
View file @ 71c6a26c
......@@ -182,7 +182,7 @@ class PlottingFunctionFactory :
# point by point
absDiffData=None, rawDiffData=None,
goodInBothMask=None,
troubleMask=None, outsideEpsilonMask=None,
mismatchMask=None, outsideEpsilonMask=None,
# only used for plotting quiver data
aUData=None, aVData=None,
......@@ -193,7 +193,9 @@ class PlottingFunctionFactory :
binName=None, tupleName=None,
# the optional epsilon for comparison of a percent of A
epsilonPercent=None
epsilonPercent=None,
# the optional units for display
units_a=None, units_b=None
) : _abstract
......@@ -226,7 +228,7 @@ class BasicComparisonPlotsFunctionFactory (PlottingFunctionFactory) :
# point by point
absDiffData=None, rawDiffData=None,
goodInBothMask=None,
troubleMask=None, outsideEpsilonMask=None,
mismatchMask=None, outsideEpsilonMask=None,
# only used for plotting quiver data
aUData=None, aVData=None,
......@@ -237,7 +239,9 @@ class BasicComparisonPlotsFunctionFactory (PlottingFunctionFactory) :
binName=None, tupleName=None,
# the optional epsilon for comparison of a percent of A
epsilonPercent=None
epsilonPercent=None,
# the optional units for display
units_a=None, units_b=None
) :
......@@ -255,7 +259,7 @@ class BasicComparisonPlotsFunctionFactory (PlottingFunctionFactory) :
("Difference in\n" + variableDisplayName),
('Value of (Data File B - Data File A) at a Data Point'),
('Number of Data Points with a Given Difference'),
True)),
True, units=units_a)),
"histogram of the amount of difference in " + variableDisplayName,
"Hist.png", compared_fig_list)
# make the scatter plot
......@@ -270,7 +274,7 @@ class BasicComparisonPlotsFunctionFactory (PlottingFunctionFactory) :
"Value in File A vs Value in File B",
"File A Value", "File B Value",
outsideEpsilonMask[goodInBothMask],
epsilon)),
epsilon, units_x=units_a, units_y=units_b)),
"scatter plot of file a values vs file b values for " + variableDisplayName,
"Scatter.png", compared_fig_list)
......@@ -282,7 +286,8 @@ class BasicComparisonPlotsFunctionFactory (PlottingFunctionFactory) :
functionsToReturn['scatterD'] = ((lambda : figures.create_hexbin_plot(aData[goodInBothMask], bData[goodInBothMask],
"Value in File A vs Value in File B",
"File A Value", "File B Value", epsilon)),
"File A Value", "File B Value", epsilon,
units_x=units_a, units_y=units_b)),
"density of file a values vs file b values for " + variableDisplayName,
"Hex.png", compared_fig_list)
......@@ -316,7 +321,7 @@ class MappedContourPlotFunctionFactory (PlottingFunctionFactory) :
# point by point
absDiffData=None, rawDiffData=None,
goodInBothMask=None,
troubleMask=None, outsideEpsilonMask=None,
mismatchMask=None, outsideEpsilonMask=None,
# only used for plotting quiver data
aUData=None, aVData=None,
......@@ -327,7 +332,9 @@ class MappedContourPlotFunctionFactory (PlottingFunctionFactory) :
binName=None, tupleName=None,
# the optional epsilon for comparison of a percent of A
epsilonPercent=None
epsilonPercent=None,
# the optional units for display
units_a=None, units_b=None
) :
......@@ -366,7 +373,8 @@ class MappedContourPlotFunctionFactory (PlottingFunctionFactory) :
invalidMask=(~goodInAMask),
dataRanges=dataRanges or sharedRange,
dataRangeNames=dataRangeNames,
dataRangeColors=dataColors)),
dataRangeColors=dataColors,
units=units_a)),
variableDisplayName + " in file a",
"A.png", original_fig_list)
......@@ -383,7 +391,8 @@ class MappedContourPlotFunctionFactory (PlottingFunctionFactory) :
invalidMask=(~goodInBMask),
dataRanges=dataRanges or sharedRange,
dataRangeNames=dataRangeNames,
dataRangeColors=dataColors)),
dataRangeColors=dataColors,
units=units_b)),
variableDisplayName + " in file b",
"B.png", original_fig_list)
......@@ -403,7 +412,8 @@ class MappedContourPlotFunctionFactory (PlottingFunctionFactory) :
baseMapInstance, fullAxis,
("Absolute value of difference in\n"
+ variableDisplayName),
invalidMask=(~goodInBothMask))),
invalidMask=(~goodInBothMask),
units=units_a)),
"absolute value of difference in " + variableDisplayName,
"AbsDiff.png", compared_fig_list)
# make the subtractive difference plot
......@@ -422,11 +432,12 @@ class MappedContourPlotFunctionFactory (PlottingFunctionFactory) :
baseMapInstance, fullAxis,
("Value of (Data File B - Data File A) for\n"
+ variableDisplayName),
invalidMask=(~goodInBothMask))),
invalidMask=(~goodInBothMask),
units=units_a)),
"the difference in " + variableDisplayName,
"Diff.png", compared_fig_list)
# make the trouble data plot
if ('do_plot_trouble' not in doPlotSettingsDict) or (doPlotSettingsDict['do_plot_trouble']) :
# make the mismatch data plot
if ('do_plot_mismatch' not in doPlotSettingsDict) or (doPlotSettingsDict['do_plot_mismatch']) :
assert(aData.shape == bData.shape)
assert(goodInAMask.shape == goodInBMask.shape)
......@@ -441,17 +452,18 @@ class MappedContourPlotFunctionFactory (PlottingFunctionFactory) :
bDataCopy = bData[:]
tempMask = goodInAMask & (~goodInBMask)
bDataCopy[tempMask] = aData[tempMask]
functionsToReturn['trouble'] = ((lambda : mappedPlottingFunction(bDataCopy,
functionsToReturn['mismatch'] = ((lambda : mappedPlottingFunction(bDataCopy,
lonLatDataDict['common']['lat'],
lonLatDataDict['common']['lon'],
baseMapInstance, fullAxis,
("Areas of trouble data in\n" + variableDisplayName),
("Areas of mismatch data in\n" + variableDisplayName),
invalidMask=(~(goodInAMask | goodInBMask)),
colorMap=mediumGrayColorMap, tagData=troubleMask,
colorMap=mediumGrayColorMap, tagData=mismatchMask,
dataRanges=dataRanges,
dataRangeNames=dataRangeNames)), # TODO, does this need modification?
"trouble data in " + variableDisplayName,
"Trouble.png", compared_fig_list)
dataRangeNames=dataRangeNames,
units=units_a)), # TODO, does this need modification?
"mismatch data in " + variableDisplayName,
"Mismatch.png", compared_fig_list)
return functionsToReturn
......@@ -488,7 +500,7 @@ class MappedQuiverPlotFunctionFactory (PlottingFunctionFactory) :
# point by point
absDiffData=None, rawDiffData=None,
goodInBothMask=None,
troubleMask=None, outsideEpsilonMask=None,
mismatchMask=None, outsideEpsilonMask=None,
# only used for plotting quiver data
aUData=None, aVData=None,
......@@ -499,7 +511,9 @@ class MappedQuiverPlotFunctionFactory (PlottingFunctionFactory) :
binName=None, tupleName=None,
# the optional epsilon for comparison of a percent of A
epsilonPercent=None
epsilonPercent=None,
# the optional units for display
units_a=None, units_b=None
) :
......@@ -537,7 +551,8 @@ class MappedQuiverPlotFunctionFactory (PlottingFunctionFactory) :
baseMapInstance, fullAxis,
(variableDisplayName + "\nin File A"),
invalidMask=(~goodInAMask),
uData=aUData, vData=aVData)),
uData=aUData, vData=aVData,
units=units_a)),
variableDisplayName + " in file a",
"A.png", original_fig_list)
......@@ -552,7 +567,8 @@ class MappedQuiverPlotFunctionFactory (PlottingFunctionFactory) :
baseMapInstance, fullAxis,
(variableDisplayName + "\nin File B"),
invalidMask=(~ goodInBMask),
uData=bUData, vData=bVData)),
uData=bUData, vData=bVData,
units=units_b)),
variableDisplayName + " in file b",
"B.png", original_fig_list)
......@@ -582,7 +598,8 @@ class MappedQuiverPlotFunctionFactory (PlottingFunctionFactory) :
("Absolute value of difference in\n"
+ variableDisplayName),
invalidMask=(~ goodInBothMask),
uData=diffUData, vData=diffVData)),
uData=diffUData, vData=diffVData,
units=units_a)),
"absolute value of difference in " + variableDisplayName,
"AbsDiff.png", compared_fig_list)
# make the subtractive difference plot
......@@ -602,11 +619,12 @@ class MappedQuiverPlotFunctionFactory (PlottingFunctionFactory) :
("Value of (Data File B - Data File A) for\n"
+ variableDisplayName),
invalidMask=(~ goodInBothMask),
uData=diffUData, vData=diffVData)),
uData=diffUData, vData=diffVData,
units=units_a)),
"the difference in " + variableDisplayName,
"Diff.png", compared_fig_list)
# make the trouble data plot
if ('do_plot_trouble' not in doPlotSettingsDict) or (doPlotSettingsDict['do_plot_trouble']) :
# make the mismatch data plot
if ('do_plot_mismatch' not in doPlotSettingsDict) or (doPlotSettingsDict['do_plot_mismatch']) :
assert(aData.shape == bData.shape)
assert(goodInAMask.shape == goodInBMask.shape)
......@@ -621,19 +639,20 @@ class MappedQuiverPlotFunctionFactory (PlottingFunctionFactory) :
bDataCopy = bData[:]
tempMask = goodInAMask & (~goodInBMask)
bDataCopy[tempMask] = aData[tempMask]
functionsToReturn['trouble'] = ((lambda : mappedPlottingFunction(bDataCopy,
functionsToReturn['mismatch'] = ((lambda : mappedPlottingFunction(bDataCopy,
lonLatDataDict['common']['lat'],
lonLatDataDict['common']['lon'],
baseMapInstance, fullAxis,
("Areas of trouble data in\n" + variableDisplayName),
("Areas of mismatch data in\n" + variableDisplayName),
invalidMask=(~(goodInAMask | goodInBMask)),
colorMap=mediumGrayColorMap, tagData=troubleMask,
colorMap=mediumGrayColorMap, tagData=mismatchMask,
dataRanges=dataRanges,
dataRangeNames=dataRangeNames,
# TODO, does this need modification?
uData=bUData, vData=bVData)),
"trouble data in " + variableDisplayName,
"Trouble.png", compared_fig_list)
uData=bUData, vData=bVData,
units=units_a)),
"mismatch data in " + variableDisplayName,
"Mismatch.png", compared_fig_list)
return functionsToReturn
......@@ -666,7 +685,7 @@ class LinePlotsFunctionFactory (PlottingFunctionFactory) :
# point by point
absDiffData=None, rawDiffData=None,
goodInBothMask=None,
troubleMask=None, outsideEpsilonMask=None,
mismatchMask=None, outsideEpsilonMask=None,
# only used for plotting quiver data
aUData=None, aVData=None,
......@@ -677,7 +696,9 @@ class LinePlotsFunctionFactory (PlottingFunctionFactory) :
binName=None, tupleName=None,
# the optional epsilon for comparison of a percent of A
epsilonPercent=None
epsilonPercent=None,
# the optional units for display
units_a=None, units_b=None
) :
"""
......@@ -694,13 +715,13 @@ class LinePlotsFunctionFactory (PlottingFunctionFactory) :
assert(aData.shape == bData.shape)
# make all our data sets for plotting ahead of time for simplicity
aList = [(aData, ~goodInAMask, 'r', 'A data', None)]
bList = [(bData, ~goodInBMask, 'b', 'B data', None)]
absDiffList = [(absDiffData, ~goodInBothMask, '', 'abs. diff. data', None)]
subDiffList = [(rawDiffData, ~goodInBothMask, '', 'sub. diff. data', None)]
aList = [(aData, ~goodInAMask, 'r', 'A data', None, units_a)]
bList = [(bData, ~goodInBMask, 'b', 'B data', None, units_b)]
absDiffList = [(absDiffData, ~goodInBothMask, '', 'abs. diff. data', None, units_a)] # todo, should this be a units?
subDiffList = [(rawDiffData, ~goodInBothMask, '', 'sub. diff. data', None, units_a)] # todo, should this be a units?
troubleList = [(aData, ~goodInAMask, 'r', 'A data', troubleMask),
(bData, ~goodInBMask, 'b', 'B data', troubleMask)]
mismatchList = [(aData, ~goodInAMask, 'r', 'A data', mismatchMask, units_a),
(bData, ~goodInBMask, 'b', 'B data', mismatchMask, units_b)]
functionsToReturn = { }
......@@ -730,16 +751,17 @@ class LinePlotsFunctionFactory (PlottingFunctionFactory) :
# make the subtractive difference plot
if ('do_plot_sub_diff' not in doPlotSettingsDict) or (doPlotSettingsDict['do_plot_sub_diff']) :
functionsToReturn['diffSub'] = ((lambda: figures.create_line_plot_figure(subDiffList,
"Value of (Data File B - Data File A) for\n" + variableDisplayName)),
"Value of (Data File B - Data File A) for\n"
+ variableDisplayName)),
"the difference in " + variableDisplayName,
"Diff.png", compared_fig_list)
# make the trouble data plot
if ('do_plot_trouble' not in doPlotSettingsDict) or (doPlotSettingsDict['do_plot_trouble']) :
functionsToReturn['trouble'] = ((lambda: figures.create_line_plot_figure(troubleList,
"Areas of trouble data in\n" + variableDisplayName)),
"trouble data in " + variableDisplayName,
"Trouble.png", compared_fig_list)
# make the mismatch data plot
if ('do_plot_mismatch' not in doPlotSettingsDict) or (doPlotSettingsDict['do_plot_mismatch']) :
functionsToReturn['mismatch'] = ((lambda: figures.create_line_plot_figure(mismatchList,
"Areas of mismatch data in\n" + variableDisplayName)),
"mismatch data in " + variableDisplayName,
"Mismatch.png", compared_fig_list)
return functionsToReturn
......@@ -768,7 +790,7 @@ class BinTupleAnalysisFunctionFactory (PlottingFunctionFactory) :
# point by point
absDiffData=None, rawDiffData=None,
goodInBothMask=None,
troubleMask=None, outsideEpsilonMask=None,
mismatchMask=None, outsideEpsilonMask=None,
# only used for plotting quiver data
aUData=None, aVData=None,
......@@ -779,7 +801,9 @@ class BinTupleAnalysisFunctionFactory (PlottingFunctionFactory) :
binName=None, tupleName=None,
# the optional epsilon for comparison of a percent of A
epsilonPercent=None
epsilonPercent=None,
# the optional units for display
units_a=None, units_b=None
) :
"""
......@@ -815,7 +839,7 @@ class BinTupleAnalysisFunctionFactory (PlottingFunctionFactory) :
absDiffData = reorderMapObject.reorder_for_bin_tuple(absDiffData)
rawDiffData = reorderMapObject.reorder_for_bin_tuple(rawDiffData)
goodInBothMask = reorderMapObject.reorder_for_bin_tuple(goodInBothMask)
troubleMask = reorderMapObject.reorder_for_bin_tuple(troubleMask)
mismatchMask = reorderMapObject.reorder_for_bin_tuple(mismatchMask)
outsideEpsilonMask = reorderMapObject.reorder_for_bin_tuple(outsideEpsilonMask)
# our list of functions that will later create the plots
......@@ -835,7 +859,7 @@ class BinTupleAnalysisFunctionFactory (PlottingFunctionFactory) :
functionsToReturn['multi-scatter'] = ((lambda : figures.create_complex_scatter_plot(scatterPlotList,
"Value in File A vs Value in File B, Colored by Bin",
"File A Value", "File B Value",
epsilon)),
epsilon, units_x=units_a, units_y=units_b)),
"scatter plot of file a values vs file b values for " + variableDisplayName + " by bin",
"MultiScatter.png", compared_fig_list)
......@@ -861,7 +885,7 @@ class BinTupleAnalysisFunctionFactory (PlottingFunctionFactory) :
"\nfor " + binName + " # " + str(binNumber + 1)),
('RMS Difference across ' + tupleName + ' dimension'),
('Number of Cases with a Given RMS Diff.'),
True)
True, units=units_a)
functionsToReturn[str(binNumber + 1) + 'histogram'] = (make_histogram,
"histogram of rms differences in " + variableDisplayName,
str(binNumber + 1) + "Hist.png", new_list)
......@@ -903,8 +927,8 @@ class BinTupleAnalysisFunctionFactory (PlottingFunctionFactory) :
caseNumText = ''
for caseIndex in caseIndexes :
caseNumText = caseNumText + str(caseIndex)
dataList = [(aData[binNumber][caseNumber], ~goodInAMask[binNumber][caseNumber], 'r', 'A case', None),
(bData[binNumber][caseNumber], ~goodInBMask[binNumber][caseNumber], 'b', 'B case', None)]
dataList = [(aData[binNumber][caseNumber], ~goodInAMask[binNumber][caseNumber], 'r', 'A case', None, units_a),
(bData[binNumber][caseNumber], ~goodInBMask[binNumber][caseNumber], 'b', 'B case', None, units_b)]
def make_lineplot(data=dataList, binNumber=binNumber, caseNumberText=caseNumText):
return figures.create_line_plot_figure(data,
variableDisplayName + " in both files" + "\n" + "for "
......@@ -912,8 +936,8 @@ class BinTupleAnalysisFunctionFactory (PlottingFunctionFactory) :
+ caseNumberText)
dataDiff = aData[binNumber][caseNumber] - bData[binNumber][caseNumber]
maskDiff = ~goodInAMask[binNumber][caseNumber] | ~goodInBMask[binNumber][caseNumber]
def make_diffplot(data=dataDiff, badMask=maskDiff, binNumber=binNumber, caseNumberText=caseNumText):
return figures.create_line_plot_figure([(data, badMask, 'm', 'A - B', None)],
def make_diffplot(data=dataDiff, badMask=maskDiff, binNumber=binNumber, caseNumberText=caseNumText, units=units_a):
return figures.create_line_plot_figure([(data, badMask, 'm', 'A - B', None, units)],
"Value of " + variableDisplayName + " in File A - the value in File B\n" +
"for " + binName + " # " + str(binNumber + 1) + " and case # " + caseNumberText)
......@@ -956,7 +980,7 @@ class IMShowPlotFunctionFactory (PlottingFunctionFactory) :
# point by point
absDiffData=None, rawDiffData=None,
goodInBothMask=None,
troubleMask=None, outsideEpsilonMask=None,
mismatchMask=None, outsideEpsilonMask=None,
# only used for plotting quiver data
aUData=None, aVData=None,
......@@ -967,7 +991,9 @@ class IMShowPlotFunctionFactory (PlottingFunctionFactory) :
binName=None, tupleName=None,
# the optional epsilon for comparison of a percent of A
epsilonPercent=None
epsilonPercent=None,
# the optional units for display
units_a=None, units_b=None
) :
"""
......@@ -995,7 +1021,8 @@ class IMShowPlotFunctionFactory (PlottingFunctionFactory) :
assert(aData.shape == goodInAMask.shape)
functionsToReturn['originalA'] = ((lambda: figures.create_simple_figure(aData, variableDisplayName + "\nin File A",
invalidMask=~goodInAMask, colorbarLimits=sharedRange)),
invalidMask=~goodInAMask, colorbarLimits=sharedRange,
units=units_a)),
variableDisplayName + " in file a",
"A.png", original_fig_list)
......@@ -1003,7 +1030,8 @@ class IMShowPlotFunctionFactory (PlottingFunctionFactory) :
assert(bData.shape == goodInBMask.shape)
functionsToReturn['originalB'] = ((lambda: figures.create_simple_figure(bData, variableDisplayName + "\nin File B",
invalidMask=~goodInBMask, colorbarLimits=sharedRange)),
invalidMask=~goodInBMask, colorbarLimits=sharedRange,
units=units_b)),
variableDisplayName + " in file b",
"B.png", original_fig_list)
......@@ -1016,7 +1044,7 @@ class IMShowPlotFunctionFactory (PlottingFunctionFactory) :
functionsToReturn['diffAbs'] = ((lambda: figures.create_simple_figure(absDiffData,
"Absolute value of difference in\n" + variableDisplayName,
invalidMask=~goodInBothMask)),
invalidMask=~goodInBothMask, units=units_a)),
"absolute value of difference in " + variableDisplayName,
"AbsDiff.png", compared_fig_list)
# make the subtractive difference plot
......@@ -1028,19 +1056,19 @@ class IMShowPlotFunctionFactory (PlottingFunctionFactory) :
functionsToReturn['diffSub'] = ((lambda: figures.create_simple_figure(rawDiffData,
"Value of (Data File B - Data File A) for\n" + variableDisplayName,
invalidMask=~goodInBothMask)),
invalidMask=~goodInBothMask, units=units_a)),
"the difference in " + variableDisplayName,
"Diff.png", compared_fig_list)
# make the trouble data plot
if ('do_plot_trouble' not in doPlotSettingsDict) or (doPlotSettingsDict['do_plot_trouble']) :
# make the mismatch data plot
if ('do_plot_mismatch' not in doPlotSettingsDict) or (doPlotSettingsDict['do_plot_mismatch']) :
assert(goodInAMask is not None)
assert(goodInBMask is not None)
assert(goodInAMask.shape == goodInBMask.shape)
assert(aData.shape == bData.shape)
assert(aData.shape == goodInAMask.shape)
assert(troubleMask is not None)
assert(troubleMask.shape == aData.shape)
assert(mismatchMask is not None)
assert(mismatchMask.shape == aData.shape)
# this is not an optimal solution, but we need to have at least somewhat valid data at any mismatched points so
......@@ -1048,11 +1076,11 @@ class IMShowPlotFunctionFactory (PlottingFunctionFactory) :
bDataCopy = bData[:]
tempMask = goodInAMask & (~goodInBMask)
bDataCopy[tempMask] = aData[tempMask]
functionsToReturn['trouble'] = ((lambda: figures.create_simple_figure(bDataCopy, "Areas of trouble data in\n" + variableDisplayName,
invalidMask=~(goodInAMask | goodInBMask), tagData=troubleMask,
colorMap=mediumGrayColorMap)),
"trouble data in " + variableDisplayName,
"Trouble.png", compared_fig_list)
functionsToReturn['mismatch'] = ((lambda: figures.create_simple_figure(bDataCopy, "Areas of mismatch data in\n" + variableDisplayName,
invalidMask=~(goodInAMask | goodInBMask), tagData=mismatchMask,
colorMap=mediumGrayColorMap, units=units_a)),
"mismatch data in " + variableDisplayName,
"Mismatch.png", compared_fig_list)
return functionsToReturn
......
pyglance/glance/stats.py
+ 11
− 11
View file @ 71c6a26c
......@@ -411,8 +411,8 @@ class NumericalComparisonStatistics (StatisticalData) :
diff_outside_epsilon_fraction - the fraction of points that fall outside the acceptable epsilon settings
perfect_match_count - the number of points that match perfectly between the sets
perfect_match_fraction - the fraction of points that match perfectly between the sets
trouble_points_count - the number of points that have possible issues according to the current analysis
trouble_points_fraction - the fraction of points that have possible issues according to the current analysis
mismatch_points_count - the number of points that have possible issues according to the current analysis
mismatch_points_fraction - the fraction of points that have possible issues according to the current analysis
It may also contain additional statistics. This is indicated by the does_include_simple boolean.
The possible additional statistics include:
......@@ -442,9 +442,9 @@ class NumericalComparisonStatistics (StatisticalData) :
'rms_diff': "root mean square (RMS) difference of finite values",
'r-squared correlation': "the square of the r correlation (see correlation)",
'std_diff': "standard deviation of difference of finite values",
'trouble_points_count': 'number of points that differ in finite/missing status between the input data sets A and B,' +
'mismatch_points_count': 'number of points that differ in finite/missing status between the input data sets A and B,' +
' or are unacceptable when compared according to the current epsilon definitions',
'trouble_points_fraction': 'fraction of points that differ in finite/missing status between the input data sets A and B,' +
'mismatch_points_fraction': 'fraction of points that differ in finite/missing status between the input data sets A and B,' +
' or are unacceptable when compared according to the current epsilon definitions',
}
......@@ -461,13 +461,13 @@ class NumericalComparisonStatistics (StatisticalData) :
# pull out some info we will use later
valid_in_both = diffInfoObject.diff_data_object.masks.valid_mask
outside_epsilon_mask = diffInfoObject.diff_data_object.masks.outside_epsilon_mask
trouble_mask = diffInfoObject.diff_data_object.masks.trouble_mask
mismatch_mask = diffInfoObject.diff_data_object.masks.mismatch_mask
aData = diffInfoObject.a_data_object.data
bData = diffInfoObject.b_data_object.data
assert (valid_in_both.shape == outside_epsilon_mask.shape)
assert (outside_epsilon_mask.shape == trouble_mask.shape)
assert (trouble_mask.shape == aData.shape)
assert (outside_epsilon_mask.shape == mismatch_mask.shape)
assert (mismatch_mask.shape == aData.shape)
assert (aData.shape == bData.shape)
# fill in some simple statistics
......@@ -476,13 +476,13 @@ class NumericalComparisonStatistics (StatisticalData) :
goodMask=valid_in_both)
self.correlation = delta.compute_correlation(aData, bData, valid_in_both)
self.r_squared_correlation = self.correlation * self.correlation
self.trouble_points_count = np.sum(trouble_mask)
self.mismatch_points_count = np.sum(mismatch_mask)
# we actually want the total number of _finite_ values rather than all the data
total_num_finite_values = np.sum(valid_in_both)
# calculate some more complex statistics
self.trouble_points_fraction = float(self.trouble_points_count) / float(aData.size)
self.mismatch_points_fraction = float(self.mismatch_points_count) / float(aData.size)
# be careful not to divide by zero if we don't have finite data
if total_num_finite_values > 0 :
self.diff_outside_epsilon_fraction = float(self.diff_outside_epsilon_count) / float(total_num_finite_values)
......@@ -522,8 +522,8 @@ class NumericalComparisonStatistics (StatisticalData) :
'diff_outside_epsilon_fraction': self.diff_outside_epsilon_fraction,
'perfect_match_count': self.perfect_match_count,
'perfect_match_fraction': self.perfect_match_fraction,
'trouble_points_count': self.trouble_points_count,
'trouble_points_fraction': self.trouble_points_fraction
'mismatch_points_count': self.mismatch_points_count,
'mismatch_points_fraction': self.mismatch_points_fraction
}
toReturn.update(self.temp_analysis)
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment