From cc8fd781f773a2599a69f8393e7d6462977ed98f Mon Sep 17 00:00:00 2001
From: "(no author)" <(no author)@8a9318a1-56ba-4d59-b755-99d26321be01>
Date: Fri, 24 Jul 2009 19:38:41 +0000
Subject: [PATCH] updating plotting and some fixes to delta functionality
git-svn-id: https://svn.ssec.wisc.edu/repos/glance/trunk@38 8a9318a1-56ba-4d59-b755-99d26321be01
---
pyglance/glance/compare.py | 72 ++++++++++----
pyglance/glance/delta.py | 29 +++---
pyglance/glance/io.py | 2 +-
pyglance/glance/plot.py | 145 ++++++++++++++++-------------
pyglance/glance/variablereport.txt | 5 +-
5 files changed, 155 insertions(+), 98 deletions(-)
diff --git a/pyglance/glance/compare.py b/pyglance/glance/compare.py
index 5e7738a..444eb00 100644
--- a/pyglance/glance/compare.py
+++ b/pyglance/glance/compare.py
@@ -27,6 +27,7 @@ glance_default_latitude_name = 'pixel_latitude'
# these are the built in default settings
glance_analysis_defaults = {'epsilon': 0.0,
'missing_value': None,
+ 'missing_value_alt_in_b': None,
'epsilon_failure_tolerance': 0.0,
'nonfinite_data_tolerance': 0.0
}
@@ -134,7 +135,12 @@ def _resolve_names(fileAObject, fileBObject, defaultValues,
# but override the values that would have been determined by _parse_varnames
finalNames[name]['variable_name'] = name
finalNames[name]['epsilon'] = epsilon
- finalNames[name]['missing_value'] = missing
+ missing_b = missing
+ if missing is None:
+ missing = fileAObject.missing_value(name)
+ missing_b = fileBObject.missing_value(name)
+ finalNames[name]['missing_value'] = missing
+ finalNames[name]['missing_value_alt_in_b'] = missing_b
# otherwise just do the ones the user asked for
else :
@@ -149,6 +155,7 @@ def _resolve_names(fileAObject, fileBObject, defaultValues,
finalNames[name] = defaultValues.copy()
finalNames[name]['variable_name'] = name
finalNames[name].update(requestedNames[name])
+ # TODO what's the correct behavior here for missing?
else:
# format command line input similarly to the stuff from the config file
print (requestedNames)
@@ -162,7 +169,7 @@ def _resolve_names(fileAObject, fileBObject, defaultValues,
# but override the values that would have been determined by _parse_varnames
finalNames[name]['variable_name'] = name
finalNames[name]['epsilon'] = epsilon
- finalNames[name]['missing_value'] = missing
+ finalNames[name]['missing_value'] = missing # TODO, what's the correct behavior here?
LOG.debug("Final selected set of variables to analyze:")
LOG.debug(str(finalNames))
@@ -303,10 +310,13 @@ def _check_lon_lat_equality(longitudeA, latitudeA,
combinedIgnoreMask = ignoreMaskA | ignoreMaskB
# get information about how the latitude and longitude differ
- longitudeDiff, finiteLongitudeMask = delta.diff(longitudeA, longitudeB, llepsilon, ignoreMask=combinedIgnoreMask)[0:2]
- latitudeDiff, finiteLatitudeMask = delta.diff(latitudeA, latitudeB, llepsilon, ignoreMask=combinedIgnoreMask)[0:2]
- lon_lat_not_equal_mask = ((abs(longitudeDiff) > llepsilon) | (abs(latitudeDiff) > llepsilon)) & (~combinedIgnoreMask) \
- & finiteLatitudeMask & finiteLongitudeMask
+ longitudeDiff, finiteLongitudeMask, _, _, _, lon_not_equal_mask = delta.diff(longitudeA, longitudeB,
+ llepsilon,
+ ignoreMask=combinedIgnoreMask)
+ latitudeDiff, finiteLatitudeMask, _, _, _, lat_not_equal_mask = delta.diff(latitudeA, latitudeB,
+ llepsilon,
+ ignoreMask=combinedIgnoreMask)
+ lon_lat_not_equal_mask = lon_not_equal_mask & lat_not_equal_mask
lon_lat_not_equal_points_count = sum(lon_lat_not_equal_mask.ravel())
lon_lat_not_equal_points_percent = (float(lon_lat_not_equal_points_count) / float(lon_lat_not_equal_mask.size)) * 100.0
@@ -396,6 +406,30 @@ def _compare_spatial_invalidity(invalid_in_a_mask, invalid_in_b_mask, spatial_in
return invalid_in_common_mask, spatial_info, longitude_common, latitude_common
+def _open_and_process_files (args, numFilesExpected):
+ """
+ open files listed in the args and get information about the variables in them
+ """
+ # get all the file names
+ fileNames = args[:numFilesExpected]
+ # open all the files & get their variable names
+ files = {}
+ commonNames = None
+ for fileName in fileNames:
+ LOG.info("opening %s" % fileName)
+ files[fileName] = {}
+ tempFileObject = (io.open(fileName))
+ files[fileName]['fileObject'] = tempFileObject
+ tempNames = set(tempFileObject())
+ files[fileName]['varNames'] = tempNames
+ if commonNames is None :
+ commonNames = tempNames
+ else :
+ commonNames = commonNames.intersection(tempNames)
+ files['commonVarNames'] = commonNames
+
+ return files
+
def main():
import optparse
usage = """
@@ -469,7 +503,7 @@ python -m glance.compare plotDiffs A.hdf B.hdf [optional output path]
"""compare sdr_cris output
parameters are variable name followed by detector number
sdr_cris desired.h5 actual.h5 ESRealLW 0
- """
+ """ # TODO ******* standardize with method?
afn,bfn = args[:2]
LOG.info("opening %s" % afn)
a = io.open(afn)
@@ -491,7 +525,7 @@ python -m glance.compare plotDiffs A.hdf B.hdf [optional output path]
"""gives statistics for dataset comparisons against truth with and without noise
usage: noisecheck truth-file noise-file actual-file variable1{:epsilon{:missing}} {variable2...}
glance noisecheck /Volumes/snaapy/data/justins/abi_graffir/coreg/pure/l2_data/geocatL2.GOES-R.2005155.220000.hdf.gz /Volumes/snaapy/data/justins/abi_graffir/noise/noise1x/l2_data/geocatL2.GOES-R.2005155.220000.hdf
- """
+ """ # TODO ******* standardize with method?
afn,noizfn,bfn = args[:3]
LOG.info("opening truth file %s" % afn)
a = io.open(afn)
@@ -542,24 +576,21 @@ python -m glance.compare plotDiffs A.hdf B.hdf [optional output path]
python -m glance.compare -w stats --epsilon=0.00001 A.hdf A.hdf imager_prof_retr_abi_total_precipitable_water_low::-999
"""
afn,bfn = args[:2]
- LOG.info("opening %s" % afn)
- a = io.open(afn)
- LOG.info("opening %s" % bfn)
- b = io.open(bfn)
- anames = set(a())
- bnames = set(b())
- cnames = anames.intersection(bnames) # common names
+ filesInfo = _open_and_process_files(args, 2)
+ aFile = filesInfo[afn]['fileObject']
+ bFile = filesInfo[bfn]['fileObject']
+
pats = args[2:] or ['.*']
- names = _parse_varnames( cnames, pats, options.epsilon, options.missing )
+ names = _parse_varnames( filesInfo['commonVarNames'], pats, options.epsilon, options.missing )
LOG.debug(str(names))
doc_each = (options.verbose or options.debug) and len(names)==1
doc_atend = (options.verbose or options.debug) and len(names)!=1
for name,epsilon,missing in names:
- aData = a[name]
- bData = b[name]
+ aData = aFile[name]
+ bData = bFile[name]
if missing is None:
- amiss = a.missing_value(name)
- bmiss = b.missing_value(name)
+ amiss = aFile.missing_value(name)
+ bmiss = bFile.missing_value(name)
else:
amiss,bmiss = missing,missing
LOG.debug('comparing %s with epsilon %s and missing %s,%s' % (name,epsilon,amiss,bmiss))
@@ -753,6 +784,7 @@ python -m glance.compare plotDiffs A.hdf B.hdf [optional output path]
if (runInfo['shouldIncludeImages']) :
doShortCircuit = ('short_circuit_diffs' in runInfo) and runInfo['short_circuit_diffs']
# create the images comparing that variable
+ print("\tcreating figures for: " + displayName)
plot.plot_and_save_figure_comparison(aData, bData, varRunInfo,
files['file A']['path'],
files['file B']['path'],
diff --git a/pyglance/glance/delta.py b/pyglance/glance/delta.py
index fa31429..71d7515 100644
--- a/pyglance/glance/delta.py
+++ b/pyglance/glance/delta.py
@@ -59,7 +59,10 @@ def diff(a, b, epsilon=0., (amissing,bmissing)=(None,None), ignoreMask=None):
# trouble areas - mismatched nans, mismatched missing-values, differences > epsilon
trouble = (anfin ^ bnfin) | (amis ^ bmis) | (abs(d)>epsilon)
- return d, mask, trouble, (anfin, bnfin), (amis, bmis)
+ # the outside epsilon mask
+ outeps = (abs(d) > epsilon) & mask
+
+ return d, mask, trouble, (anfin, bnfin), (amis, bmis), outeps
def corr(x,y,mask):
"compute correlation coefficient"
@@ -81,7 +84,7 @@ def rms_corr_withnoise(truth, actual, noiz, epsilon=0., (amissing,bmissing)=(Non
x=truth
y=actual
z=noiz
- d,good,bad,_,_ = diff(x,y,epsilon,(amissing,bmissing))
+ d,good,bad,_,_,_ = diff(x,y,epsilon,(amissing,bmissing))
# compute RMS error
rmse = sqrt(sum(d[good]**2)) / d.size
gf = good.flatten()
@@ -97,7 +100,7 @@ def rms_corr_withnoise(truth, actual, noiz, epsilon=0., (amissing,bmissing)=(Non
xpn[good] += z[good]
xpnf = xpn.flatten()[gf]
# compute RMS error versus noise
- dpn,good,bad,_,_ = diff(xpn,y,epsilon,(amissing,bmissing))
+ dpn,good,bad,_,_,_ = diff(xpn,y,epsilon,(amissing,bmissing))
rmsepn = sqrt(sum(dpn[good]**2)) / d.size
assert(sum(~isfinite(xpnf))==0)
rpn = compute_r(xpnf,yf)[0]
@@ -109,12 +112,13 @@ def rms_corr_withnoise(truth, actual, noiz, epsilon=0., (amissing,bmissing)=(Non
}
def stats(diffData, mask, *etc):
- rms = sum(abs(diffData[mask] ** 2)) / diffData.size
+ absDiffData = abs(diffData)
+ rms = sqrt( sum(diffData[mask] ** 2) / sum(mask) )
return { 'rms_diff': rms,
- 'std_diff': std(diffData[mask]),
- 'mean_diff': mean(diffData[mask]),
- 'median_diff': median(diffData[mask]),
- 'max_diff': max(diffData[mask])
+ 'std_diff': std(absDiffData[mask]),
+ 'mean_diff': mean(absDiffData[mask]),
+ 'median_diff': median(absDiffData[mask]),
+ 'max_diff': max(absDiffData[mask])
}
def _get_num_perfect(a, b, ignoreMask=None):
@@ -230,14 +234,15 @@ def _get_general_data_stats(a_missing_value, b_missing_value, epsilon, trouble_m
return general_stats
-def _get_numerical_data_stats(a, b, diff_data, data_is_finite_mask, epsilon, additional_statistics={}) :
+def _get_numerical_data_stats(a, b, diff_data, data_is_finite_mask, outside_epsilon_mask,
+ additional_statistics={}) :
"""
Get a list of numerical comparison related statistics about a and b,
given a and b and some other information about them.
the return value will be a dictionary of statistics
"""
# calculate our various statistics
- num_finite_values_too_different = sum(abs(diff_data[data_is_finite_mask]) > epsilon)
+ num_finite_values_too_different = sum(outside_epsilon_mask)
num_perfect = _get_num_perfect(a, b, ~data_is_finite_mask)
r_corr = corr(a, b, data_is_finite_mask)
@@ -269,7 +274,7 @@ def summarize(a, b, epsilon=0., (a_missing_value, b_missing_value)=(None,None),
ignoreInBMask = zeros(b.shape, dtype=bool)
ignoreMask = ignoreInAMask | ignoreInBMask
- d, mask, trouble, (anfin, bnfin), (amis, bmis) = nfo = diff(a,b,epsilon,(a_missing_value, b_missing_value),ignoreMask)
+ d, mask, trouble, (anfin, bnfin), (amis, bmis), outside_epsilon = nfo = diff(a,b,epsilon,(a_missing_value, b_missing_value),ignoreMask)
# build some other finite data masks that we'll need
finite_a_mask = ~(anfin | amis)
@@ -284,7 +289,7 @@ def summarize(a, b, epsilon=0., (a_missing_value, b_missing_value)=(None,None),
general_stats = _get_general_data_stats(a_missing_value, b_missing_value, epsilon, trouble, ignoreInAMask, ignoreInBMask)
additional_statistics = stats(*nfo) # grab some additional comparison statistics
- comparison_stats = _get_numerical_data_stats(a, b, d, finite_mask, epsilon, additional_statistics)
+ comparison_stats = _get_numerical_data_stats(a, b, d, finite_mask, outside_epsilon, additional_statistics)
nan_stats = _get_nan_stats(a, b)
missing_stats = _get_missing_value_stats(amis, bmis)
finite_stats = _get_finite_data_stats(finite_a_mask, finite_b_mask)
diff --git a/pyglance/glance/io.py b/pyglance/glance/io.py
index 0e695be..f65a754 100644
--- a/pyglance/glance/io.py
+++ b/pyglance/glance/io.py
@@ -76,7 +76,7 @@ class hdf(SD):
"""
if (scaling_method == 0) :
return raw_data_copy
- if not ((scaling_method is None) or (scaling_method > 1)) :
+ if not ((scaling_method is None) or (atoi(scaling_method) > 1)) :
LOG.warn ('Scaling method of \"' + str(scaling_method) + '\" will be ignored in favor of hdf standard method. '
+ 'This may cause problems with data consistency')
diff --git a/pyglance/glance/plot.py b/pyglance/glance/plot.py
index 4c26693..078ec1f 100644
--- a/pyglance/glance/plot.py
+++ b/pyglance/glance/plot.py
@@ -185,7 +185,7 @@ def _create_mapped_figure(data, latitude, longitude, boundingAxes, title,
# build extra info to go to the map plotting function
kwargs = {}
- # figure the range for the color bars
+ # figure the range for the color bars TODO this should be user controllable for discrete data
if not (data is None) :
# todo, the use off the offset here is covering a problem with
# contourf hiding data exactly at the end of the range and should
@@ -204,9 +204,9 @@ def _create_mapped_figure(data, latitude, longitude, boundingAxes, title,
latitudeRange = abs(boundingAxes[3] - boundingAxes[2])
# chose the projection based on the range we have to cover
if (longitudeRange > 180) :
- kwargs['projection'] = 'merc' # use a mercator projection to show the whole world
- elif (longitudeRange > 80) or (latitudeRange > 80) :
- kwargs['projection'] = 'ortho'
+ kwargs['projection'] = 'mill' # use a miller cylindrical projection to show the whole world
+ elif (longitudeRange > 100) or (latitudeRange > 70) :
+ kwargs['projection'] = 'ortho' # use an orthographic projection to show half the globe
# otherwise the default is just fine!
# draw our data placed on a map
@@ -340,9 +340,11 @@ def plot_and_save_spacial_trouble(longitude, latitude,
visibleAxes = _get_visible_axes(longitude, latitude, spaciallyInvalidMask)
# make the figure
+ LOG.info("Creating spatial trouble image")
spatialTroubleFig = _create_mapped_figure(None, latitude, longitude, visibleAxes, title,
spaciallyInvalidMask, None, spacialTroubleMask)
# save the figure
+ LOG.info("Saving spatial trouble image")
spatialTroubleFig.savefig(outputPath + "/" + fileBaseName + "." + fileNameDiscriminator + ".png", dpi=200)
# we may also save a smaller versions of the figure
@@ -383,17 +385,17 @@ def plot_and_save_figure_comparison(aData, bData,
"""
# if we weren't given a variable display name,
# just use the standard variable name instead
- variableDisplayName = variableRunInfo['variable_name']
+ variableName = variableRunInfo['variable_name']
+ variableDisplayName = variableName
if 'display_name' in variableRunInfo :
variableDisplayName = variableRunInfo['display_name']
- print("\tCreating figures for: " + variableDisplayName)
-
# compare the two data sets to get our difference data and trouble info
rawDiffData, goodMask, troubleMask, (aNotFiniteMask, bNotFiniteMask), \
- (aMissingMask, bMissingMask) = delta.diff(aData, bData, variableRunInfo['epsilon'],
- (variableRunInfo['missing_value'], variableRunInfo['missing_value']),
- spaciallyInvalidMaskBoth)
+ (aMissingMask, bMissingMask), outsideEpsilonMask = delta.diff(aData, bData, variableRunInfo['epsilon'],
+ (variableRunInfo['missing_value'],
+ variableRunInfo['missing_value']),
+ spaciallyInvalidMaskBoth)
diffData = np.abs(rawDiffData) # we want to show the distance between our two, rather than which one's bigger
# mark where our invalid data is for each of the files (a and b)
@@ -401,8 +403,6 @@ def plot_and_save_figure_comparison(aData, bData,
invalidDataMaskB = bMissingMask | bNotFiniteMask
# this mask potentially represents data we don't want to try to plot in our diff because it may be malformed
everyThingWrongButEpsilon = spaciallyInvalidMaskBoth | invalidDataMaskA | invalidDataMaskB
- # use an exclusive or to get a mask for just the points deemed "bad" by epsilon comparison
- tooDifferentMask = everyThingWrongButEpsilon ^ troubleMask
# calculate the bounding range for the display
# this is in the form [longitude min, longitude max, latitude min, latitude max]
@@ -413,68 +413,85 @@ def plot_and_save_figure_comparison(aData, bData,
LOG.debug ("visible axes in B: " + str(visibleAxesB))
LOG.debug ("visible axes in Both: " + str(visibleAxesBoth))
- # make the original data figures
- print("\t\tcreating image of file a")
+ # the original data figures
+ LOG.info("\t\tcreating image of file a")
figureA = _create_mapped_figure(aData, latitudeAData, longitudeAData, visibleAxesA,
(variableDisplayName + "\nin File A"),
invalidMask=(spaciallyInvalidMaskA | invalidDataMaskA))
- print("\t\tcreating image of file b")
+ LOG.info("\t\tsaving image of file a")
+ figureA.savefig(outputPath + "/" + variableName + ".A.png", dpi=200)
+
+ LOG.info("\t\tcreating image of file b")
figureB = _create_mapped_figure(bData, latitudeBData, longitudeBData, visibleAxesB,
(variableDisplayName + "\nin File B"),
invalidMask=(spaciallyInvalidMaskB | invalidDataMaskB))
+ LOG.info("\t\tsaving image of file b")
+ figureB.savefig(outputPath + "/" + variableName + ".B.png", dpi=200)
# make the data comparison figures
- if not shortCircuitComparisons :
- print("\t\tcreating image of the absolute value of difference")
- figureAbsDiff = _create_mapped_figure(diffData, latitudeCommonData, longitudeCommonData, visibleAxesBoth,
- ("Absolute value of difference in\n" + variableDisplayName),
- invalidMask=(everyThingWrongButEpsilon))
- print("\t\tcreating image of the difference")
- figureDiff = _create_mapped_figure(rawDiffData, latitudeCommonData, longitudeCommonData, visibleAxesBoth,
- ("Value of (Data File B - Data File A) for\n" + variableDisplayName),
- invalidMask=(everyThingWrongButEpsilon))
- # this figure is more complex because we want to mark the trouble points on it
- print("\t\tcreating image marking trouble data")
- figureBadDataInDiff = _create_mapped_figure(bData, latitudeCommonData, longitudeCommonData, visibleAxesBoth,
- ("Areas of trouble data in\n" + variableDisplayName),
- spaciallyInvalidMaskBoth | invalidDataMaskB,
- mediumGrayColorMap, troubleMask)
- # a histogram of the values of fileA - file B so that the distribution of error is visible (hopefully)
- print("\t\tcreating histogram of the amount of difference")
- numBinsToUse = 50
- diffHistogramFigure = _create_histogram(rawDiffData[~everyThingWrongButEpsilon].ravel(), numBinsToUse,
- ("Difference in\n" + variableDisplayName),
- ('Value of (Data File B - Data File A) at a Data Point'),
- ('Number of Data Points with a Given Difference'),
- True)
- # scatter plot of file a and b comparison
- print("\t\tcreating scatter plot of file a values vs file b values")
- diffScatterPlot = _create_scatter_plot(aData[~everyThingWrongButEpsilon].ravel(), bData[~everyThingWrongButEpsilon].ravel(),
- "Value in File A vs Value in File B", "File A Value", "File B Value",
- tooDifferentMask[~everyThingWrongButEpsilon].ravel(), variableRunInfo['epsilon'])
-
- # save the figures to disk
- variableName = variableRunInfo['variable_name']
- print("\tSaving figures for: " + variableDisplayName)
- print("\t\tsaving image of file a")
- figureA.savefig(outputPath + "/" + variableName + ".A.png", dpi=200)
- print("\t\tsaving image of file b")
- figureB.savefig(outputPath + "/" + variableName + ".B.png", dpi=200)
- if not shortCircuitComparisons :
- print("\t\tsaving image of the absolute value of difference")
- figureAbsDiff.savefig(outputPath + "/" + variableName + ".AbsDiff.png", dpi=200)
- print("\t\tsaving image of the difference")
- figureDiff.savefig(outputPath + "/" + variableName + ".Diff.png", dpi=200)
- print("\t\tsaving image marking trouble data")
- figureBadDataInDiff.savefig(outputPath + "/" + variableName + ".Trouble.png", dpi=200)
- print("\t\tsaving histogram of the amount of difference")
- diffHistogramFigure.savefig(outputPath + "/" + variableName + ".Hist.png", dpi=200)
- print("\t\tsaving scatter plot of file a values vs file b values")
- diffScatterPlot.savefig(outputPath + "/" + variableName + ".Scatter.png", dpi=200)
+ #if not shortCircuitComparisons :
+ LOG.info("\t\tcreating image of the absolute value of difference")
+ figureAbsDiff = _create_mapped_figure(diffData, latitudeCommonData, longitudeCommonData, visibleAxesBoth,
+ ("Absolute value of difference in\n" + variableDisplayName),
+ invalidMask=(everyThingWrongButEpsilon))
+ LOG.info("\t\tsaving image of the absolute value of difference")
+ figureAbsDiff.savefig(outputPath + "/" + variableName + ".AbsDiff.png", dpi=200)
+
+ LOG.info("\t\tcreating image of the difference")
+ figureDiff = _create_mapped_figure(rawDiffData, latitudeCommonData, longitudeCommonData, visibleAxesBoth,
+ ("Value of (Data File B - Data File A) for\n" + variableDisplayName),
+ invalidMask=(everyThingWrongButEpsilon))
+ LOG.info("\t\tsaving image of the difference")
+ figureDiff.savefig(outputPath + "/" + variableName + ".Diff.png", dpi=200)
+
+ # this figure is more complex because we want to mark the trouble points on it
+ LOG.info("\t\tcreating image marking trouble data")
+ figureBadDataInDiff = _create_mapped_figure(bData, latitudeCommonData, longitudeCommonData, visibleAxesBoth,
+ ("Areas of trouble data in\n" + variableDisplayName),
+ spaciallyInvalidMaskBoth | invalidDataMaskB,
+ mediumGrayColorMap, troubleMask)
+ LOG.info("\t\tsaving image marking trouble data")
+ figureBadDataInDiff.savefig(outputPath + "/" + variableName + ".Trouble.png", dpi=200)
+
+ # a histogram of the values of fileA - file B so that the distribution of error is visible (hopefully)
+ LOG.info("\t\tcreating histogram of the amount of difference")
+ numBinsToUse = 50
+ valuesForHist = rawDiffData[~everyThingWrongButEpsilon]
+ diffHistogramFigure = _create_histogram(valuesForHist, numBinsToUse,
+ ("Difference in\n" + variableDisplayName),
+ ('Value of (Data File B - Data File A) at a Data Point'),
+ ('Number of Data Points with a Given Difference'),
+ True)
+ LOG.info("\t\tsaving histogram of the amount of difference")
+ diffHistogramFigure.savefig(outputPath + "/" + variableName + ".Hist.png", dpi=200)
+
+ '''
+ # a histogram of the values of fileA - file B, excluding epsilon mismatched values (to show errors more clearly)
+ LOG.info("\t\tcreating histogram of the amount of difference for imperfect matches")
+ numBinsToUse = 50
+ valuesForHist = rawDiffData[outsideEpsilonMask] # select only the imperfectly matched points
+ imperfectHistogramFigure = None
+ if (valuesForHist.size > 0) :
+ imperfectHistogramFigure = _create_histogram(valuesForHist, numBinsToUse,
+ ("Difference in " + variableDisplayName + "\nExcluding Epsilon Matches"),
+ ('Value of (Data File B - Data File A) at a Data Point'),
+ ('Number of Data Points with a Given Difference'),
+ True)
+ LOG.info("\t\tsaving histogram of the amount of difference for imperfect matches")
+ imperfectHistogramFigure.savefig(outputPath + "/" + variableName + ".ImpHist.png", dpi=200)
+ '''
+
+ # scatter plot of file a and b comparison
+ LOG.info("\t\tcreating scatter plot of file a values vs file b values")
+ diffScatterPlot = _create_scatter_plot(aData[~everyThingWrongButEpsilon].ravel(), bData[~everyThingWrongButEpsilon].ravel(),
+ "Value in File A vs Value in File B", "File A Value", "File B Value",
+ outsideEpsilonMask[~everyThingWrongButEpsilon].ravel(), variableRunInfo['epsilon'])
+ LOG.info("\t\tsaving scatter plot of file a values vs file b values")
+ diffScatterPlot.savefig(outputPath + "/" + variableName + ".Scatter.png", dpi=200)
# also save smaller versions of the figures if the parameter says the caller wants us to
if (makeSmall) :
- print("\t\tsaving smaller versions of images")
+ LOG.info("\t\tsaving smaller versions of images")
figureA.savefig(outputPath + "/" + variableName + ".A.small.png", dpi=50)
figureB.savefig(outputPath + "/" + variableName + ".B.small.png", dpi=50)
if not shortCircuitComparisons :
@@ -482,6 +499,8 @@ def plot_and_save_figure_comparison(aData, bData,
figureDiff.savefig(outputPath + "/" + variableName + ".Diff.small.png", dpi=50)
figureBadDataInDiff.savefig(outputPath + "/" + variableName + ".Trouble.small.png", dpi=50)
diffHistogramFigure.savefig(outputPath + "/" + variableName + ".Hist.small.png", dpi=50)
+ if not (imperfectHistogramFigure is None) :
+ imperfectHistogramFigure.savefig(outputPath + "/" + variableName + ".ImpHist.small.png", dpi=50)
diffScatterPlot.savefig(outputPath + "/" + variableName + ".Scatter.small.png", dpi=50)
return
diff --git a/pyglance/glance/variablereport.txt b/pyglance/glance/variablereport.txt
index a0085c5..5c1dd34 100644
--- a/pyglance/glance/variablereport.txt
+++ b/pyglance/glance/variablereport.txt
@@ -16,7 +16,7 @@ Copyright (c) 2009 University of Wisconsin SSEC. All rights reserved.
variableDisplayName = variableName
if (runInfo.has_key('display_name')):
variableDisplayName = runInfo['display_name']
- hideComparisonImages = ('short_circuit_diffs' in runInfo) and runInfo['short_circuit_diffs']
+ #hideComparisonImages = ('short_circuit_diffs' in runInfo) and runInfo['short_circuit_diffs']
%>
<title>${variableDisplayName} Variable Comparison</title>
@@ -135,12 +135,13 @@ Copyright (c) 2009 University of Wisconsin SSEC. All rights reserved.
% endfor
</dl>
- % if shouldIncludeImages and (not hideComparisonImages) :
+ % if shouldIncludeImages : ## and (not hideComparisonImages) :
<p>
<a href="./${variableName}.AbsDiff.png"><img src="./${variableName}.AbsDiff.small.png"></a>
<a href="./${variableName}.Diff.png"><img src="./${variableName}.Diff.small.png"></a>
<a href="./${variableName}.Trouble.png"><img src="./${variableName}.Trouble.small.png"></a>
<a href="./${variableName}.Hist.png"><img src="./${variableName}.Hist.small.png"></a>
+ ##<a href="./${variableName}.ImpHist.png"><img src="./${variableName}.ImpHist.small.png"></a>
<a href="./${variableName}.Scatter.png"><img src="./${variableName}.Scatter.small.png"></a>
</p>
% endif
--
GitLab