From 3a87d3cd26e54d37f5bc33c71189819095e39fcc Mon Sep 17 00:00:00 2001
From: "(no author)" <(no author)@8a9318a1-56ba-4d59-b755-99d26321be01>
Date: Thu, 6 Aug 2009 22:19:23 +0000
Subject: [PATCH] changed version number; refactored delta.diff to correct
ignore mask interaction bug and also returned more comprehensive data to
simplify other code; added more stats for min/max values from both data sets;
documented investigation of connic projection plotting issues;
_create_mapped_figure now finds it's own bounding axes; fix data existance
mismatch bug on trouble plot; added filtering functions for lon/lat and var
data; added range display control for plots; cut back on process
prolifferation to (hopefully) limit how often I cause systems to thrash
wildly with large data sets; also smaller bug fixes
git-svn-id: https://svn.ssec.wisc.edu/repos/glance/trunk@47 8a9318a1-56ba-4d59-b755-99d26321be01
---
pyglance/glance/compare.py | 100 +++++++++++++++-------
pyglance/glance/delta.py | 147 ++++++++++++++++++--------------
pyglance/glance/exconfig.py | 44 +++++++++-
pyglance/glance/plot.py | 163 ++++++++++++++++++++----------------
pyglance/setup.py | 2 +-
5 files changed, 290 insertions(+), 166 deletions(-)
diff --git a/pyglance/glance/compare.py b/pyglance/glance/compare.py
index 43bd32d..9c10019 100644
--- a/pyglance/glance/compare.py
+++ b/pyglance/glance/compare.py
@@ -22,8 +22,13 @@ import glance.report as report
LOG = logging.getLogger(__name__)
-glance_default_longitude_name = 'pixel_longitude'
-glance_default_latitude_name = 'pixel_latitude'
+glance_lon_lat_defaults = {'longitude': 'pixel_longitude',
+ 'latitude': 'pixel_latitude',
+ 'data_filter_function_lon_in_a': None,
+ 'data_filter_function_lat_in_a': None,
+ 'data_filter_function_lon_in_b': None,
+ 'data_filter_function_lat_in_b': None
+ }
# these are the built in default settings
glance_analysis_defaults = {'epsilon': 0.0,
@@ -208,8 +213,7 @@ def _load_config_or_options(optionsSet, originalArgs) :
runInfo = {}
runInfo['shouldIncludeReport'] = True
runInfo['shouldIncludeImages'] = False
- runInfo['latitude'] = glance_default_latitude_name
- runInfo['longitude'] = glance_default_longitude_name
+ runInfo.update(glance_lon_lat_defaults) # get the default lon/lat info
runInfo['lon_lat_epsilon'] = 0.0
runInfo['version'] = _get_glance_version_string()
@@ -284,7 +288,9 @@ def _load_config_or_options(optionsSet, originalArgs) :
return paths, runInfo, defaultsToUse, requestedNames, usedConfigFile
-def _get_and_analyze_lon_lat (fileObject, latitudeVariableName, longitudeVariableName) :
+def _get_and_analyze_lon_lat (fileObject,
+ latitudeVariableName, longitudeVariableName,
+ latitudeDataFilterFn=None, longitudeDataFilterFn=None) :
"""
get the longitude and latitude data from the given file, assuming they are in the given variable names
and analyze them to identify spacially invalid data (ie. data that would fall off the earth)
@@ -293,6 +299,14 @@ def _get_and_analyze_lon_lat (fileObject, latitudeVariableName, longitudeVariabl
longitudeData = array(fileObject[longitudeVariableName], dtype=float)
latitudeData = array(fileObject[latitudeVariableName], dtype=float)
+ # if we have filters, use them
+ if not (latitudeDataFilterFn is None) :
+ latitudeData = latitudeDataFilterFn(latitudeData)
+ LOG.debug ('latitude size after application of filter: ' + str(latitudeData.shape))
+ if not (longitudeDataFilterFn is None) :
+ longitudeData = longitudeDataFilterFn(longitudeData)
+ LOG.debug ('longitude size after application of filter: ' + str(longitudeData.shape))
+
# build a mask of our spacially invalid data
invalidLatitude = (latitudeData < -90) | (latitudeData > 90)
invalidLongitude = (longitudeData < -180) | (longitudeData > 360)
@@ -309,9 +323,11 @@ def _get_percentage_from_mask(dataMask) :
given a mask that marks the elements we want the percentage of as True (and is the size of our original data),
figure out what percentage of the whole they are
"""
- numMarkedDataPts = len(dataMask[dataMask].ravel())
- dataShape = dataMask.shape
- totalDataPts = dataShape[0] * dataShape[1]
+ numMarkedDataPts = sum(dataMask)
+ totalDataPts = dataMask.size
+ # avoid dividing by 0
+ if totalDataPts is 0 :
+ return 0.0, 0
percentage = 100.0 * float(numMarkedDataPts) / float(totalDataPts)
return percentage, numMarkedDataPts
@@ -331,17 +347,27 @@ def _check_lon_lat_equality(longitudeA, latitudeA,
lon_lat_not_equal_points_count = 0
lon_lat_not_equal_points_percent = 0.0
- combinedIgnoreMask = ignoreMaskA | ignoreMaskB
# get information about how the latitude and longitude differ
+ longitudeDiff, finiteLongitudeMask, _, _, lon_not_equal_mask, _, _, _ = delta.diff(longitudeA, longitudeB,
+ llepsilon,
+ (None, None),
+ (ignoreMaskA, ignoreMaskB))
+ latitudeDiff, finiteLatitudeMask, _, _, lat_not_equal_mask, _, _, _ = delta.diff(latitudeA, latitudeB,
+ llepsilon,
+ (None, None),
+ (ignoreMaskA, ignoreMaskB))
+ '''
longitudeDiff, finiteLongitudeMask, _, _, _, lon_not_equal_mask = delta.diff(longitudeA, longitudeB,
llepsilon,
ignoreMask=combinedIgnoreMask)
latitudeDiff, finiteLatitudeMask, _, _, _, lat_not_equal_mask = delta.diff(latitudeA, latitudeB,
llepsilon,
ignoreMask=combinedIgnoreMask)
+ '''
+
lon_lat_not_equal_mask = lon_not_equal_mask | lat_not_equal_mask
- lon_lat_not_equal_points_count = sum(lon_lat_not_equal_mask.ravel())
+ lon_lat_not_equal_points_count = sum(lon_lat_not_equal_mask)
lon_lat_not_equal_points_percent = (float(lon_lat_not_equal_points_count) / float(lon_lat_not_equal_mask.size)) * 100.0
# if we have unequal points, create user legible info about the problem
@@ -682,7 +708,6 @@ python -m glance
print name
print
lal = list(delta.summarize(aval,bval,epsilon,(amiss,bmiss)).items())
- # lal = list(delta.stats(*delta.diff(aval,bval,epsilon,(amiss,bmiss))).items())
lal.sort()
for dictionary_title, dict_data in lal:
print '%s' % dictionary_title
@@ -802,9 +827,11 @@ python -m glance
if ( 'latitude_alt_name_in_b' in runInfo):
b_latitude = runInfo['latitude_alt_name_in_b']
longitudeA, latitudeA, spaciallyInvalidMaskA, spatialInfo['file A'] = \
- _get_and_analyze_lon_lat (aFile, runInfo['latitude'], runInfo['longitude'])
+ _get_and_analyze_lon_lat (aFile, runInfo['latitude'], runInfo['longitude'],
+ runInfo['data_filter_function_lat_in_a'], runInfo['data_filter_function_lon_in_a'])
longitudeB, latitudeB, spaciallyInvalidMaskB, spatialInfo['file B'] = \
- _get_and_analyze_lon_lat (bFile, b_latitude, b_longitude)
+ _get_and_analyze_lon_lat (bFile, b_latitude, b_longitude,
+ runInfo['data_filter_function_lat_in_b'], runInfo['data_filter_function_lon_in_b'])
# test the "valid" values in our lon/lat
moreSpatialInfo = _check_lon_lat_equality(longitudeA, latitudeA, longitudeB, latitudeB,
@@ -832,7 +859,7 @@ python -m glance
_compare_spatial_invalidity(spaciallyInvalidMaskA, spaciallyInvalidMaskB, spatialInfo,
longitudeA, longitudeB, latitudeA, latitudeB,
runInfo['shouldIncludeImages'], outputPath)
-
+
# set some things up to hold info for our reports
# this will hold our variable report information in the form
@@ -868,6 +895,14 @@ python -m glance
aData = aFile[varRunInfo['variable_name']]
bData = bFile[b_variable]
+ # apply any data filter functions we may have
+ if ('data_filter_function_a' in varRunInfo) :
+ aData = varRunInfo['data_filter_function_a'](aData)
+ LOG.debug ("filter function was applied to file A data for variable: " + explanationName)
+ if ('data_filter_function_b' in varRunInfo) :
+ bData = varRunInfo['data_filter_function_b'](bData)
+ LOG.debug ("filter function was applied to file B data for variable: " + explanationName)
+
# check if this data can be displayed
if ((aData.shape == bData.shape) and
(aData.shape == longitudeCommon.shape) and
@@ -908,29 +943,32 @@ python -m glance
# loop to create the images for all our variables
if (runInfo['shouldIncludeImages']) :
for name in variableAnalysisInfo :
+ """ TODO for the moment, I think this is resulting in too many processes, so only generate figs for one var at a time
# create a child to handle this variable's images
pid = os.fork()
isParent = not (pid is 0)
+ # only the child needs to make figures, the parent can move on
if (isParent) :
childPids.append(pid)
LOG.debug ("Started child process (pid: " + str(pid) + ") to create reports for variable " + name)
+
else :
- # create the images comparing that variable
- print("\tcreating figures for: " + variableAnalysisInfo[name]['exp_name'])
- plot.plot_and_save_figure_comparison(variableAnalysisInfo[name]['data']['A'],
- variableAnalysisInfo[name]['data']['B'],
- variableAnalysisInfo[name]['run_info'],
- files['file A']['path'],
- files['file B']['path'],
- latitudeA, longitudeA,
- latitudeB, longitudeB,
- latitudeCommon, longitudeCommon,
- spaciallyInvalidMaskA,
- spaciallyInvalidMaskB,
- spaciallyInvalidMask,
- outputPath, True)
- print("\tfinished creating figures for: " + variableAnalysisInfo[name]['exp_name'])
- sys.exit(0) # this child has successfully finished it's tasks
+ """
+ # create the images comparing that variable
+ print("\tcreating figures for: " + variableAnalysisInfo[name]['exp_name'])
+ plot.plot_and_save_figure_comparison(variableAnalysisInfo[name]['data']['A'],
+ variableAnalysisInfo[name]['data']['B'],
+ variableAnalysisInfo[name]['run_info'],
+ files['file A']['path'],
+ files['file B']['path'],
+ latitudeA, longitudeA,
+ latitudeB, longitudeB,
+ latitudeCommon, longitudeCommon,
+ spaciallyInvalidMaskA,
+ spaciallyInvalidMaskB,
+ outputPath, True)
+ print("\tfinished creating figures for: " + variableAnalysisInfo[name]['exp_name'])
+ #sys.exit(0) # this child has successfully finished it's tasks
# reports are fast, so the parent thread will just do this
# generate our general report pages once we've looked at all the variables
@@ -974,7 +1012,7 @@ python -m glance
# if we're the parent, wait for any children to catch up
if isParent:
if len(childPids) > 0 :
- print ("waiting for completion of report and/or figure generation...")
+ print ("waiting for completion of figure generation...")
for pid in childPids:
os.waitpid(pid, 0)
diff --git a/pyglance/glance/delta.py b/pyglance/glance/delta.py
index 29f8de6..65ccf37 100644
--- a/pyglance/glance/delta.py
+++ b/pyglance/glance/delta.py
@@ -21,7 +21,9 @@ def _missing(x,missing_value=None):
return isnan(x) | (x==missing_value)
return isnan(x)
-def diff(a, b, epsilon=0., (amissing,bmissing)=(None,None), ignoreMask=None):
+def diff(aData, bData, epsilon=0.,
+ (a_missing_value, b_missing_value)=(None, None),
+ (ignore_mask_a, ignore_mask_b)=(None, None)):
"""
take two arrays of similar size and composition
if an ignoreMask is passed in values in the mask will not be analysed to
@@ -33,36 +35,44 @@ def diff(a, b, epsilon=0., (amissing,bmissing)=(None,None), ignoreMask=None):
(a-notfinite-mask, b-notfinite-mask)
(a-missing-mask, b-missing-mask)
"""
- shape = a.shape
- assert(b.shape==shape)
- assert(a.dtype==b.dtype)
+ shape = aData.shape
+ assert(bData.shape==shape)
+ assert(aData.dtype==bData.dtype)
- # if the ignore mask does not exist, set it to include none of the data
- if (ignoreMask is None) :
- ignoreMask = zeros(shape,dtype=bool)
+ # if the ignore masks do not exist, set them to include none of the data
+ if (ignore_mask_a is None) :
+ ignore_mask_a = zeros(shape,dtype=bool)
+ if (ignore_mask_b is None) :
+ ignore_mask_b = zeros(shape,dtype=bool)
# deal with the basic masks
- anfin, bnfin = ~isfinite(a) & ~ignoreMask, ~isfinite(b) & ~ignoreMask
- amis, bmis = zeros(shape,dtype=bool), zeros(shape,dtype=bool)
- if amissing is not None:
- amis[a==amissing] = True
- amis[ignoreMask] = False # don't analyse the ignored values
- if bmissing is not None:
- bmis[b==bmissing] = True
- bmis[ignoreMask] = False # don't analyse the ignored values
+ a_not_finite_mask, b_not_finite_mask = ~isfinite(aData) & ~ignore_mask_a, ~isfinite(bData) & ~ignore_mask_b
+ a_missing_mask, b_missing_mask = zeros(shape,dtype=bool), zeros(shape,dtype=bool)
+ # if we were given missing values, mark where they are in the data
+ if a_missing_value is not None:
+ a_missing_mask[aData == a_missing_value] = True
+ a_missing_mask[ignore_mask_a] = False # don't analyse the ignored values
+ if b_missing_value is not None:
+ b_missing_mask[bData == b_missing_value] = True
+ b_missing_mask[ignore_mask_b] = False # don't analyse the ignored values
# build the comparison data that includes the "good" values
- d = empty_like(a)
- mask = ~(anfin | bnfin | amis | bmis | ignoreMask) # mask to get just the "valid" data
- d[~mask] = nan # throw away invalid data
- d[mask] = b[mask] - a[mask]
+ valid_in_a_mask = ~(a_not_finite_mask | a_missing_mask | ignore_mask_a)
+ valid_in_b_mask = ~(b_not_finite_mask | b_missing_mask | ignore_mask_b)
+ valid_in_both = valid_in_a_mask & valid_in_b_mask
- # the valid data that's outside epsilon
- outeps = (abs(d) > epsilon) & mask
- # trouble areas - mismatched nans, mismatched missing-values, differences > epsilon
- trouble = (anfin ^ bnfin) | (amis ^ bmis) | outeps
+ # construct our diff'ed array
+ raw_diff = empty_like(aData)
+ raw_diff[~valid_in_both] = nan # throw away invalid data
+ raw_diff[valid_in_both] = bData[valid_in_both] - aData[valid_in_both]
- return d, mask, trouble, (anfin, bnfin), (amis, bmis), outeps
+ # the valid data which is too different between the two sets according to the given epsilon
+ outside_epsilon_mask = (abs(raw_diff) > epsilon) & valid_in_both
+ # trouble points - mismatched nans, mismatched missing-values, differences that are too large
+ trouble_pt_mask = (a_not_finite_mask ^ b_not_finite_mask) | (a_missing_mask ^ b_missing_mask) | outside_epsilon_mask
+
+ return raw_diff, valid_in_both, (valid_in_a_mask, valid_in_b_mask), trouble_pt_mask, outside_epsilon_mask, \
+ (a_not_finite_mask, b_not_finite_mask), (a_missing_mask, b_missing_mask), (ignore_mask_a, ignore_mask_b)
def corr(x,y,mask):
"compute correlation coefficient"
@@ -84,7 +94,8 @@ def rms_corr_withnoise(truth, actual, noiz, epsilon=0., (amissing,bmissing)=(Non
x=truth
y=actual
z=noiz
- d,good,bad,_,_,_ = diff(x,y,epsilon,(amissing,bmissing))
+ d, good, _, bad, _, _, _, _ = diff(x,y,epsilon,(amissing,bmissing))
+ #d,good,bad,_,_,_ = diff(x,y,epsilon,(amissing,bmissing))
# compute RMS error
rmse = sqrt(sum(d[good]**2)) / d.size
gf = good.flatten()
@@ -100,7 +111,8 @@ def rms_corr_withnoise(truth, actual, noiz, epsilon=0., (amissing,bmissing)=(Non
xpn[good] += z[good]
xpnf = xpn.flatten()[gf]
# compute RMS error versus noise
- dpn,good,bad,_,_,_ = diff(xpn,y,epsilon,(amissing,bmissing))
+ dpn, good, _, bad, _, _, _, _ = diff(xpn,y,epsilon,(amissing,bmissing))
+ #dpn,good,bad,_,_,_ = diff(xpn,y,epsilon,(amissing,bmissing))
rmsepn = sqrt(sum(dpn[good]**2)) / d.size
assert(sum(~isfinite(xpnf))==0)
rpn = compute_r(xpnf,yf)[0]
@@ -129,20 +141,18 @@ def _get_num_perfect(a, b, ignoreMask=None):
numPerfect = sum(a == b)
return numPerfect
-def _get_nan_stats(a, b) :
+def _get_nan_stats(a_nan_mask, b_nan_mask) :
"""
Get a list of statistics about non-numerical values in data sets a and b,
the return value will be a dictionary of statistics
"""
# find the nan values in the data
- a_nans = ~isfinite(a)
- num_a_nans = sum(a_nans)
- b_nans = ~isfinite(b)
- num_b_nans = sum(b_nans)
- num_common_nans = sum(a_nans & b_nans)
+ num_a_nans = sum(a_nan_mask)
+ num_b_nans = sum(b_nan_mask)
+ num_common_nans = sum(a_nan_mask & b_nan_mask)
# make the assumption that a and b are the same size and only use the size of a
- total_num_values = a.size
+ total_num_values = a_nan_mask.size
nan_stats = {'a_nan_count': num_a_nans,
'a_nan_fraction': (float(num_a_nans) / float(total_num_values)),
@@ -207,7 +217,12 @@ def _get_finite_data_stats(a_is_finite_mask, b_is_finite_mask, common_ignore_mas
return finite_value_stats
-def _get_general_data_stats(a_missing_value, b_missing_value, epsilon, trouble_mask, ignore_in_a_mask, ignore_in_b_mask) :
+def _get_general_data_stats(a, b,
+ a_missing_value, b_missing_value,
+ epsilon, trouble_mask,
+ spatial_ignore_in_a_mask, spatial_ignore_in_b_mask,
+ bad_in_a, bad_in_b
+ ) :
"""
Get a list of general statistics about a and b, given a and b and some other information
about them.
@@ -215,8 +230,8 @@ def _get_general_data_stats(a_missing_value, b_missing_value, epsilon, trouble_m
"""
# figure out how much trouble we had
num_trouble = sum(trouble_mask)
- num_ignored_in_a = sum(ignore_in_a_mask)
- num_ignored_in_b = sum(ignore_in_b_mask)
+ num_ignored_in_a = sum(spatial_ignore_in_a_mask)
+ num_ignored_in_b = sum(spatial_ignore_in_b_mask)
# make the assumption that a and b are the same size/shape as their trouble mask
total_num_values = trouble_mask.size
@@ -224,12 +239,16 @@ def _get_general_data_stats(a_missing_value, b_missing_value, epsilon, trouble_m
general_stats = {'a_missing_value': a_missing_value,
'b_missing_value': b_missing_value,
'epsilon': epsilon,
+ 'max_a': max_with_mask(a, bad_in_a),
+ 'max_b': max_with_mask(b, bad_in_b),
+ 'min_a': min_with_mask(a, bad_in_a),
+ 'min_b': min_with_mask(b, bad_in_b),
'num_data_points': total_num_values,
'shape': trouble_mask.shape,
'spatially_invalid_pts_ignored_in_a': num_ignored_in_a,
'spatially_invalid_pts_ignored_in_b': num_ignored_in_b,
'trouble_points_count': num_trouble,
- 'trouble_points_fraction': num_trouble/ float(total_num_values)
+ 'trouble_points_fraction': float(num_trouble) / float(total_num_values)
}
return general_stats
@@ -259,38 +278,38 @@ def _get_numerical_data_stats(a, b, diff_data, data_is_finite_mask, outside_epsi
return comparison
+# get the min, ignoring the stuff in mask
+def min_with_mask(data, mask) :
+ return data[~mask][data[~mask].argmin()]
+
+# get the max, ignoring the stuff in mask
+def max_with_mask(data, mask) :
+ return data[~mask][data[~mask].argmax()]
+
def summarize(a, b, epsilon=0., (a_missing_value, b_missing_value)=(None,None), ignoreInAMask=None, ignoreInBMask=None):
"""return dictionary of statistics dictionaries
stats not including 'nan' in name exclude nans in either arrays
"""
- #print('a type: ' + str(a.dtype))
- #print('b type: ' + str(b.dtype))
-
- # select/build our ignore masks
- # if the user didn't send us any, don't ignore anything
- if (ignoreInAMask is None) :
- ignoreInAMask = zeros(a.shape, dtype=bool)
- if (ignoreInBMask is None) :
- ignoreInBMask = zeros(b.shape, dtype=bool)
- ignoreMask = ignoreInAMask | ignoreInBMask
-
- d, mask, trouble, (anfin, bnfin), (amis, bmis), outside_epsilon = nfo = diff(a,b,epsilon,(a_missing_value, b_missing_value),ignoreMask)
- # build some other finite data masks that we'll need
- finite_a_mask = ~(anfin | amis)
- finite_b_mask = ~(bnfin | bmis)
- finite_mask = finite_a_mask & finite_b_mask
- # also factor in the ignore masks
- finite_a_mask = finite_a_mask & (~ ignoreInAMask)
- finite_b_mask = finite_b_mask & (~ ignoreInBMask)
- finite_mask = finite_mask & (~ ignoreMask)
+ diffData, finite_mask, (finite_a_mask, finite_b_mask), \
+ trouble, outside_epsilon, (anfin, bnfin), \
+ (amis, bmis), (ignoreInAMask, ignoreInBMask) = nfo = diff(a, b, epsilon,
+ (a_missing_value, b_missing_value),
+ (ignoreInAMask, ignoreInBMask))
+ '''
+ d, valid_mask, trouble, (anfin, bnfin), (amis, bmis), outside_epsilon = nfo = diff(a,b,
+ epsilon,
+ (a_missing_value, b_missing_value),
+ (ignoreInAMask, ignoreInBMask))
+ '''
- general_stats = _get_general_data_stats(a_missing_value, b_missing_value, epsilon, trouble, ignoreInAMask, ignoreInBMask)
- additional_statistics = stats(*nfo) # grab some additional comparison statistics
- comparison_stats = _get_numerical_data_stats(a, b, d, finite_mask, outside_epsilon, additional_statistics)
- nan_stats = _get_nan_stats(a, b)
+ general_stats = _get_general_data_stats(a, b, a_missing_value, b_missing_value, epsilon, trouble,
+ ignoreInAMask, ignoreInBMask, ~finite_a_mask, ~finite_b_mask)
+ additional_statistics = stats(*nfo) # grab some additional comparison statistics
+ comparison_stats = _get_numerical_data_stats(a, b, diffData, finite_mask, outside_epsilon, additional_statistics)
+ nan_stats = _get_nan_stats(anfin, bnfin)
missing_stats = _get_missing_value_stats(amis, bmis)
- finite_stats = _get_finite_data_stats(finite_a_mask, finite_b_mask, ignoreMask)
+ finite_stats = _get_finite_data_stats(finite_a_mask, finite_b_mask, (ignoreInAMask | ignoreInBMask))
out = {}
out['NaN Statistics'] = nan_stats
@@ -308,6 +327,10 @@ STATISTICS_DOC = { 'general': "Finite values are non-missing and finite (not Na
'a_missing_value': 'the value that is considered \"missing\" data when it is found in A',
'b_missing_value': 'the value that is considered \"missing\" data when it is found in B',
'epsilon': 'amount of difference between matching data points in A and B that is considered acceptable',
+ 'max_a': 'the maximum finite, non-missing value found in A',
+ 'max_b': 'the maximum finite, non-missing value found in B',
+ 'min_a': 'the minimum finite, non-missing value found in A',
+ 'min_b': 'the minimum finite, non-missing value found in B',
'num_data_points': "number of data values in A",
'shape': "shape of A",
'spatially_invalid_pts_ignored_in_a': 'number of points with invalid latitude/longitude information in A that were' +
diff --git a/pyglance/glance/exconfig.py b/pyglance/glance/exconfig.py
index eb93cea..122f9bb 100644
--- a/pyglance/glance/exconfig.py
+++ b/pyglance/glance/exconfig.py
@@ -12,17 +12,30 @@ Copyright (c) 2009 University of Wisconsin SSEC. All rights reserved.
"""
# whether or not images should be generated and shown in the report
-shouldIncludeImages = False
+shouldIncludeImages = True
# the names of the latitude and longitude variables that will be used
lat_lon_info = {}
lat_lon_info['longitude'] = 'imager_prof_retr_abi_r4_generic1' # 'pixel_longitude'# the name of the longitude variable
lat_lon_info ['latitude'] = 'imager_prof_retr_abi_r4_generic2' # 'pixel_latitude' # the name of the latitude variable
+
+"""
+# the following two functions can be defined in order to filter the longitude and latitude data, for example, these could be used to compensate
+# for differening data types (like ints/floats or float32/float64) or to handle slicing out only a subset of the data for analysis
+# note: these two filters will only be applied to the longitude and latitude data in file A
+lat_lon_info['data_filter_function_lon_in_a'] = (insert lambda function here)
+lat_lon_info['data_filter_function_lat_in_a'] = (insert lambda function here)
+
# the following two values are optional and only need to be set if the the latitude and longitude have
# different names in file A and file B
-"""
lat_lon_info['longitude_alt_name_in_b'] = 'resampled_longitude' # the alternate name of the longitude in file B
lat_lon_info ['latitude_alt_name_in_b'] = 'resampled_latitude' # the alternate name of the latitude in file A
+
+# the following two functions can be defined in order to filter the longitude and latitude data, for example, these could be used to compensate
+# for differening data types (like ints/floats or float32/float64) or to handle slicing out only a subset of the data for analysis
+# note: these two filters will only be applied to the longitude and latitude data in file B
+lat_lon_info['data_filter_function_lon_in_b'] = (insert lambda function here)
+lat_lon_info['data_filter_function_lon_in_b'] = (insert lambda function here)
"""
# this value can be used to control how similar the longitude and latitude must be to be considered matching
# if all of your longitude and latitude do not match under this epsilon, most of the comparison report will
@@ -87,6 +100,33 @@ setOfVariables['imager_prof_retr_abi_total_totals_index'] = { # this s
# None indicates that this variable should not be tested
# on amount of non-finite data
# note, this setting overrides the default
+
+ 'display_ranges': [13.0, 14.0, 15.0, 20.0, 32.0, 40.0, 54.0, 60.0],
+ # this array of ranges can be defined in order to control
+ # a custom display of color ranges in any graphs produced
+ # for this variable, ranges will fall between the numbers
+ # in the list (inclusive of the beginning and endding numbers)
+ # please list your range in increasing value if you choose
+ # to use this feature
+
+ 'display_range_names': [ 'a', 'b', 'c', 'd', 'e', 'f', 'g'],
+ # this array of labels can be defined in order to control
+ # the display labels associated with your range boundaries
+ # if less labels are given than boundaries, the boundaries
+ # will be labeled starting with the lowest valued boundary
+ # and working upwards until there are no more labels, any
+ # remaining boundaries will be left unlabeled
+ # TODO In the future this array may have the ability to
+ # let you label your ranges (ie, the space between two
+ # boundaries) or your boundaries.
+
+
+ # the following two functions can be defined in order to filter the variable data,
+ # for example, these could be used to compensate
+ # for differening data types (like ints/floats or float32/float64)
+ # or to handle slicing out only a subset of the data for analysis
+# 'data_filter_function_a': (insert lambda function here), # note: will only be applied to file A data
+# 'data_filter_function_b': (insert lambda function here) # note: will only be applied to file B data
}
setOfVariables['imager_prof_retr_abi_total_precipitable_water_high'] = {
'display_name': 'Total Precipitable Water, High',
diff --git a/pyglance/glance/plot.py b/pyglance/glance/plot.py
index 5d66c31..29a3b6d 100644
--- a/pyglance/glance/plot.py
+++ b/pyglance/glance/plot.py
@@ -51,6 +51,7 @@ mediumGrayColorMap = colors.LinearSegmentedColormap('mediumGrayColorMap', medium
# build a scatter plot of the x,y points
def _create_scatter_plot(dataX, dataY, title, xLabel, yLabel, badMask=None, epsilon=None) :
+
# make the figure
figure = plt.figure()
axes = figure.add_subplot(111)
@@ -64,8 +65,14 @@ def _create_scatter_plot(dataX, dataY, title, xLabel, yLabel, badMask=None, epsi
dataX = dataX[~badMask]
dataY = dataY[~badMask]
- # the scatter plot of the good data
+ # the scatter plot of the good data
axes.plot(dataX, dataY, 'b,', label='within\nepsilon')
+ # TODO, if we eventually want to plot the data with some sort of color, I think we will have to use scatter()
+ '''
+ if len(dataX) > 0 and len(dataY) > 0 :
+ diffData = np.abs(dataX - dataY)
+ axes.scatter(dataX, dataY, c=diffData, vmin=diffData.min(), vmax=diffData.max(), label='within\nepsilon')
+ '''
# plot the bad data
numTroublePts = 0
@@ -163,21 +170,31 @@ def _create_histogram(data, bins, title, xLabel, yLabel, displayStats=False) :
return figure
+# make a "clean" copy of the longitude or latitude array passed in by replacing all
+# values which would fall in the provided "invalid" mask with the default bad-lon-lat
+# value, which will keep them from being plotted
+def _clean_lon_or_lat_with_mask(lon_or_lat_data, invalid_data_mask):
+ clean_data = empty_like(lon_or_lat_data)
+ clean_data[~invalid_data_mask] = lon_or_lat_data[~invalid_data_mask]
+ clean_data[ invalid_data_mask] = badLonLat
+
+ return clean_data
+
# create a figure including our data mapped onto a map at the lon/lat given
# the colorMap parameter can be used to control the colors the figure is drawn in
# if any masks are passed in in the tagData list they will be plotted as an overlays
# set on the existing image
-def _create_mapped_figure(data, latitude, longitude, boundingAxes, title,
+def _create_mapped_figure(data, latitude, longitude, title,
invalidMask=None, colorMap=None, tagData=None,
dataRanges=None, dataRangeNames=None) :
- # this is very inefficient, TODO find a better way?
- latitudeCleaned = empty_like(latitude)
- latitudeCleaned[~invalidMask] = latitude[~invalidMask]
- latitudeCleaned[invalidMask] = badLonLat
- longitudeCleaned = empty_like(longitude)
- longitudeCleaned[~invalidMask] = longitude[~invalidMask]
- longitudeCleaned[invalidMask] = badLonLat
+ # make a clean version of our lon/lat
+ latitudeClean = _clean_lon_or_lat_with_mask(latitude, invalidMask)
+ longitudeClean = _clean_lon_or_lat_with_mask(longitude, invalidMask)
+
+ # get the bounding axes
+ boundingAxes = _get_visible_axes (longitudeClean, latitudeClean, invalidMask)
+ LOG.debug("Visible axes for figure \"" + title + "\" are: " + str(boundingAxes))
# build the plot
figure = plt.figure()
@@ -193,8 +210,8 @@ def _create_mapped_figure(data, latitude, longitude, boundingAxes, title,
# todo, the use off the offset here is covering a problem with
# contourf hiding data exactly at the end of the range and should
# be removed if a better solution can be found
- minVal = _min_with_mask(data, invalidMask) - offsetToRange
- maxVal = _max_with_mask(data, invalidMask) + offsetToRange
+ minVal = delta.min_with_mask(data, invalidMask) - offsetToRange
+ maxVal = delta.max_with_mask(data, invalidMask) + offsetToRange
dataRanges = np.linspace(minVal, maxVal, 50)
else: # make sure the user range will not discard data TODO, find a better way to handle this
dataRanges[0] = dataRanges[0] - offsetToRange
@@ -214,11 +231,23 @@ def _create_mapped_figure(data, latitude, longitude, boundingAxes, title,
elif (longitudeRange > 100) or (latitudeRange > 70) :
kwargs['projection'] = 'ortho' # use an orthographic projection to show half the globe
else :
- # TODO figure out why the default is cutting off the field of view, until then, use miller
+ # TODO at the moment the default (lcc) is cutting off the field of view,
+ # I think the same problem will occur with all of the conic projections, because
+ # they all allow you to specify the field of view either as corners or as a width/height in
+ # meters, but they do not take the distortion that a conic projection causes into account.
+ # This means that either the corners or the bottom curve of the data area will be clipped off
+ # the edge of the screen. There is also some sort of persistent bug that causes the basemap
+ # to ignore the requested corners for some data sets and shift west and north, cutting off
+ # a pretty considerable amount of data. I've tried various tactics to control the field of
+ # view and can't find any way to get the basemap to show an acceptable area programatically
+ # that will match arbitrary data sets.
+ # For the moment, I am setting this to use a cylindrical projection rather than a conic.
+ # At some point in the future this should be revisited so that glance will be able to handle
+ # a wider range of projections.
kwargs['projection'] = 'mill'
# draw our data placed on a map
- bMap, x, y = maps.mapshow(longitudeCleaned, latitudeCleaned, data, boundingAxes, **kwargs)
+ bMap, x, y = maps.mapshow(longitudeClean, latitudeClean, data, boundingAxes, **kwargs)
# and some informational stuff
axes.set_title(title)
@@ -228,18 +257,19 @@ def _create_mapped_figure(data, latitude, longitude, boundingAxes, title,
# if there are specific requested labels, add them
if not (dataRangeNames is None) :
yPosition = None
- '''
- # TODO, this strategy doesn't work, find some other way to label the centers of
- # the sections, or build a legend/key instead?
+ # if we have fewer labels than tick marks, assume they wanted ranges labeled
if (len(dataRangeNames) < len(dataRanges)) :
+ # TODO, try to display the range colors in an axis instead of a color bar?
+ '''
yPosition=[]
tickPositions = cbar.ax.get_yticks()
for posNum in range(len(tickPositions) - 1) :
currentPos = tickPositions[posNum]
nextPos = tickPositions[posNum + 1]
yPosition.append(0) #(currentPos + ((nextPos - currentPos) / 2.0))
- '''
- cbar.ax.set_yticklabels(dataRangeNames, y=yPosition)
+ '''
+ else : # we have enough data names to label the tick marks
+ cbar.ax.set_yticklabels(dataRangeNames, y=yPosition)
# if there are "tag" masks, plot them over the existing map
if not (tagData is None) :
@@ -248,15 +278,16 @@ def _create_mapped_figure(data, latitude, longitude, boundingAxes, title,
newX = x[tagData]
newY = y[tagData]
+
# look at how many trouble points we have
- numTroublePoints = newX.shape[0]
+ numTroublePoints = newX.size
hasTrouble = False
neededHighlighting = False
if numTroublePoints > 0 :
hasTrouble = True
# figure out how many bad points there are
- totalNumPoints = x[~invalidMask].shape[0]
+ totalNumPoints = x.size # the number of points
percentBad = (float(numTroublePoints) / float(totalNumPoints)) * 100.0
LOG.debug('\t\tnumber of trouble points: ' + str(numTroublePoints))
LOG.debug('\t\tpercent of trouble points: ' + str(percentBad))
@@ -277,24 +308,16 @@ def _create_mapped_figure(data, latitude, longitude, boundingAxes, title,
# display the number of trouble points on the report if we were passed a set of tag data
# I'm not thrilled with this solution for getting it below the labels drawn by the basemap
# but I don't think there's a better one at the moment given matplotlib's workings
- troublePtString = '\n\n\nShowing ' + str(numTroublePoints) + ' Trouble Points'
+ troublePtString = '\n\nShowing ' + str(numTroublePoints) + ' Trouble Points'
# if our plot is more complex, add clarification
if hasTrouble :
troublePtString = troublePtString + ' in Green'
if neededHighlighting :
troublePtString = troublePtString + '\nwith Purple Circles for Visual Clarity'
plt.xlabel(troublePtString)
-
+
return figure
-# get the min, ignoring the stuff in mask
-def _min_with_mask(data, mask) :
- return data[~mask].ravel()[data[~mask].argmin()]
-
-# get the max, ignoring the stuff in mask
-def _max_with_mask(data, mask) :
- return data[~mask].ravel()[data[~mask].argmax()]
-
# figure out the bounding axes for the display given a set of
# longitude and latitude and possible a mask of invalid values
# that we should ignore in them
@@ -302,10 +325,10 @@ def _get_visible_axes(longitudeData, latitudeData, toIgnoreMask) :
# calculate the bounding range for the display
# this is in the form [longitude min, longitude max, latitude min, latitude max]
- visibleAxes = [_min_with_mask(longitudeData, toIgnoreMask),
- _max_with_mask(longitudeData, toIgnoreMask),
- _min_with_mask(latitudeData, toIgnoreMask),
- _max_with_mask(latitudeData, toIgnoreMask)]
+ visibleAxes = [delta.min_with_mask(longitudeData, toIgnoreMask),
+ delta.max_with_mask(longitudeData, toIgnoreMask),
+ delta.min_with_mask(latitudeData, toIgnoreMask),
+ delta.max_with_mask(latitudeData, toIgnoreMask)]
return visibleAxes
@@ -359,12 +382,10 @@ def plot_and_save_spacial_trouble(longitude, latitude,
on top of a background plot of a's data shown in grayscale, save this plot to the output path given
if makeSmall is passed as true a smaller version of the image will also be saved
"""
- # get bounding axes
- visibleAxes = _get_visible_axes(longitude, latitude, spaciallyInvalidMask)
# make the figure
LOG.info("Creating spatial trouble image")
- spatialTroubleFig = _create_mapped_figure(None, latitude, longitude, visibleAxes, title,
+ spatialTroubleFig = _create_mapped_figure(None, latitude, longitude, title,
spaciallyInvalidMask, None, spacialTroubleMask)
# save the figure
LOG.info("Saving spatial trouble image")
@@ -385,7 +406,6 @@ def plot_and_save_figure_comparison(aData, bData,
latitudeCommonData, longitudeCommonData,
spaciallyInvalidMaskA,
spaciallyInvalidMaskB,
- spaciallyInvalidMaskBoth,
outputPath,
makeSmall=False,
shortCircuitComparisons=False) :
@@ -413,28 +433,25 @@ def plot_and_save_figure_comparison(aData, bData,
if 'display_name' in variableRunInfo :
variableDisplayName = variableRunInfo['display_name']
+ # figure out what missing values we should be using
+ missing_value = variableRunInfo['missing_value']
+ missing_value_b = missing_value
+ if ('missing_value_alt_in_b' in variableRunInfo) :
+ missing_value_b = variableRunInfo['missing_value_alt_in_b']
+
# compare the two data sets to get our difference data and trouble info
+ rawDiffData, goodMask, (goodInAMask, goodInBMask), troubleMask, outsideEpsilonMask, \
+ (aNotFiniteMask, bNotFiniteMask), (aMissingMask, bMissingMask), \
+ (spaciallyInvalidMaskA, spaciallyInvalidMaskB) = delta.diff(aData, bData, variableRunInfo['epsilon'],
+ (missing_value, missing_value_b),
+ (spaciallyInvalidMaskA, spaciallyInvalidMaskB))
+ '''
rawDiffData, goodMask, troubleMask, (aNotFiniteMask, bNotFiniteMask), \
(aMissingMask, bMissingMask), outsideEpsilonMask = delta.diff(aData, bData, variableRunInfo['epsilon'],
- (variableRunInfo['missing_value'],
- variableRunInfo['missing_value']),
+ (missing_value, missing_value_b),
spaciallyInvalidMaskBoth)
- diffData = np.abs(rawDiffData) # we want to show the distance between our two, rather than which one's bigger
-
- # mark where our invalid data is for each of the files (a and b)
- invalidDataMaskA = aMissingMask | aNotFiniteMask
- invalidDataMaskB = bMissingMask | bNotFiniteMask
- # this mask potentially represents data we don't want to try to plot in our diff because it may be malformed
- everyThingWrongButEpsilon = spaciallyInvalidMaskBoth | invalidDataMaskA | invalidDataMaskB
-
- # calculate the bounding range for the display
- # this is in the form [longitude min, longitude max, latitude min, latitude max]
- visibleAxesA = _get_visible_axes (longitudeAData, latitudeAData, spaciallyInvalidMaskA)
- visibleAxesB = _get_visible_axes (longitudeBData, latitudeBData, spaciallyInvalidMaskB)
- visibleAxesBoth = _get_visible_axes (longitudeCommonData, latitudeCommonData, spaciallyInvalidMaskBoth)
- LOG.debug ("visible axes in A: " + str(visibleAxesA))
- LOG.debug ("visible axes in B: " + str(visibleAxesB))
- LOG.debug ("visible axes in Both: " + str(visibleAxesBoth))
+ '''
+ absDiffData = np.abs(rawDiffData) # we also want to show the distance between our two, rather than just which one's bigger/smaller
# some more display info, pull it out for convenience
dataRanges = None
@@ -458,9 +475,9 @@ def plot_and_save_figure_comparison(aData, bData,
childPids.append(pid)
LOG.debug ("Started child process (pid: " + str(pid) + ") to create file a image for " + variableDisplayName)
else :
- figureA = _create_mapped_figure(aData, latitudeAData, longitudeAData, visibleAxesA,
+ figureA = _create_mapped_figure(aData, latitudeAData, longitudeAData,
(variableDisplayName + "\nin File A"),
- invalidMask=(spaciallyInvalidMaskA | invalidDataMaskA),
+ invalidMask=(~goodInAMask),
dataRanges=dataRanges,
dataRangeNames=dataRangeNames)
LOG.info("\t\tsaving image of " + variableDisplayName + " for file a")
@@ -476,9 +493,9 @@ def plot_and_save_figure_comparison(aData, bData,
childPids.append(pid)
LOG.debug ("Started child process (pid: " + str(pid) + ") to create file b image for " + variableDisplayName)
else :
- figureB = _create_mapped_figure(bData, latitudeBData, longitudeBData, visibleAxesB,
+ figureB = _create_mapped_figure(bData, latitudeBData, longitudeBData,
(variableDisplayName + "\nin File B"),
- invalidMask=(spaciallyInvalidMaskB | invalidDataMaskB),
+ invalidMask=(~ goodInBMask),
dataRanges=dataRanges,
dataRangeNames=dataRangeNames)
LOG.info("\t\tsaving image of " + variableDisplayName + " in file b")
@@ -497,9 +514,9 @@ def plot_and_save_figure_comparison(aData, bData,
childPids.append(pid)
LOG.debug ("Started child process (pid: " + str(pid) + ") to create absolute value of difference image for " + variableDisplayName)
else :
- figureAbsDiff = _create_mapped_figure(diffData, latitudeCommonData, longitudeCommonData, visibleAxesBoth,
+ figureAbsDiff = _create_mapped_figure(absDiffData, latitudeCommonData, longitudeCommonData,
("Absolute value of difference in\n" + variableDisplayName),
- invalidMask=(everyThingWrongButEpsilon))
+ invalidMask=(~ goodMask))
LOG.info("\t\tsaving image of the absolute value of difference for " + variableDisplayName)
figureAbsDiff.savefig(outputPath + "/" + variableName + ".AbsDiff.png", dpi=200)
if (makeSmall) :
@@ -513,9 +530,9 @@ def plot_and_save_figure_comparison(aData, bData,
childPids.append(pid)
LOG.debug ("Started child process (pid: " + str(pid) + ") to create difference image for " + variableDisplayName)
else :
- figureDiff = _create_mapped_figure(rawDiffData, latitudeCommonData, longitudeCommonData, visibleAxesBoth,
+ figureDiff = _create_mapped_figure(rawDiffData, latitudeCommonData, longitudeCommonData,
("Value of (Data File B - Data File A) for\n" + variableDisplayName),
- invalidMask=(everyThingWrongButEpsilon))
+ invalidMask=(~ goodMask))
LOG.info("\t\tsaving image of the difference in " + variableDisplayName)
figureDiff.savefig(outputPath + "/" + variableName + ".Diff.png", dpi=200)
if (makeSmall) :
@@ -530,9 +547,15 @@ def plot_and_save_figure_comparison(aData, bData,
childPids.append(pid)
LOG.debug ("Started child process (pid: " + str(pid) + ") to create trouble image for " + variableDisplayName)
else :
- figureBadDataInDiff = _create_mapped_figure(bData, latitudeCommonData, longitudeCommonData, visibleAxesBoth,
+ # this is not an optimal solution, but we need to have at least somewhat valid data at any mismatched points so
+ # that our plot won't be totally destroyed by missing or non-finite data from B
+ bDataCopy = bData[:]
+ tempMask = goodInAMask & (~goodInBMask)
+ bDataCopy[tempMask] = aData[tempMask]
+ # create the figure marked with the trouble points on top of B's data in grayscale
+ figureBadDataInDiff = _create_mapped_figure(bDataCopy, latitudeCommonData, longitudeCommonData,
("Areas of trouble data in\n" + variableDisplayName),
- spaciallyInvalidMaskBoth | invalidDataMaskB,
+ (~(goodInAMask | goodInBMask)),
mediumGrayColorMap, troubleMask,
dataRanges=dataRanges,
dataRangeNames=dataRangeNames)
@@ -551,7 +574,7 @@ def plot_and_save_figure_comparison(aData, bData,
LOG.debug ("Started child process (pid: " + str(pid) + ") to create difference histogram image for " + variableDisplayName)
else :
numBinsToUse = 50
- valuesForHist = rawDiffData[~everyThingWrongButEpsilon]
+ valuesForHist = rawDiffData[goodMask]
diffHistogramFigure = _create_histogram(valuesForHist, numBinsToUse,
("Difference in\n" + variableDisplayName),
('Value of (Data File B - Data File A) at a Data Point'),
@@ -563,7 +586,7 @@ def plot_and_save_figure_comparison(aData, bData,
diffHistogramFigure.savefig(outputPath + "/" + variableName + ".Hist.small.png", dpi=50)
sys.exit(0) # the child is done now
- ''' TODO, is this actually useful in many cases?
+ ''' TODO, is this actually useful?
# a histogram of the values of fileA - file B, excluding epsilon mismatched values (to show errors more clearly)
LOG.info("\t\tcreating histogram of the amount of difference for imperfect matches")
numBinsToUse = 50
@@ -589,9 +612,9 @@ def plot_and_save_figure_comparison(aData, bData,
childPids.append(pid)
LOG.debug ("Started child process (pid: " + str(pid) + ") to create scatter plot image for " + variableDisplayName)
else :
- diffScatterPlot = _create_scatter_plot(aData[~everyThingWrongButEpsilon].ravel(), bData[~everyThingWrongButEpsilon].ravel(),
+ diffScatterPlot = _create_scatter_plot(aData[goodMask], bData[goodMask],
"Value in File A vs Value in File B", "File A Value", "File B Value",
- outsideEpsilonMask[~everyThingWrongButEpsilon].ravel(), variableRunInfo['epsilon'])
+ outsideEpsilonMask[goodMask], variableRunInfo['epsilon'])
LOG.info("\t\tsaving scatter plot of file a values vs file b values in " + variableDisplayName)
diffScatterPlot.savefig(outputPath + "/" + variableName + ".Scatter.png", dpi=200)
if (makeSmall):
diff --git a/pyglance/setup.py b/pyglance/setup.py
index b4d18f2..a08caf7 100644
--- a/pyglance/setup.py
+++ b/pyglance/setup.py
@@ -18,7 +18,7 @@ easy_install -d $HOME/Library/Python -vi http://larch.ssec.wisc.edu/eggs/repos g
from setuptools import setup, find_packages
setup( name="glance",
- version="0.2.6.5",
+ version="0.2.6.6",
zip_safe = True,
entry_points = { 'console_scripts': [ 'glance = glance.compare:main' ] },
packages = find_packages('.'),
--
GitLab