diff --git a/pyglance/glance/compare.py b/pyglance/glance/compare.py
index f323b27f682299a2651c4d4f17f318fb242a6d7f..285bdf8ba87dc4dc0cad4a9dbcffcbb9e004a324 100644
--- a/pyglance/glance/compare.py
+++ b/pyglance/glance/compare.py
@@ -30,7 +30,8 @@ glance_setting_defaults = {'shouldIncludeReport': True,
'shouldIncludeImages': False,
'doFork': False,
'useThreadsToControlMemory': False,
- 'useSharedRangeForOriginal': False}
+ 'useSharedRangeForOriginal': False,
+ 'noLonLatVars': False}
# these are the built in longitude/latitude defaults
glance_lon_lat_defaults = {'longitude': 'pixel_longitude',
@@ -261,10 +262,11 @@ def _load_config_or_options(aPath, bPath, optionsSet, requestedVars = [ ]) :
# basic defaults for stuff we will need to return
runInfo = {}
runInfo.update(glance_setting_defaults) # get the default settings
- runInfo.update(glance_lon_lat_defaults) # get the default lon/lat info
+ if ('noLonLatVars' not in optionsSet) or (not optionsSet['noLonLatVars']):
+ runInfo.update(glance_lon_lat_defaults) # get the default lon/lat info
# by default, we don't have any particular variables to analyze
- desiredVariables = {}
+ desiredVariables = { }
# use the built in default values, to start with
defaultsToUse = glance_analysis_defaults.copy()
@@ -304,10 +306,12 @@ def _load_config_or_options(aPath, bPath, optionsSet, requestedVars = [ ]) :
# (at least not at the moment, it could be added later and if they did happen to put it in the
# config file, it would override this line)
runInfo['shouldIncludeReport'] = not optionsSet['imagesOnly']
+ runInfo['noLonLatVars'] = optionsSet['noLonLatVars']
# get everything from the config file
runInfo.update(glanceRunConfig.settings)
- runInfo.update(glanceRunConfig.lat_lon_info) # get info on the lat/lon variables
+ if ('noLonLatVars' not in runInfo) or (not runInfo['noLonLatVars']) :
+ runInfo.update(glanceRunConfig.lat_lon_info) # get info on the lat/lon variables
# get any requested names
requestedNames = glanceRunConfig.setOfVariables.copy()
@@ -315,7 +319,7 @@ def _load_config_or_options(aPath, bPath, optionsSet, requestedVars = [ ]) :
defaultsToUse.update(glanceRunConfig.defaultValues)
usedConfigFile = True
-
+
# if we didn't get the info from the config file for some reason
# (the user didn't want to, we couldn't, etc...) get it from the command line options
if not usedConfigFile:
@@ -326,9 +330,13 @@ def _load_config_or_options(aPath, bPath, optionsSet, requestedVars = [ ]) :
runInfo['shouldIncludeReport'] = not optionsSet['imagesOnly']
runInfo['shouldIncludeImages'] = not optionsSet['htmlOnly']
runInfo['doFork'] = optionsSet['doFork']
- runInfo['latitude'] = optionsSet['latitudeVar'] or runInfo['latitude']
- runInfo['longitude'] = optionsSet['longitudeVar'] or runInfo['longitude']
- runInfo['lon_lat_epsilon'] = optionsSet['lonlatepsilon']
+
+ # only record these if we are using lon/lat
+ runInfo['noLonLatVars'] = optionsSet['noLonLatVars']
+ if not runInfo['noLonLatVars'] :
+ runInfo['latitude'] = optionsSet['latitudeVar'] or runInfo['latitude']
+ runInfo['longitude'] = optionsSet['longitudeVar'] or runInfo['longitude']
+ runInfo['lon_lat_epsilon'] = optionsSet['lonlatepsilon']
# get any requested names from the command line
requestedNames = requestedVars or ['.*']
@@ -337,13 +345,7 @@ def _load_config_or_options(aPath, bPath, optionsSet, requestedVars = [ ]) :
defaultsToUse['epsilon'] = optionsSet['epsilon']
defaultsToUse['missing_value'] = optionsSet['missing']
- # note: there is no way to set the tolerances from the command line
-
- # if we can't use a longitude / latitude
- # we also don't want to make images!
- # TODO, make this actually control the comparison logic and report
- if ("shouldIgnoreLonLat" in runInfo) and (runInfo["shouldIgnoreLonLat"]) :
- runInfo["shouldIncludeImages"] = False
+ # note: there is no way to set the tolerances from the command line
return paths, runInfo, defaultsToUse, requestedNames, usedConfigFile
@@ -354,7 +356,7 @@ def _get_and_analyze_lon_lat (fileObject,
get the longitude and latitude data from the given file, assuming they are in the given variable names
and analyze them to identify spacially invalid data (ie. data that would fall off the earth)
"""
- # get the data from the file
+ # get the data from the file TODO, handle these exits out in the calling method?
LOG.info ('longitude name: ' + longitudeVariableName)
try :
longitudeData = array(fileObject[longitudeVariableName], dtype=float)
@@ -390,8 +392,10 @@ def _get_and_analyze_lon_lat (fileObject,
# analyze our spacially invalid data
percentageOfSpaciallyInvalidPts, numberOfSpaciallyInvalidPts = _get_percentage_from_mask(spaciallyInvalidMask)
- return longitudeData, latitudeData, spaciallyInvalidMask, {'totNumInvPts': numberOfSpaciallyInvalidPts,
- 'perInvPts': percentageOfSpaciallyInvalidPts}
+ return longitudeData, latitudeData, spaciallyInvalidMask, {
+ 'totNumInvPts': numberOfSpaciallyInvalidPts,
+ 'perInvPts': percentageOfSpaciallyInvalidPts
+ }
def _get_percentage_from_mask(dataMask) :
"""
@@ -536,12 +540,18 @@ def _handle_lon_lat_info (lon_lat_settings, a_file_object, b_file_object, output
Note: if the error message is returned as anything but None, something uncrecoverable
occured while trying to get the lon/lat info. TODO, replace this with a proper thrown exception
"""
-
# a place to save some general stats about our lon/lat data
- spatialInfo = {}
+ spatialInfo = { }
# a place to put possible error messages TODO remove this in favor of an exception
error_msg = None
+ # if there is no lon/lat specified, stop now
+ if ('longitude' not in lon_lat_settings) or ('latitude' not in lon_lat_settings) :
+ return { }, spatialInfo, error_msg
+
+ # if we should not be comparing against the logitude and latitude, stop now
+ print ('lon_lat_settings: ' + str(lon_lat_settings))
+
# figure out the names to be used for the longitude and latitude variables
a_longitude_name = lon_lat_settings['longitude']
a_latitude_name = lon_lat_settings['latitude']
@@ -588,7 +598,7 @@ def _handle_lon_lat_info (lon_lat_settings, a_file_object, b_file_object, output
+ str(lon_lat_settings['latitude']) + str(latitude_a.shape) + " in file A and variables "
+ str(b_longitude_name) + str(longitude_b.shape) + "/"
+ str(b_latitude_name) + str(latitude_b.shape) + " in file B. Aborting attempt to compare files.")
- return None, None, None, None, error_msg # things have gone wrong
+ return { }, { }, error_msg # things have gone wrong
# update our existing spatial information
spatialInfo.update(moreSpatialInfo)
@@ -598,9 +608,9 @@ def _handle_lon_lat_info (lon_lat_settings, a_file_object, b_file_object, output
longitude_a, longitude_b, latitude_a, latitude_b,
should_make_images, output_path)
- return {"lon": longitude_a, "lat": latitude_a, "inv_mask": spaciallyInvalidMaskA}, \
- {"lon": longitude_b, "lat": latitude_b, "inv_mask": spaciallyInvalidMaskB}, \
- {"lon": longitude_common, "lat": latitude_common, "inv_mask": spaciallyInvalidMask}, \
+ return {'a': {"lon": longitude_a, "lat": latitude_a, "inv_mask": spaciallyInvalidMaskA},
+ 'b': {"lon": longitude_b, "lat": latitude_b, "inv_mask": spaciallyInvalidMaskB},
+ 'common': {"lon": longitude_common, "lat": latitude_common, "inv_mask": spaciallyInvalidMask} }, \
spatialInfo, error_msg
def _open_and_process_files (args, numFilesExpected):
@@ -787,19 +797,22 @@ def reportGen_library_call (a_path, b_path, var_list=[ ],
print("output dir: " + str(pathsTemp['out']))
- # return for lon_lat_data variables will be in the form
+ # return for lon_lat_data variables will be in the form
#{"lon": longitude_data, "lat": latitude_data, "inv_mask": spaciallyInvalidMaskData}
- a_lon_lat_data, b_lon_lat_data, common_lon_lat_data, \
- spatialInfo, fatalErrorMsg = _handle_lon_lat_info (runInfo, #runInfo['lon_lat'],
- aFile, bFile,
- pathsTemp['out'],
- should_make_images = runInfo["shouldIncludeImages"]
- )
-
+ # or { } if there is no lon/lat info
+ lon_lat_data, spatialInfo, fatalErrorMsg = _handle_lon_lat_info (runInfo,
+ aFile, bFile,
+ pathsTemp['out'],
+ should_make_images = runInfo["shouldIncludeImages"])
if fatalErrorMsg is not None :
LOG.warn(fatalErrorMsg)
sys.exit(1)
+ # if there is an approved lon/lat shape, hang on to that for future checks
+ good_shape_from_lon_lat = None
+ if len(lon_lat_data.keys()) > 0:
+ good_shape_from_lon_lat = lon_lat_data['common']['lon'].shape
+
# this will hold information for the summary report
# it will be in the form
# [displayName] = {"passEpsilonPercent": percent ok with epsilon,
@@ -833,9 +846,13 @@ def reportGen_library_call (a_path, b_path, var_list=[ ],
LOG.debug ("filter function was applied to file B data for variable: " + explanationName)
# pre-check if this data should be plotted and if it should be compared to the longitude and latitude
- # TODO, this test may change depending on how plugable plotting functions are implemented
- include_images_for_this_variable = ((not('shouldIncludeImages' in varRunInfo)) or (varRunInfo['shouldIncludeImages']))
- do_not_test_with_lon_lat = (('shouldIncludeImages' in varRunInfo) and (not varRunInfo['shouldIncludeImages']))
+ include_images_for_this_variable = ((not('shouldIncludeImages' in runInfo)) or (runInfo['shouldIncludeImages']))
+ if 'shouldIncludeImages' in varRunInfo :
+ include_images_for_this_variable = varRunInfo['shouldIncludeImages']
+ do_not_test_with_lon_lat = (not include_images_for_this_variable) or (len(lon_lat_data.keys()) <= 0)
+
+ LOG.debug ("do_not_test_with_lon_lat = " + str(do_not_test_with_lon_lat))
+ LOG.debug ("include_images_for_this_variable = " + str(include_images_for_this_variable))
# check if this data can be displayed but
# don't compare lon/lat sizes if we won't be plotting
@@ -843,7 +860,7 @@ def reportGen_library_call (a_path, b_path, var_list=[ ],
and
( do_not_test_with_lon_lat
or
- ((aData.shape == common_lon_lat_data['lon'].shape) and (bData.shape == common_lon_lat_data['lon'].shape)) ) ) :
+ ((aData.shape == good_shape_from_lon_lat) and (bData.shape == good_shape_from_lon_lat)) ) ) :
# check to see if there is a directory to put information about this variable in,
# if not then create it
@@ -857,11 +874,11 @@ def reportGen_library_call (a_path, b_path, var_list=[ ],
os.makedirs(variableDir)
# figure out the masks we want, and then do our statistical analysis
- mask_a_to_use = a_lon_lat_data['inv_mask']
- mask_b_to_use = b_lon_lat_data['inv_mask']
- if do_not_test_with_lon_lat :
- mask_a_to_use = None
- mask_b_to_use = None
+ mask_a_to_use = None
+ mask_b_to_use = None
+ if not do_not_test_with_lon_lat :
+ mask_a_to_use = lon_lat_data['a']['inv_mask']
+ mask_b_to_use = lon_lat_data['b']['inv_mask']
variable_stats = delta.summarize(aData, bData,
varRunInfo['epsilon'],
(varRunInfo['missing_value'],
@@ -888,27 +905,82 @@ def reportGen_library_call (a_path, b_path, var_list=[ ],
}
# create the images for this variable
+ # TODO, will need to handle averaged/sliced 3D data at some point
if (include_images_for_this_variable) :
- # create the images comparing the variable
- print("\tcreating figures for: " + explanationName)
- image_names['original'], image_names['compared'] = \
- plot.plot_and_save_figure_comparison(aData,
- bData,
- varRunInfo,
- files['file A']['path'],
- files['file B']['path'],
- # TODO, pass this in a more efficient way?
- a_lon_lat_data['lat'], a_lon_lat_data['lon'],
- b_lon_lat_data['lat'], b_lon_lat_data['lon'],
- common_lon_lat_data['lat'], common_lon_lat_data['lon'],
- a_lon_lat_data['inv_mask'],
- b_lon_lat_data['inv_mask'],
- varRunInfo['variable_dir'],
- True,
- doFork=runInfo['doFork'],
- shouldClearMemoryWithThreads=runInfo['useThreadsToControlMemory'],
- shouldUseSharedRangeForOriginal=runInfo['useSharedRangeForOriginal'])
+ # if we only have 1D data, we will want a line plot
+ if (len(aData.shape) is 1) :
+ image_names['original'], image_names['compared'] = \
+ plot.plot_and_save_comparison_figures \
+ (aData, bData,
+ plot.make_line_plot_plotting_functions,
+ varRunInfo['variable_dir'],
+ displayName,
+ varRunInfo['epsilon'],
+ varRunInfo['missing_value'],
+ missingValueAltInB = varRunInfo['missingValueAltInB'] if 'missingValueAltInB' in varRunInfo else None,
+ makeSmall=True,
+ doFork=runInfo['doFork'],
+ shouldClearMemoryWithThreads=runInfo['useThreadsToControlMemory'],
+ shouldUseSharedRangeForOriginal=runInfo['useSharedRangeForOriginal'],
+ doPlotOriginal = varRunInfo['do_plot_originals'] if 'do_plot_originals' in varRunInfo else True,
+ doPlotAbsDiff = varRunInfo['do_plot_abs_diff'] if 'do_plot_abs_diff' in varRunInfo else True,
+ doPlotSubDiff = varRunInfo['do_plot_sub_diff'] if 'do_plot_sub_diff' in varRunInfo else True,
+ doPlotTrouble = varRunInfo['do_plot_trouble'] if 'do_plot_trouble' in varRunInfo else True,
+ doPlotHistogram= varRunInfo['do_plot_histogram'] if 'do_plot_histogram' in varRunInfo else True,
+ doPlotScatter = varRunInfo['do_plot_scatter'] if 'do_plot_scatter' in varRunInfo else True)
+
+ elif (len(aData.shape) is 2) :
+
+ # create 2D images comparing the variable
+ print("\tcreating figures for: " + explanationName)
+ if do_not_test_with_lon_lat :
+ # plot our non lon/lat info
+ image_names['original'], image_names['compared'] = \
+ plot.plot_and_save_comparison_figures \
+ (aData, bData,
+ plot.make_pure_data_plotting_functions,
+ varRunInfo['variable_dir'],
+ displayName,
+ varRunInfo['epsilon'],
+ varRunInfo['missing_value'],
+ missingValueAltInB = varRunInfo['missingValueAltInB'] if 'missingValueAltInB' in varRunInfo else None,
+ makeSmall=True,
+ doFork=runInfo['doFork'],
+ shouldClearMemoryWithThreads=runInfo['useThreadsToControlMemory'],
+ shouldUseSharedRangeForOriginal=runInfo['useSharedRangeForOriginal'],
+ doPlotOriginal = varRunInfo['do_plot_originals'] if 'do_plot_originals' in varRunInfo else True,
+ doPlotAbsDiff = varRunInfo['do_plot_abs_diff'] if 'do_plot_abs_diff' in varRunInfo else True,
+ doPlotSubDiff = varRunInfo['do_plot_sub_diff'] if 'do_plot_sub_diff' in varRunInfo else True,
+ doPlotTrouble = varRunInfo['do_plot_trouble'] if 'do_plot_trouble' in varRunInfo else True,
+ doPlotHistogram= varRunInfo['do_plot_histogram'] if 'do_plot_histogram' in varRunInfo else True,
+ doPlotScatter = varRunInfo['do_plot_scatter'] if 'do_plot_scatter' in varRunInfo else True)
+ else :
+ # plot our lon/lat related info
+ image_names['original'], image_names['compared'] = \
+ plot.plot_and_save_comparison_figures \
+ (aData, bData,
+ plot.make_geolocated_plotting_functions,
+ varRunInfo['variable_dir'],
+ displayName,
+ varRunInfo['epsilon'],
+ varRunInfo['missing_value'],
+ missingValueAltInB = varRunInfo['missingValueAltInB'] if 'missingValueAltInB' in varRunInfo else None,
+ lonLatDataDict=lon_lat_data,
+ dataRanges = varRunInfo['display_ranges'] if 'display_ranges' in varRunInfo else None,
+ dataRangeNames = varRunInfo['display_range_names'] if 'display_range_names' in varRunInfo else None,
+ dataColors = varRunInfo['display_colors'] if 'display_colors' in varRunInfo else None,
+ makeSmall=True,
+ doFork=runInfo['doFork'],
+ shouldClearMemoryWithThreads=runInfo['useThreadsToControlMemory'],
+ shouldUseSharedRangeForOriginal=runInfo['useSharedRangeForOriginal'],
+ doPlotOriginal = varRunInfo['do_plot_originals'] if 'do_plot_originals' in varRunInfo else True,
+ doPlotAbsDiff = varRunInfo['do_plot_abs_diff'] if 'do_plot_abs_diff' in varRunInfo else True,
+ doPlotSubDiff = varRunInfo['do_plot_sub_diff'] if 'do_plot_sub_diff' in varRunInfo else True,
+ doPlotTrouble = varRunInfo['do_plot_trouble'] if 'do_plot_trouble' in varRunInfo else True,
+ doPlotHistogram= varRunInfo['do_plot_histogram'] if 'do_plot_histogram' in varRunInfo else True,
+ doPlotScatter = varRunInfo['do_plot_scatter'] if 'do_plot_scatter' in varRunInfo else True)
+
print("\tfinished creating figures for: " + explanationName)
# create the report page for this variable
@@ -932,12 +1004,16 @@ def reportGen_library_call (a_path, b_path, var_list=[ ],
# if we can't compare the variable, we should tell the user
else :
- LOG.warn(explanationName + ' ' +
+ message = (explanationName + ' ' +
'could not be compared. This may be because the data for this variable does not match in shape ' +
'between the two files (file A data shape: ' + str(aData.shape) + '; file B data shape: '
- + str(bData.shape) + ') or the data may not match the shape of the selected longitude '
- + str(common_lon_lat_data['lon'].shape) + ' and ' +
- 'latitude ' + str(common_lon_lat_data['lat'].shape) + ' variables.')
+ + str(bData.shape) + ')')
+ if do_not_test_with_lon_lat :
+ message = message + '.'
+ else :
+ message = (message + ' or the data may not match the shape of the selected longitude ' +
+ str(good_shape_from_lon_lat) + ' and ' + 'latitude ' + str(good_shape_from_lon_lat) + ' variables.')
+ LOG.warn(message)
# the end of the loop to examine all the variables
@@ -1067,6 +1143,8 @@ python -m glance
action="store_true", default=False, help="view the glance version")
parser.add_option('-f', '--fork', dest='doFork',
action="store_true", default=False, help="start multiple processes to create images in parallel")
+ parser.add_option('-d', '--nolonlat', dest='noLonLatVars',
+ action="store_true", default=False, help="do not try to find or analyze logitude and latitude")
options, args = parser.parse_args()
@@ -1250,6 +1328,7 @@ python -m glance
tempOptions['imagesOnly'] = options.imagesOnly
tempOptions['htmlOnly'] = options.htmlOnly
tempOptions['doFork'] = options.doFork
+ tempOptions['noLonLatVars'] = options.noLonLatVars
tempOptions['latitudeVar'] = options.latitudeVar
tempOptions['longitudeVar'] = options.longitudeVar
tempOptions['lonlatepsilon'] = options.lonlatepsilon
diff --git a/pyglance/glance/filters.py b/pyglance/glance/filters.py
index 2d08c67d39d0b1816a027ff28df6261931585880..ba81d09830a96b46083977b472415033b642660d 100644
--- a/pyglance/glance/filters.py
+++ b/pyglance/glance/filters.py
@@ -7,7 +7,7 @@ Created by Eva Schiffer August 2009.
Copyright (c) 2009 University of Wisconsin SSEC. All rights reserved.
"""
-import numpy as np
+import numpy as np
def trim_off_of_top (data, num_elements_to_trim) :
"""
@@ -161,5 +161,99 @@ def set_to_value_between_bounds(data, value_to_set_to, bottom_bound_exclusive, t
return data
+def collapse_to_index(data, index, collapsing_function=np.mean,
+ missing_value=None, ignore_below_exclusive=None, ignore_above_exclusive=None) :
+ """
+ collapse the data given along the selected index, using the collapsing_function
+ """
+
+ # figure the order to put the index we want first
+ new_index_order = range(len(data.shape))
+ del(new_index_order[index])
+ new_index_order = [index] + new_index_order
+
+ # copy our data and put the index we won't collapse first
+ new_data = data.copy()
+ new_data = new_data.transpose(new_index_order)
+
+ # find any bad points that we don't want to collapse
+ bad_mask = np.zeros(new_data.shape, dtype=bool)
+ if missing_value is not None :
+ bad_mask[new_data == missing_value] = True
+ if ignore_above_exclusive is not None :
+ bad_mask[new_data > ignore_above_exclusive] = True
+ if ignore_below_exclusive is not None :
+ bad_mask[new_data < ignore_below_exclusive] = True
+
+ # collapse any non bad data
+ final_num_pts = new_data.shape[0]
+ collapsed_data = np.zeros((final_num_pts), dtype=new_data.dtype)
+ for index_1_val in range(final_num_pts) :
+ values_to_use = new_data[index_1_val][~(bad_mask[index_1_val])]
+ if len(values_to_use) > 0 :
+ collapsed_data[index_1_val] = collapsing_function(values_to_use)
+ else:
+ collapsed_data[index_1_val] = missing_value
+
+ return collapsed_data
+
+def organize_ipopp_data_into_image(original_ipopp_data, wave_number=None, missing_value=None,
+ propagate_partial_missing_values=False) :
+ """
+ organize the ipopp data spatially into an 'image' of sorts
+ this basically consists of:
+
+ -> the 30 fields of regard
+ ---------------
+ | | |
+ | | V
+ | | the 4 scan lines
+ | |
+ ---------------
+
+ for each field of regard/scan line
+ _______
+ |_|_|_|
+ |_|_|_|
+ |_|_|_|
+
+ block of the 9 detectors (TODO order?
+ for the moment assuming increasing linear by line then field of regard)
+
+ for each detector point, if the wave_number was given
+ in the parameters, that specific interferogram data pt will be used
+ if no wave_number was given, the mean of the 717 pts will be used
+ """
+
+ new_data_image = np.zeros((4 * 3, 30 * 3), dtype=original_ipopp_data.dtype)
+
+ # loop to the place in the old array to get each data point
+ for scan_line in range(4) :
+ for field_of_regard in range(30) :
+
+ for detector in range(9) :
+
+ # figure out the value we're moving
+ data_pt = None
+ data_array = original_ipopp_data[scan_line][field_of_regard][detector]
+ if (wave_number is not None):
+ data_pt = data_array[wave_number]
+ else:
+ if (propagate_partial_missing_values
+ and
+ sum(data_array == missing_value) > 0) :
+
+ data_pt = missing_value
+ else:
+ # remember to remove any remaining missing values
+ data_pt = np.mean(data_array[~(data_array == missing_value)])
+
+ # figure out where to put the value and put it there
+ index1 = scan_line * 3 + (detector / 3)
+ index2 = field_of_regard * 3 + (detector % 3)
+ new_data_image[index1][index2] = data_pt
+
+ return new_data_image
+
if __name__=='__main__':
sys.exit(main())
\ No newline at end of file
diff --git a/pyglance/glance/mainreport.txt b/pyglance/glance/mainreport.txt
index 98b04228d6b724d3beb2843a16857f7036a8cdb6..7681a7714903dcdc541d834e0e85d9f8d2b23317 100644
--- a/pyglance/glance/mainreport.txt
+++ b/pyglance/glance/mainreport.txt
@@ -47,107 +47,113 @@ Copyright (c) 2009 University of Wisconsin SSEC. All rights reserved.
</p>
% endif
- ## display the latitude and longitude variable names
- <p>
- % if ('latitude_alt_name_in_b' in runInfo) :
- latitude in A: ${runInfo['latitude']}<br>
- latitude in B: ${runInfo['latitude_alt_name_in_b']}<br>
- % else :
- latitude: ${runInfo['latitude']} <br>
- % endif
- % if ('longitude_alt_name_in_b' in runInfo) :
- longitude in A: ${runInfo['longitude']}<br>
- longitude in B: ${runInfo['longitude_alt_name_in_b']}<br>
- % else :
- longitude: ${runInfo['longitude']}<br>
- % endif
-
- % if ('lon_lat_epsilon' in runInfo) and (runInfo['lon_lat_epsilon'] > 0.0) :
- longitude/latitude comparison epsilon: ${runInfo['lon_lat_epsilon']}<br>
- % endif
-
- <br>
- ## display information about any data filtering on the lons/lats
- % if ('data_filter_function_lat_in_a' in runInfo) and (not (runInfo['data_filter_function_lat_in_a'] is None)) :
- Note: The latitude in file A was filtered.
- Please see <a href="${runInfo['config_file_name']}">this copy of
- the original configuration file</a> to view the data filtering function.<br>
- % endif
- % if ('data_filter_function_lat_in_b' in runInfo) and (not (runInfo['data_filter_function_lat_in_b'] is None)) :
- Note: The latitude in file B was filtered.
- Please see <a href="${runInfo['config_file_name']}">this copy of
- the original configuration file</a> to view the data filtering function.<br>
- % endif
- % if ('data_filter_function_lon_in_a' in runInfo) and (not (runInfo['data_filter_function_lon_in_a'] is None)) :
- Note: The longitude in file A was filtered.
- Please see <a href="${runInfo['config_file_name']}">this copy of
- the original configuration file</a> to view the data filtering function.<br>
- % endif
- % if ('data_filter_function_lon_in_b' in runInfo) and (not (runInfo['data_filter_function_lon_in_b'] is None)) :
- Note: The longitude in file B was filtered.
- Please see <a href="${runInfo['config_file_name']}">this copy of
- the original configuration file</a> to view the data filtering function.<br>
- % endif
- </p>
+ ## if the lon/lat variables exist, display info on them
+ %if ('latitude' in runInfo) and ('longitude' in runInfo) :
- ## if there is a problem with the longitude/latitude correlation between the two files,
- ## make a nice big warning for the user
- % if spatial.has_key('lon_lat_not_equal_points_count') and (spatial['lon_lat_not_equal_points_count'] > 0) :
+ ## display the latitude and longitude variable names
<p>
- WARNING: ${spatial['lon_lat_not_equal_points_count']} data points
- (${report.make_formatted_display_string(spatial['lon_lat_not_equal_points_percent'])}% of all data)
- show possible mismatch in values stored in file a
- and file b longitude and latitude values. Depending on the degree of mismatch, some data value comparisons
- in this report may be distorted or spatially nonsensical. Please consider re-running this report and including an
- examination of your longitude and latitude variables with appropriate epsilons in order to analyze the significance
- of the difference.<br>
- ## if we're showing images, link to graphs showing the problem
- % if runInfo['shouldIncludeImages'] :
- <a href="./LonLatMismatch.A.png">
- View mismatching points in A's lon/lat system
- </a><br>
- <a href="./LonLatMismatch.B.png">
- View mismatching points in B's lon/lat system
- </a>
+ % if ('latitude_alt_name_in_b' in runInfo) :
+ latitude in A: ${runInfo['latitude']}<br>
+ latitude in B: ${runInfo['latitude_alt_name_in_b']}<br>
+ % else :
+ latitude: ${runInfo['latitude']} <br>
% endif
- </p>
- % endif
-
- ## if we have some unique spatially invalid points, report them
- ## are there any in file A?
- %if spatial.has_key('file A') and spatial['file A'].has_key('numInvPts') and (spatial['file A']['numInvPts'] > 0) :
- <%
- fileAInfo = spatial['file A']
- %>
- <p>
- % if runInfo['shouldIncludeImages'] :
- <a href="./SpatialMismatch.A.png"><img src="./SpatialMismatch.A.small.png"></a><br>
+ % if ('longitude_alt_name_in_b' in runInfo) :
+ longitude in A: ${runInfo['longitude']}<br>
+ longitude in B: ${runInfo['longitude_alt_name_in_b']}<br>
+ % else :
+ longitude: ${runInfo['longitude']}<br>
% endif
- File A has ${fileAInfo['numInvPts']} valid data points not present in File B.<br>
- File A has a total of ${report.make_formatted_display_string(fileAInfo['perInvPts'])}% invalid longitude/latitude data. <br>
- </p>
- % endif
- ## are there any in file B?
- %if spatial.has_key('file B') and spatial['file B'].has_key('numInvPts') and (spatial['file B']['numInvPts'] > 0) :
- <%
- fileBInfo = spatial['file B']
- %>
- <p>
- % if runInfo['shouldIncludeImages'] :
- <a href="./SpatialMismatch.B.png"><img src="./SpatialMismatch.B.small.png"></a><br>
+
+ % if ('lon_lat_epsilon' in runInfo) and (runInfo['lon_lat_epsilon'] > 0.0) :
+ longitude/latitude comparison epsilon: ${runInfo['lon_lat_epsilon']}<br>
+ % endif
+
+ <br>
+ ## display information about any data filtering on the lons/lats
+ % if ('data_filter_function_lat_in_a' in runInfo) and (not (runInfo['data_filter_function_lat_in_a'] is None)) :
+ Note: The latitude in file A was filtered.
+ Please see <a href="${runInfo['config_file_name']}">this copy of
+ the original configuration file</a> to view the data filtering function.<br>
+ % endif
+ % if ('data_filter_function_lat_in_b' in runInfo) and (not (runInfo['data_filter_function_lat_in_b'] is None)) :
+ Note: The latitude in file B was filtered.
+ Please see <a href="${runInfo['config_file_name']}">this copy of
+ the original configuration file</a> to view the data filtering function.<br>
+ % endif
+ % if ('data_filter_function_lon_in_a' in runInfo) and (not (runInfo['data_filter_function_lon_in_a'] is None)) :
+ Note: The longitude in file A was filtered.
+ Please see <a href="${runInfo['config_file_name']}">this copy of
+ the original configuration file</a> to view the data filtering function.<br>
+ % endif
+ % if ('data_filter_function_lon_in_b' in runInfo) and (not (runInfo['data_filter_function_lon_in_b'] is None)) :
+ Note: The longitude in file B was filtered.
+ Please see <a href="${runInfo['config_file_name']}">this copy of
+ the original configuration file</a> to view the data filtering function.<br>
% endif
- File B has ${fileBInfo['numInvPts']} valid data points not present in File A.<br>
- File B has a total of ${report.make_formatted_display_string(fileBInfo['perInvPts'])}% longitude/latitude invalid data. <br>
</p>
- % endif
- ## report on shared spatial invalidity if there is any
- % if spatial.has_key('perInvPtsInBoth') :
- <% perInBoth = spatial['perInvPtsInBoth'] %>
- % if perInBoth > 0 :
+
+ ## if there is a problem with the longitude/latitude correlation between the two files,
+ ## make a nice big warning for the user
+ % if spatial.has_key('lon_lat_not_equal_points_count') and (spatial['lon_lat_not_equal_points_count'] > 0) :
<p>
- lon/lat data that is invalid in either file A or file B: ${report.make_formatted_display_string(perInBoth)}%
+ WARNING: ${spatial['lon_lat_not_equal_points_count']} data points
+ (${report.make_formatted_display_string(spatial['lon_lat_not_equal_points_percent'])}% of all data)
+ show possible mismatch in values stored in file a
+ and file b longitude and latitude values. Depending on the degree of mismatch, some data value comparisons
+ in this report may be distorted or spatially nonsensical. Please consider re-running this report and including an
+ examination of your longitude and latitude variables with appropriate epsilons in order to analyze the significance
+ of the difference.<br>
+ ## if we're showing images, link to graphs showing the problem
+ % if runInfo['shouldIncludeImages'] :
+ <a href="./LonLatMismatch.A.png">
+ View mismatching points in A's lon/lat system
+ </a><br>
+ <a href="./LonLatMismatch.B.png">
+ View mismatching points in B's lon/lat system
+ </a>
+ % endif
</p>
% endif
+
+ ## if we have some unique spatially invalid points, report them
+ ## are there any in file A?
+ %if spatial.has_key('file A') and spatial['file A'].has_key('numInvPts') and (spatial['file A']['numInvPts'] > 0) :
+ <%
+ fileAInfo = spatial['file A']
+ %>
+ <p>
+ % if runInfo['shouldIncludeImages'] :
+ <a href="./SpatialMismatch.A.png"><img src="./SpatialMismatch.A.small.png"></a><br>
+ % endif
+ File A has ${fileAInfo['numInvPts']} valid data points not present in File B.<br>
+ File A has a total of ${report.make_formatted_display_string(fileAInfo['perInvPts'])}% invalid longitude/latitude data. <br>
+ </p>
+ % endif
+ ## are there any in file B?
+ %if spatial.has_key('file B') and spatial['file B'].has_key('numInvPts') and (spatial['file B']['numInvPts'] > 0) :
+ <%
+ fileBInfo = spatial['file B']
+ %>
+ <p>
+ % if runInfo['shouldIncludeImages'] :
+ <a href="./SpatialMismatch.B.png"><img src="./SpatialMismatch.B.small.png"></a><br>
+ % endif
+ File B has ${fileBInfo['numInvPts']} valid data points not present in File A.<br>
+ File B has a total of ${report.make_formatted_display_string(fileBInfo['perInvPts'])}% longitude/latitude invalid data. <br>
+ </p>
+ % endif
+ ## report on shared spatial invalidity if there is any
+ % if spatial.has_key('perInvPtsInBoth') :
+ <% perInBoth = spatial['perInvPtsInBoth'] %>
+ % if perInBoth > 0 :
+ <p>
+ lon/lat data that is invalid in either file A or file B: ${report.make_formatted_display_string(perInBoth)}%
+ </p>
+ % endif
+ % endif
+
+ ## end of the if to display lon/lat info
% endif
% if len(variables.keys()) > 0 :
diff --git a/pyglance/glance/plot.py b/pyglance/glance/plot.py
index 40164b6f56f26ec7b66094af9da4128e72c6edf2..4f57bf1cfab06cccbc9bbd994ebce6c364b9f557 100644
--- a/pyglance/glance/plot.py
+++ b/pyglance/glance/plot.py
@@ -22,6 +22,7 @@ from PIL import Image
import os, sys, logging
import numpy as np
+from numpy import ma
import glance.graphics as maps
import glance.delta as delta
@@ -132,6 +133,9 @@ def _create_histogram(data, bins, title, xLabel, yLabel, displayStats=False) :
figure = plt.figure()
axes = figure.add_subplot(111)
+ if (data is None) or (len(data) <= 0) :
+ return figure
+
# the histogram of the data
n, bins, patches = plt.hist(data, bins)
@@ -367,6 +371,152 @@ def _create_mapped_figure(data, latitude, longitude, baseMapInstance, boundingAx
return figure
+def _create_simple_figure(data, figureTitle, invalidMask=None, tagData=None, colorMap=None) :
+ """
+ create a simple figure showing the data given masked by the invalid mask
+ any tagData passed in will be interpreted as trouble points on the image and plotted as a
+ filled contour overlay in green on the image
+ if a colorMap is given it will be used to plot the data,
+ if not the default colorMap for imshow will be used
+ """
+
+ cleanData = ma.array(data, mask=invalidMask)
+
+ # build the plot
+ figure = plt.figure()
+ axes = figure.add_subplot(111)
+
+ # build extra info to go to the map plotting function
+ kwargs = { }
+
+ # if we've got a color map, pass it to the list of things we want to tell the plotting function
+ if not (colorMap is None) :
+ kwargs['cmap'] = colorMap
+
+ if (data is not None) and (sum(invalidMask) < invalidMask.size) :
+ # draw our data
+ im = imshow(cleanData, **kwargs)
+ # make a color bar
+ cbar = colorbar(format='%.3f')
+
+ # and some informational stuff
+ axes.set_title(figureTitle)
+
+ # if there are "tag" masks, plot them over the existing map
+ if not (tagData is None) :
+
+ numTroublePoints = sum(tagData)
+
+ # if we have trouble points, we need to show them
+ if numTroublePoints > 0:
+
+ # figure out how many bad points there are
+ totalNumPoints = tagData.size # the number of points
+ percentBad = (float(numTroublePoints) / float(totalNumPoints)) * 100.0
+ LOG.debug('\t\tnumber of trouble points: ' + str(numTroublePoints))
+ LOG.debug('\t\tpercent of trouble points: ' + str(percentBad))
+
+
+ new_kwargs = {}
+ new_kwargs['cmap'] = greenColorMap
+ cleanTagData = ma.array(tagData, mask=~tagData)
+ p = contourf(cleanTagData, **new_kwargs)
+
+ # display the number of trouble points on the report if we were passed a set of tag data
+ troublePtString = '\n\nShowing ' + str(numTroublePoints) + ' Trouble Points'
+ # if our plot is more complex, add clarification
+ if numTroublePoints > 0 :
+ troublePtString = troublePtString + ' in Green'
+ plt.xlabel(troublePtString)
+
+ return figure
+
+def _create_line_plot_figure(dataList, figureTitle, tagData=None) :
+ """
+ create a basic line plot of the data vs. it's index, ignoring any invalid data
+ if tagData is given, under-label those points with green circles
+
+ Each entry in the dataList should be a tupple containing:
+ (data, invalidMask, colorString, labelName)
+
+ The color string describes a color for plotting in matplotlib.
+ The label names will be used for the legend, which will be shown if there is
+ more than one set of data plotted or if there is tag data plotted. Invalid
+ masks, colors, and label names may be given as None, in which case no data
+ will be masked and a default label of "data#" (where # is an arbitrary
+ unique counter) will be used.
+ """
+
+ # build the plot
+ figure = plt.figure()
+ axes = figure.add_subplot(111)
+
+ # plot each of the data sets
+ dataSetLabelNumber = 1
+ minTagPts = -1
+ maxTagPts = -1
+ for dataSet, invalidMask, colorString, labelName in dataList :
+
+ # if we don't have these, set them to defaults
+ if invalidMask is None :
+ invalidMask = zeros(dataSet.shape, dtype=bool)
+ if labelName is None :
+ labelName = 'data' + str(dataSetLabelNumber)
+ dataSetLabelNumber = dataSetLabelNumber + 1
+ if colorString is None:
+ colorString = ''
+
+ if (dataSet is not None) and (sum(invalidMask) < invalidMask.size) :
+
+ # if we don't have a real min yet, set it based on the size
+ if minTagPts < 0 :
+ minTagPts = dataSet.size + 1
+
+ indexData = ma.array(range(dataSet.size), mask=invalidMask)
+ cleanData = ma.array(dataSet, mask=invalidMask)
+
+ # plot the tag data and gather information about it
+ if tagData is not None :
+
+ numTroublePoints = sum(tagData)
+ LOG.debug('\t\tnumber of trouble points: ' + str(numTroublePoints))
+ if numTroublePoints < minTagPts:
+ minTagPts = numTroublePoints
+ if numTroublePoints > maxTagPts :
+ maxTagPts = numTroublePoints
+
+ # if we have trouble points, we need to show them
+ if numTroublePoints > 0:
+
+ cleanTagData = ma.array(dataSet, mask=~tagData | invalidMask)
+ axes.plot(indexData, cleanTagData, 'yo', label='trouble point')
+
+ axes.plot(indexData, cleanData, '-' + colorString, label=labelName)
+
+ # display the number of trouble points on the report if we were passed
+ # a set of tag data and we were able to compare it to some actual data
+ if ((tagData is not None) and (minTagPts >= 0) and (maxTagPts >=0)) :
+
+ troublePtString = '\nMarking '
+
+ if (minTagPts == maxTagPts) :
+ troublePtString = troublePtString + str(minTagPts) + ' Trouble Points with Yellow Circles'
+ else :
+ troublePtString = (troublePtString + 'between ' + str(minTagPts) + ' and ' + str(maxTagPts) + ' Trouble Points'
+ + '\non the various data sets (using Yellow Circles)')
+
+ plt.xlabel(troublePtString)
+
+ if (len(dataList) > 1) or (tagData is not None) :
+ # make a key to explain our plot
+ # as long as things have been plotted with proper labels they should show up here
+ axes.legend(loc=0, markerscale=3.0) # Note: at the moment markerscale doesn't seem to work
+
+ # and some informational stuff
+ axes.set_title(figureTitle)
+
+ return figure
+
# figure out the bounding axes for the display given a set of
# longitude and latitude and possible a mask of invalid values
# that we should ignore in them
@@ -500,76 +650,57 @@ def _handle_fig_creation_task(child_figure_function, log_message,
# if we didn't fork, return the 0 pid to indicate that
return pid
-def plot_and_save_figure_comparison(aData, bData,
- variableRunInfo,
- fileAName, fileBName,
- latitudeAData, longitudeAData,
- latitudeBData, longitudeBData,
- latitudeCommonData, longitudeCommonData,
- spaciallyInvalidMaskA,
- spaciallyInvalidMaskB,
- outputPath,
- makeSmall=False,
- shortCircuitComparisons=False,
- doFork=False,
- shouldClearMemoryWithThreads=False,
- shouldUseSharedRangeForOriginal=False) :
+def _log_spawn_and_wait_if_needed (imageDescription, childPids, imageList,
+ taskFunction, taskOutputPath, taskFigName,
+ doMakeThumb=True, doFork=False, shouldClearMemoryWithThreads=False) :
"""
- given two files, and information on what to compare, make comparison
- figures and save them to the given output graph.
- Four figures will be generated, one for each file, a comparison image
- that represents the amount of difference in that variable between the
- two files, and an image highlighting the trouble spots where the
- difference exceeds epsilon or there are missing or nan values in one
- or both of the files
-
- variableRunInfo is a dictionary in the form
- variableRunInfo = { 'variable_name': variableName,
- 'epsilon': epsilon,
- 'missing_value': missingDataValue,
- 'display_name': displayName # this entry is optional; the variable_name should be used
- # for descriptive purposes if it is not defined
- }
-
- returns a list of "original" data images and a list of "compared" data images
- these are only the image names, not the full path to the images
+ create a figure generation task, spawning a process as needed
+ save the childPid to the list of pids if the process will remain outstanding after this method ends
+ the name of the figure that was generated will be added to the image list
"""
+ LOG.info("creating image of "+ imageDescription)
+
+ # start the actual task
+ pid = _handle_fig_creation_task(taskFunction,
+ "saving image of " + imageDescription,
+ taskOutputPath, taskFigName,
+ doMakeThumb, doFork or shouldClearMemoryWithThreads)
+
+ # wait based on the state of the pid we received and why we would have forked
+ childPid = None
+ if not (pid is 0) :
+ if doFork :
+ childPids.append(pid)
+ LOG.debug ("Started child process (pid: " + str(pid) + ") to create image of " + imageDescription)
+ else :
+ os.waitpid(pid, 0)
+ imageList.append(taskFigName)
- # lists to hold information on the images we make
- original_images = [ ]
- compared_images = [ ]
-
- # if we weren't given a variable display name,
- # just use the standard variable name instead
- variableName = variableRunInfo['variable_name']
- variableDisplayName = variableName
- if 'display_name' in variableRunInfo :
- variableDisplayName = variableRunInfo['display_name']
-
- # figure out what missing values we should be using
- missing_value = variableRunInfo['missing_value']
- missing_value_b = missing_value
- if ('missing_value_alt_in_b' in variableRunInfo) :
- missing_value_b = variableRunInfo['missing_value_alt_in_b']
-
- # compare the two data sets to get our difference data and trouble info
- rawDiffData, goodMask, (goodInAMask, goodInBMask), troubleMask, outsideEpsilonMask, \
- (aNotFiniteMask, bNotFiniteMask), (aMissingMask, bMissingMask), \
- (spaciallyInvalidMaskA, spaciallyInvalidMaskB) = delta.diff(aData, bData, variableRunInfo['epsilon'],
- (missing_value, missing_value_b),
- (spaciallyInvalidMaskA, spaciallyInvalidMaskB))
- absDiffData = np.abs(rawDiffData) # we also want to show the distance between our two, not just which one's bigger/smaller
-
- # some more display info, pull it out for convenience
- dataRanges = None
- if ('display_ranges' in variableRunInfo) :
- dataRanges = variableRunInfo['display_ranges']
- dataRangeNames = None
- if ('display_range_names' in variableRunInfo) :
- dataRangeNames = variableRunInfo['display_range_names']
- dataColors = None
- if ('display_colors' in variableRunInfo) :
- dataColors = variableRunInfo['display_colors']
+ return
+
+def make_geolocated_plotting_functions(aData, bData,
+ absDiffData, rawDiffData,
+ variableDisplayName,
+ epsilon,
+ goodInAMask, goodInBMask, goodMask,
+ troubleMask, outsideEpsilonMask,
+
+ # parameters that are only needed for geolocated data
+ lonLatDataDict,
+ shouldUseSharedRangeForOriginal,
+ dataRanges, dataRangeNames, dataColors) :
+
+ functionsToReturn = { }
+
+ # TODO, possibly remove this section of definitions in the future?
+ latitudeAData = lonLatDataDict['a']['lat']
+ longitudeAData = lonLatDataDict['a']['lon']
+ latitudeBData = lonLatDataDict['b']['lat']
+ longitudeBData = lonLatDataDict['b']['lon']
+ latitudeCommonData = lonLatDataDict['common']['lat']
+ longitudeCommonData = lonLatDataDict['common']['lon']
+ spaciallyInvalidMaskA = lonLatDataDict['a']['inv_mask']
+ spaciallyInvalidMaskB = lonLatDataDict['b']['inv_mask']
# figure out the bounding axis
aAxis = _get_visible_axes(longitudeAData, latitudeAData, ~goodInAMask)
@@ -589,180 +720,351 @@ def plot_and_save_figure_comparison(aData, bData,
LOG.info('\t\tloading base map data')
baseMapInstance, fullAxis = maps.create_basemap(longitudeCommonData, latitudeCommonData, fullAxis, _select_projection(fullAxis))
- # from this point on, we will be forking to create child processes so we can parallelize our image and
- # report generation
- isParent = True
- childPids = []
-
- # the original data figures
-
# figure out the shared range for A and B's data, by default don't share a range
shared_range = None
if (shouldUseSharedRangeForOriginal) :
shared_range = _make_range(aData, ~goodInAMask, 50, offset_to_range=offsetToRange,
data_b=bData, invalid_b_mask=~goodInBMask)
+ # make the plotting functions
+
+ functionsToReturn['originalA'] = (lambda : _create_mapped_figure(aData, latitudeAData, longitudeAData,
+ baseMapInstance, fullAxis,
+ (variableDisplayName + "\nin File A"),
+ invalidMask=(~goodInAMask),
+ dataRanges=dataRanges or shared_range,
+ dataRangeNames=dataRangeNames,
+ dataRangeColors=dataColors))
+ functionsToReturn['originalB'] = (lambda : _create_mapped_figure(bData, latitudeBData, longitudeBData,
+ baseMapInstance, fullAxis,
+ (variableDisplayName + "\nin File B"),
+ invalidMask=(~ goodInBMask),
+ dataRanges=dataRanges or shared_range,
+ dataRangeNames=dataRangeNames,
+ dataRangeColors=dataColors))
+
+ functionsToReturn['absDiff'] = (lambda : _create_mapped_figure(absDiffData,
+ latitudeCommonData, longitudeCommonData,
+ baseMapInstance, fullAxis,
+ ("Absolute value of difference in\n" + variableDisplayName),
+ invalidMask=(~ goodMask)))
+ functionsToReturn['subDiff'] = (lambda : _create_mapped_figure(rawDiffData, latitudeCommonData, longitudeCommonData,
+ baseMapInstance, fullAxis,
+ ("Value of (Data File B - Data File A) for\n" + variableDisplayName),
+ invalidMask=(~ goodMask)))
+ # this is not an optimal solution, but we need to have at least somewhat valid data at any mismatched points so
+ # that our plot won't be totally destroyed by missing or non-finite data from B
+ bDataCopy = bData[:]
+ tempMask = goodInAMask & (~goodInBMask)
+ bDataCopy[tempMask] = aData[tempMask]
+ functionsToReturn['trouble'] = (lambda : _create_mapped_figure(bDataCopy, latitudeCommonData, longitudeCommonData,
+ baseMapInstance, fullAxis,
+ ("Areas of trouble data in\n" + variableDisplayName),
+ (~(goodInAMask | goodInBMask)),
+ mediumGrayColorMap, troubleMask,
+ dataRanges=dataRanges,
+ dataRangeNames=dataRangeNames))
+
+ # setup the data bins for the histogram
+ numBinsToUse = 50
+ valuesForHist = rawDiffData[goodMask]
+ functionsToReturn['histogram'] = (lambda : _create_histogram(valuesForHist, numBinsToUse,
+ ("Difference in\n" + variableDisplayName),
+ ('Value of (Data File B - Data File A) at a Data Point'),
+ ('Number of Data Points with a Given Difference'),
+ True))
+ functionsToReturn['scatter'] = (lambda : _create_scatter_plot(aData[goodMask], bData[goodMask],
+ "Value in File A vs Value in File B",
+ "File A Value", "File B Value",
+ outsideEpsilonMask[goodMask],
+ epsilon))
+
+ return functionsToReturn
+
+def make_pure_data_plotting_functions (aData, bData,
+ absDiffData, rawDiffData,
+ variableDisplayName,
+ epsilon,
+ goodInAMask, goodInBMask, goodMask,
+ troubleMask, outsideEpsilonMask,
+
+ # additional parameters this function isn't using
+ lonLatDataDict=None,
+ shouldUseSharedRangeForOriginal=False,
+ dataRanges=None, dataRangeNames=None, dataColors=None) :
+
+ functionsToReturn = { }
+
+ functionsToReturn['originalA'] = (lambda: _create_simple_figure(aData, variableDisplayName + "\nin File A",
+ invalidMask=~goodInAMask))
+ functionsToReturn['originalB'] = (lambda: _create_simple_figure(bData, variableDisplayName + "\nin File B",
+ invalidMask=~goodInBMask))
+
+ functionsToReturn['absDiff'] = (lambda: _create_simple_figure(absDiffData,
+ "Absolute value of difference in\n" + variableDisplayName,
+ invalidMask=~goodMask))
+ functionsToReturn['subDiff'] = (lambda: _create_simple_figure(rawDiffData,
+ "Value of (Data File B - Data File A) for\n" + variableDisplayName,
+ invalidMask=~goodMask))
+ # this is not an optimal solution, but we need to have at least somewhat valid data at any mismatched points so
+ # that our plot won't be totally destroyed by missing or non-finite data from B
+ bDataCopy = bData[:]
+ tempMask = goodInAMask & (~goodInBMask)
+ bDataCopy[tempMask] = aData[tempMask]
+ functionsToReturn['trouble'] = (lambda: _create_simple_figure(bDataCopy, "Areas of trouble data in\n" + variableDisplayName,
+ invalidMask=~(goodInAMask | goodInBMask), tagData=troubleMask,
+ colorMap=mediumGrayColorMap))
+
+ # set up the bins and data for a histogram of the values of fileA - file B
+ numBinsToUse = 50
+ valuesForHist = rawDiffData[goodMask]
+ functionsToReturn['histogram'] = (lambda : _create_histogram(valuesForHist, numBinsToUse,
+ ("Difference in\n" + variableDisplayName),
+ ('Value of (Data File B - Data File A) at a Data Point'),
+ ('Number of Data Points with a Given Difference'),
+ True))
+ functionsToReturn['scatter'] = (lambda : _create_scatter_plot(aData[goodMask], bData[goodMask],
+ "Value in File A vs Value in File B",
+ "File A Value", "File B Value",
+ outsideEpsilonMask[goodMask],
+ epsilon))
+
+ return functionsToReturn
+
+def make_line_plot_plotting_functions (aData, bData,
+ absDiffData, rawDiffData,
+ variableDisplayName,
+ epsilon,
+ goodInAMask, goodInBMask, goodMask,
+ troubleMask, outsideEpsilonMask,
+
+ # additional parameters this function isn't using
+ lonLatDataDict=None,
+ shouldUseSharedRangeForOriginal=False,
+ dataRanges=None, dataRangeNames=None, dataColors=None) :
+
+ functionsToReturn = { }
+
+ functionsToReturn['originalA'] = (lambda: _create_line_plot_figure([(aData, ~goodInAMask, 'r', 'A data'),
+ (bData, ~goodInBMask, 'b', 'B data')],
+ variableDisplayName + "\nin File A"))
+ #functionsToReturn['originalA'] = (lambda: _create_line_plot_figure(aData, variableDisplayName + "\nin File A",
+ # invalidMask=~goodInAMask))
+ functionsToReturn['originalB'] = (lambda: _create_line_plot_figure([(bData, ~goodInBMask, 'b', 'B data')],
+ variableDisplayName + "\nin File B"))
+
+ functionsToReturn['absDiff'] = (lambda: _create_line_plot_figure([(absDiffData, ~goodMask, 'k', 'abs. diff.')],
+ "Absolute value of difference in\n" + variableDisplayName))
+ functionsToReturn['subDiff'] = (lambda: _create_line_plot_figure([(rawDiffData, ~goodMask, 'k', 'raw diff.')],
+ "Value of (Data File B - Data File A) for\n" + variableDisplayName))
+ functionsToReturn['trouble'] = (lambda: _create_line_plot_figure([(aData, ~goodInAMask, 'r', 'A data'),
+ (bData, ~goodInBMask, 'b', 'B data')],
+ "Areas of trouble data in\n" + variableDisplayName,
+ tagData=troubleMask))
+
+ # set up the bins and data for a histogram of the values of fileA - file B
+ numBinsToUse = 50
+ valuesForHist = rawDiffData[goodMask]
+ functionsToReturn['histogram'] = (lambda : _create_histogram(valuesForHist, numBinsToUse,
+ ("Difference in\n" + variableDisplayName),
+ ('Value of (Data File B - Data File A) at a Data Point'),
+ ('Number of Data Points with a Given Difference'),
+ True))
+ functionsToReturn['scatter'] = (lambda : _create_scatter_plot(aData[goodMask], bData[goodMask],
+ "Value in File A vs Value in File B",
+ "File A Value", "File B Value",
+ outsideEpsilonMask[goodMask],
+ epsilon))
+
+ return functionsToReturn
+
+def plot_and_save_comparison_figures (aData, bData,
+ plottingFunctionGenerationFunction,
+ outputPath,
+ variableDisplayName,
+ epsilon,
+ missingValue,
+ missingValueAltInB=None,
+ lonLatDataDict=None,
+ dataRanges=None,
+ dataRangeNames=None,
+ dataColors=None,
+ makeSmall=False,
+ shortCircuitComparisons=False,
+ doFork=False,
+ shouldClearMemoryWithThreads=False,
+ shouldUseSharedRangeForOriginal=False,
+ doPlotOriginal=True,
+ doPlotAbsDiff=True,
+ doPlotSubDiff=True,
+ doPlotTrouble=True,
+ doPlotHistogram=True,
+ doPlotScatter=True) :
+ """
+ Plot images for a set of figures based on the data sets and settings
+ passed in. The images will be saved to disk according to the settings.
+
+ aData - the A file data
+ bData - the B file Data
+ lonLatDataDict - a dictionary of longitude and latitude info in the form
+ (or it may be empty if there is no lon/lat info or the
+ available lon/lat info should not be used for this data
+ set)
+
+ lonLatDataDict = {
+ 'a' = {
+ 'lon': longitudeDataForFileA,
+ 'lat': latitudeDataForFileA,
+ 'inv_mask': invalidMaskForFileA
+ },
+ 'b' = {
+ 'lon': longitudeDataForFileB,
+ 'lat': latitudeDataForFileB,
+ 'inv_mask': invalidMaskForFileB
+ },
+ 'common' = {
+ 'lon': longitudeDataCommonToBothFiles,
+ 'lat': latitudeDataCommonToBothFiles,
+ 'inv_mask': invalidMaskCommonToBothFiles
+ }
+ }
+
+ required parameters:
+
+ outputPath - the path where the output images will be placed
+ variableDisplayName - a descriptive name that will be shown on the
+ images to label the variable being analyzed
+ epsilon - the epsilon that should be used for the data comparison
+ missingValue - the missing value that should be used for the data comparison
+
+ optional parameters:
+
+ missingValueAltInB - an alternative missing value in the B file; if None is given
+ then the missingValue will be used for the B file
+ plottingFunction - the plotting function that will be used to make the plots,
+ defaults to using basemap to place your data on the globe
+ makeSmall - should small "thumbnail" images be made for each large image?
+ shortCircuitComparisons - should the comparison plots be disabled?
+ doFork - should the process fork to create new processes to create each
+ image? **
+ shouldClearMemoryWithThreads - should the process use fork to control long term
+ memory bloating? **
+ shouldUseSharedRangeForOriginal - should the original images share an all-inclusive
+ data range?
+ doPlotOriginal - should the plots of the two original data sets be made?
+ doPlotAbsDiff - should the plot of the absolute difference comparison image be made?
+ doPlotSubDiff - should the plot of the subtractive difference comparison image be made?
+ doPlotTrouble - should the plot of the trouble point image be made?
+ doPlotHistogram - should the plot of the historgram be made?
+ doPlotScatter - should the plot of the scatter plot be made?
+
+ ** May fail due to a known bug on MacOSX systems.
+ """
+
+ # lists to hold information on the images we make
+ # TODO, this will interact poorly with doFork
+ original_images = [ ]
+ compared_images = [ ]
+
+ # figure out what missing values we should be using
+ if missingValueAltInB is None :
+ missingValueAltInB = missingValue
+
+ # figure out if we have spatially invalid masks to consider
+ spaciallyInvalidMaskA = None
+ spaciallyInvalidMaskB = None
+ if lonLatDataDict is not None:
+ spaciallyInvalidMaskA = lonLatDataDict['a']['inv_mask']
+ spaciallyInvalidMaskB = lonLatDataDict['b']['inv_mask']
+
+ # compare the two data sets to get our difference data and trouble info
+ rawDiffData, goodMask, (goodInAMask, goodInBMask), troubleMask, outsideEpsilonMask, \
+ (aNotFiniteMask, bNotFiniteMask), (aMissingMask, bMissingMask), \
+ (spaciallyInvalidMaskA, spaciallyInvalidMaskB) = delta.diff(aData, bData, epsilon, (missingValue, missingValueAltInB),
+ (spaciallyInvalidMaskA, spaciallyInvalidMaskB))
+ absDiffData = np.abs(rawDiffData) # we also want to show the distance between our two, not just which one's bigger/smaller
+
+ LOG.debug("Visible axes will not be calculated for variable data (" + variableDisplayName
+ + ") due to lack of lon/lat comparison.")
+
+ # from this point on, we will be forking to create child processes so we can parallelize our image and
+ # report generation
+ isParent = True
+ childPids = [ ]
+
+ # generate our plotting functions
+ plottingFunctions = plottingFunctionGenerationFunction (aData, bData,
+ absDiffData, rawDiffData,
+ variableDisplayName,
+ epsilon,
+ goodInAMask, goodInBMask, goodMask,
+ troubleMask, outsideEpsilonMask,
+ # below this comment are parameters only needed
+ # for geolocated images
+ lonLatDataDict,
+ shouldUseSharedRangeForOriginal,
+ dataRanges, dataRangeNames, dataColors)
+
# only plot the two original data plots if they haven't been turned off
- if (not ('do_plot_originals' in variableRunInfo)) or variableRunInfo['do_plot_originals'] :
-
- # the original A data
- LOG.info("\t\tcreating image of " + variableDisplayName + " in file a")
- pid = _handle_fig_creation_task((lambda : _create_mapped_figure(aData, latitudeAData, longitudeAData,
- baseMapInstance, fullAxis,
- (variableDisplayName + "\nin File A"),
- invalidMask=(~goodInAMask),
- dataRanges=dataRanges or shared_range,
- dataRangeNames=dataRangeNames,
- dataRangeColors=dataColors)),
- "\t\tsaving image of " + variableDisplayName + " for file a",
- outputPath, variableName + ".A.png",
- makeSmall, doFork or shouldClearMemoryWithThreads)
- if not (pid is 0) :
- if doFork :
- childPids.append(pid)
- LOG.debug ("Started child process (pid: " + str(pid) + ") to create file a image for " + variableDisplayName)
- else :
- os.waitpid(pid, 0)
- original_images.append(variableName + ".A.png")
+ if doPlotOriginal :
+
+ _log_spawn_and_wait_if_needed(variableDisplayName + " in file a", childPids, original_images,
+ plottingFunctions['originalA'],
+ outputPath, "A.png",
+ makeSmall, doFork, shouldClearMemoryWithThreads)
+
+ _log_spawn_and_wait_if_needed(variableDisplayName + " in file b", childPids, original_images,
+ plottingFunctions['originalB'],
+ outputPath, "B.png",
+ makeSmall, doFork, shouldClearMemoryWithThreads)
- # the original B data
- LOG.info("\t\tcreating image of " + variableDisplayName + " in file b")
- pid = _handle_fig_creation_task((lambda : _create_mapped_figure(bData, latitudeBData, longitudeBData,
- baseMapInstance, fullAxis,
- (variableDisplayName + "\nin File B"),
- invalidMask=(~ goodInBMask),
- dataRanges=dataRanges or shared_range,
- dataRangeNames=dataRangeNames,
- dataRangeColors=dataColors)),
- "\t\tsaving image of " + variableDisplayName + " for file b",
- outputPath, variableName + ".B.png",
- makeSmall, doFork or shouldClearMemoryWithThreads)
- if not (pid is 0) :
- if doFork:
- childPids.append(pid)
- LOG.debug ("Started child process (pid: " + str(pid) + ") to create file b image for " + variableDisplayName)
- else :
- os.waitpid(pid, 0)
- original_images.append(variableName + ".B.png")
# this is the end of the if to plot the original data
# make the data comparison figures
if not shortCircuitComparisons :
# only plot the absolute difference if if it hasn't been turned off
- if (not ('do_plot_abs_diff' in variableRunInfo)) or (variableRunInfo['do_plot_abs_diff']) :
+ if doPlotAbsDiff :
- # the distance between the two data sets
- LOG.info("\t\tcreating image of the absolute value of difference in " + variableDisplayName)
- pid = _handle_fig_creation_task((lambda : _create_mapped_figure(absDiffData,
- latitudeCommonData, longitudeCommonData,
- baseMapInstance, fullAxis,
- ("Absolute value of difference in\n" + variableDisplayName),
- invalidMask=(~ goodMask))),
- "\t\tsaving image of the absolute value of difference for " + variableDisplayName,
- outputPath, variableName + ".AbsDiff.png",
- makeSmall, doFork or shouldClearMemoryWithThreads)
- if not (pid is 0) :
- if doFork :
- childPids.append(pid)
- LOG.debug ("Started child process (pid: " + str(pid)
- + ") to create absolute value of difference image for " + variableDisplayName)
- else :
- os.waitpid(pid, 0)
- compared_images.append(variableName + ".AbsDiff.png")
+ _log_spawn_and_wait_if_needed("absolute value of difference in " + variableDisplayName, childPids, compared_images,
+ plottingFunctions['absDiff'],
+ outputPath, "AbsDiff.png",
+ makeSmall, doFork, shouldClearMemoryWithThreads)
# only plot the difference plot if it hasn't been turned off
- if (not ('do_plot_sub_diff' in variableRunInfo)) or (variableRunInfo['do_plot_sub_diff']) :
+ if doPlotSubDiff :
- # the subtraction of one data set from the other
- LOG.info("\t\tcreating image of the difference in " + variableDisplayName)
- pid = _handle_fig_creation_task((lambda : _create_mapped_figure(rawDiffData, latitudeCommonData, longitudeCommonData,
- baseMapInstance, fullAxis,
- ("Value of (Data File B - Data File A) for\n" + variableDisplayName),
- invalidMask=(~ goodMask))),
- "\t\tsaving image of the difference in " + variableDisplayName,
- outputPath, variableName + ".Diff.png",
- makeSmall, doFork or shouldClearMemoryWithThreads)
- if not (pid is 0) :
- if doFork :
- childPids.append(pid)
- LOG.debug ("Started child process (pid: " + str(pid) + ") to create difference image for " + variableDisplayName)
- else :
- os.waitpid(pid, 0)
- compared_images.append(variableName + ".Diff.png")
+ _log_spawn_and_wait_if_needed("the difference in " + variableDisplayName, childPids, compared_images,
+ plottingFunctions['subDiff'],
+ outputPath, "Diff.png",
+ makeSmall, doFork, shouldClearMemoryWithThreads)
# only plot the trouble plot if it hasn't been turned off
- if (not ('do_plot_trouble' in variableRunInfo)) or (variableRunInfo['do_plot_trouble']) :
+ if doPlotTrouble :
- # mark the trouble points
- LOG.info("\t\tcreating image marking trouble data in " + variableDisplayName)
- # this is not an optimal solution, but we need to have at least somewhat valid data at any mismatched points so
- # that our plot won't be totally destroyed by missing or non-finite data from B
- bDataCopy = bData[:]
- tempMask = goodInAMask & (~goodInBMask)
- bDataCopy[tempMask] = aData[tempMask]
- pid = _handle_fig_creation_task((lambda : _create_mapped_figure(bDataCopy, latitudeCommonData, longitudeCommonData,
- baseMapInstance, fullAxis,
- ("Areas of trouble data in\n" + variableDisplayName),
- (~(goodInAMask | goodInBMask)),
- mediumGrayColorMap, troubleMask,
- dataRanges=dataRanges,
- dataRangeNames=dataRangeNames)),
- "\t\tsaving image marking trouble data in " + variableDisplayName,
- outputPath, variableName + ".Trouble.png",
- makeSmall, doFork or shouldClearMemoryWithThreads)
- if not (pid is 0) :
- if doFork :
- childPids.append(pid)
- LOG.debug ("Started child process (pid: " + str(pid) + ") to create trouble image for " + variableDisplayName)
- else :
- os.waitpid(pid, 0)
- compared_images.append(variableName + ".Trouble.png")
+ _log_spawn_and_wait_if_needed("trouble data in " + variableDisplayName, childPids, compared_images,
+ plottingFunctions['trouble'],
+ outputPath, "Trouble.png",
+ makeSmall, doFork, shouldClearMemoryWithThreads)
# only plot the histogram if it hasn't been turned off
- if (not ('do_plot_histogram' in variableRunInfo)) or (variableRunInfo['do_plot_histogram']) :
+ if doPlotHistogram :
- # a histogram of the values of fileA - file B
- LOG.info("\t\tcreating histogram of the amount of difference in " + variableDisplayName)
- numBinsToUse = 50
- valuesForHist = rawDiffData[goodMask]
- pid = _handle_fig_creation_task((lambda : _create_histogram(valuesForHist, numBinsToUse,
- ("Difference in\n" + variableDisplayName),
- ('Value of (Data File B - Data File A) at a Data Point'),
- ('Number of Data Points with a Given Difference'),
- True)),
- "\t\tsaving histogram of the amount of difference in " + variableDisplayName,
- outputPath, variableName + ".Hist.png",
- makeSmall, doFork or shouldClearMemoryWithThreads)
- if not (pid is 0) :
- if doFork :
- childPids.append(pid)
- LOG.debug ("Started child process (pid: " + str(pid)
- + ") to create difference histogram image for " + variableDisplayName)
- else :
- os.waitpid(pid, 0)
- compared_images.append(variableName + ".Hist.png")
+ _log_spawn_and_wait_if_needed("histogram of the amount of difference in " + variableDisplayName, childPids, compared_images,
+ plottingFunctions['histogram'],
+ outputPath, "Hist.png",
+ makeSmall, doFork, shouldClearMemoryWithThreads)
# only plot the scatter plot if it hasn't been turned off
- if (not ('do_plot_scatter' in variableRunInfo)) or (variableRunInfo['do_plot_scatter']) :
+ if doPlotScatter :
# scatter plot of file a vs file b values
- LOG.info("\t\tcreating scatter plot of file a values vs file b values for " + variableDisplayName)
- pid = _handle_fig_creation_task((lambda : _create_scatter_plot(aData[goodMask], bData[goodMask],
- "Value in File A vs Value in File B",
- "File A Value", "File B Value",
- outsideEpsilonMask[goodMask],
- variableRunInfo['epsilon'])),
- "\t\tsaving scatter plot of file a values vs file b values in " + variableDisplayName,
- outputPath, variableName + ".Scatter.png",
- makeSmall, doFork or shouldClearMemoryWithThreads)
- if not (pid is 0) :
- if doFork :
- childPids.append(pid)
- LOG.debug ("Started child process (pid: " + str(pid) + ") to create scatter plot image for " + variableDisplayName)
- else :
- os.waitpid(pid, 0)
- compared_images.append(variableName + ".Scatter.png")
+
+ _log_spawn_and_wait_if_needed("scatter plot of file a values vs file b values for " + variableDisplayName,
+ childPids, compared_images,
+ plottingFunctions['scatter'],
+ outputPath, "Scatter.png",
+ makeSmall, doFork, shouldClearMemoryWithThreads)
# now we need to wait for all of our child processes to terminate before returning
if (isParent) : # just in case
diff --git a/pyglance/glance/variablereport.txt b/pyglance/glance/variablereport.txt
index 29aa4dd68e61b90d116fe29587a1f3485639dd93..4032bc7c828cbe623b1895d6e7667739296e6cb3 100644
--- a/pyglance/glance/variablereport.txt
+++ b/pyglance/glance/variablereport.txt
@@ -57,70 +57,76 @@ Copyright (c) 2009 University of Wisconsin SSEC. All rights reserved.
</p>
% endif
- ## display the latitude and longitude variable names
- <p>
- % if ('latitude_alt_name_in_b' in runInfo) :
- latitude in A: ${runInfo['latitude']}<br>
- latitude in B: ${runInfo['latitude_alt_name_in_b']}<br>
- % else :
- latitude: ${runInfo['latitude']} <br>
- % endif
- % if ('longitude_alt_name_in_b' in runInfo) :
- longitude in A: ${runInfo['longitude']}<br>
- longitude in B: ${runInfo['longitude_alt_name_in_b']}<br>
- % else :
- longitude: ${runInfo['longitude']}<br>
- % endif
-
- % if ('lon_lat_epsilon' in runInfo) and (runInfo['lon_lat_epsilon'] > 0.0) :
- longitude/latitude comparison epsilon: ${runInfo['lon_lat_epsilon']}<br>
- % endif
+ ## if the lon/lat variables exist, display info on them
+ %if ('latitude' in runInfo) and ('longitude' in runInfo) :
- <br>
- ## display information about any data filtering on the lons/lats
- % if ('data_filter_function_lat_in_a' in runInfo) and (not (runInfo['data_filter_function_lat_in_a'] is None)) :
- Note: The latitude in file A was filtered.
- Please see <a href="../${runInfo['config_file_name']}">this copy of
- the original configuration file</a> to view the data filtering function.<br>
- % endif
- % if ('data_filter_function_lat_in_b' in runInfo) and (not (runInfo['data_filter_function_lat_in_b'] is None)) :
- Note: The latitude in file B was filtered.
- Please see <a href="../${runInfo['config_file_name']}">this copy of
- the original configuration file</a> to view the data filtering function.<br>
- % endif
- % if ('data_filter_function_lon_in_a' in runInfo) and (not (runInfo['data_filter_function_lon_in_a'] is None)) :
- Note: The longitude in file A was filtered.
- Please see <a href="../${runInfo['config_file_name']}">this copy of
- the original configuration file</a> to view the data filtering function.<br>
- % endif
- % if ('data_filter_function_lon_in_b' in runInfo) and (not (runInfo['data_filter_function_lon_in_b'] is None)) :
- Note: The longitude in file B was filtered.
- Please see <a href="../${runInfo['config_file_name']}">this copy of
- the original configuration file</a> to view the data filtering function.<br>
- % endif
- </p>
-
- ## if there is a problem with the longitude/latitude correlation between the two files,
- ## make a nice big warning for the user
- % if spatial.has_key('lon_lat_not_equal_points_count') and (spatial['lon_lat_not_equal_points_count'] > 0) :
+ ## display the latitude and longitude variable names
<p>
- WARNING: ${spatial['lon_lat_not_equal_points_count']} data points
- (${report.make_formatted_display_string(spatial['lon_lat_not_equal_points_percent'])}% of all data)
- show possible mismatch in values stored in file a
- and file b longitude and latitude values. Depending on the degree of mismatch, some data value comparisons
- in this report may be distorted or spatially nonsensical. Please consider re-running this report and including an
- examination of your longitude and latitude variables with appropriate epsilons in order to analyze the significance
- of the difference.<br>
- ## if we're showing images, link to graphs showing the problem
- % if runInfo['shouldIncludeImages'] :
- <a href="../LonLatMismatch.A.png">
- View mismatching points in A's lon/lat system
- </a><br>
- <a href="../LonLatMismatch.B.png">
- View mismatching points in B's lon/lat system
- </a>
+ % if ('latitude_alt_name_in_b' in runInfo) :
+ latitude in A: ${runInfo['latitude']}<br>
+ latitude in B: ${runInfo['latitude_alt_name_in_b']}<br>
+ % else :
+ latitude: ${runInfo['latitude']} <br>
+ % endif
+ % if ('longitude_alt_name_in_b' in runInfo) :
+ longitude in A: ${runInfo['longitude']}<br>
+ longitude in B: ${runInfo['longitude_alt_name_in_b']}<br>
+ % else :
+ longitude: ${runInfo['longitude']}<br>
+ % endif
+
+ % if ('lon_lat_epsilon' in runInfo) and (runInfo['lon_lat_epsilon'] > 0.0) :
+ longitude/latitude comparison epsilon: ${runInfo['lon_lat_epsilon']}<br>
+ % endif
+
+ <br>
+ ## display information about any data filtering on the lons/lats
+ % if ('data_filter_function_lat_in_a' in runInfo) and (not (runInfo['data_filter_function_lat_in_a'] is None)) :
+ Note: The latitude in file A was filtered.
+ Please see <a href="../${runInfo['config_file_name']}">this copy of
+ the original configuration file</a> to view the data filtering function.<br>
+ % endif
+ % if ('data_filter_function_lat_in_b' in runInfo) and (not (runInfo['data_filter_function_lat_in_b'] is None)) :
+ Note: The latitude in file B was filtered.
+ Please see <a href="../${runInfo['config_file_name']}">this copy of
+ the original configuration file</a> to view the data filtering function.<br>
+ % endif
+ % if ('data_filter_function_lon_in_a' in runInfo) and (not (runInfo['data_filter_function_lon_in_a'] is None)) :
+ Note: The longitude in file A was filtered.
+ Please see <a href="../${runInfo['config_file_name']}">this copy of
+ the original configuration file</a> to view the data filtering function.<br>
+ % endif
+ % if ('data_filter_function_lon_in_b' in runInfo) and (not (runInfo['data_filter_function_lon_in_b'] is None)) :
+ Note: The longitude in file B was filtered.
+ Please see <a href="../${runInfo['config_file_name']}">this copy of
+ the original configuration file</a> to view the data filtering function.<br>
% endif
</p>
+
+ ## if there is a problem with the longitude/latitude correlation between the two files,
+ ## make a nice big warning for the user
+ % if spatial.has_key('lon_lat_not_equal_points_count') and (spatial['lon_lat_not_equal_points_count'] > 0) :
+ <p>
+ WARNING: ${spatial['lon_lat_not_equal_points_count']} data points
+ (${report.make_formatted_display_string(spatial['lon_lat_not_equal_points_percent'])}% of all data)
+ show possible mismatch in values stored in file a
+ and file b longitude and latitude values. Depending on the degree of mismatch, some data value comparisons
+ in this report may be distorted or spatially nonsensical. Please consider re-running this report and including an
+ examination of your longitude and latitude variables with appropriate epsilons in order to analyze the significance
+ of the difference.<br>
+ ## if we're showing images, link to graphs showing the problem
+ % if runInfo['shouldIncludeImages'] :
+ <a href="../LonLatMismatch.A.png">
+ View mismatching points in A's lon/lat system
+ </a><br>
+ <a href="../LonLatMismatch.B.png">
+ View mismatching points in B's lon/lat system
+ </a>
+ % endif
+ </p>
+ % endif
+
+ ## end of the if to display lon/lat info
% endif
% if (len(imageNames['original']) > 0) :