diff --git a/pyglance/glance/compare.py b/pyglance/glance/compare.py index 5029a94c131df4c47b252c2a735c1c30f5d4ae42..6504dfb5ec242a88da3af27f6f7b8e7dbe1188e6 100644 --- a/pyglance/glance/compare.py +++ b/pyglance/glance/compare.py @@ -1161,10 +1161,10 @@ def reportGen_raw_data_simple_call (aData, bData, variableDisplayName, variableSettings['doc_path'] = quote(os.path.join(outputDirectory, './' + 'doc.html')) # calculate the variable statistics - variable_stats = statistics.StatisticalAnalysis(aData, bData, - missingValue, missingValue, - None, None, - epsilon, None).dictionary_form() + variable_stats = statistics.StatisticalAnalysis.withSimpleData(aData, bData, + missingValue, missingValue, + None, None, + epsilon, None).dictionary_form() # add a little additional info variableSettings['time'] = datetime.datetime.ctime(datetime.datetime.now()) @@ -1417,10 +1417,10 @@ def reportGen_library_call (a_path, b_path, var_list=[ ], if not do_not_test_with_lon_lat : mask_a_to_use = lon_lat_data['a']['inv_mask'] mask_b_to_use = lon_lat_data['b']['inv_mask'] - variable_stats = statistics.StatisticalAnalysis(aData, bData, - varRunInfo['missing_value'], varRunInfo['missing_value_alt_in_b'], - mask_a_to_use, mask_b_to_use, - varRunInfo['epsilon'], varRunInfo['epsilon_percent']).dictionary_form() + variable_stats = statistics.StatisticalAnalysis.withSimpleData(aData, bData, + varRunInfo['missing_value'], varRunInfo['missing_value_alt_in_b'], + mask_a_to_use, mask_b_to_use, + varRunInfo['epsilon'], varRunInfo['epsilon_percent']).dictionary_form() # add a little additional info to our variable run info before we squirrel it away varRunInfo['time'] = datetime.datetime.ctime(datetime.datetime.now()) # todo is this needed? @@ -1637,7 +1637,7 @@ def stats_library_call(afn, bfn, var_list=[ ], print >> output_channel, '-'*32 print >> output_channel, name print >> output_channel, '' - variable_stats = statistics.StatisticalAnalysis(aData, bData, amiss, bmiss, epsilon=epsilon) + variable_stats = statistics.StatisticalAnalysis.withSimpleData(aData, bData, amiss, bmiss, epsilon=epsilon) # if we're doing pass/fail testing, do that now if do_pass_fail : didPass, _, _, _ =_check_pass_or_fail(defaultVariablePassFailSettings, diff --git a/pyglance/glance/data.py b/pyglance/glance/data.py index 4f9f0adca19830088378e280b0e89f851ae95837..b7c179389d456475a66ac626442a6bb8f26facb6 100644 --- a/pyglance/glance/data.py +++ b/pyglance/glance/data.py @@ -89,9 +89,14 @@ class DataObject (object) : data - the raw array of data (generally this should be a numpy array) fill_value - the fill value used in the data array masks - the set of masks that apply to this data + + override_fill_value - should the fill_value be used rather than the default_fill_value + (this defaults to True so the fill_value is used, insuring backwards compatability) + default_fill_value - the default fill value that will be used if override_fill_value is False """ - def __init__(self, dataArray, fillValue=None, ignoreMask=None) : + def __init__(self, dataArray, fillValue=None, ignoreMask=None, + overrideFillValue=True, defaultFillValue=None) : """ Create the data object. @@ -103,6 +108,9 @@ class DataObject (object) : self.data = dataArray self.fill_value = fillValue self.masks = BasicMaskSetObject(ignoreMask) + + self.override_fill_value = overrideFillValue + self.default_fill_value = defaultFillValue def self_analysis(self) : """ @@ -122,9 +130,10 @@ class DataObject (object) : # find and mark the missing values missing_mask = np.zeros(shape, dtype=np.bool) # if the data has a fill value, mark where the missing data is - if self.fill_value is not None : - missing_mask[self.data == self.fill_value] = True - missing_mask[self.masks.ignore_mask] = False + tempFillValue = self.select_fill_value() + if tempFillValue is not None : + missing_mask[self.data == tempFillValue] = True + missing_mask[self.masks.ignore_mask] = False # define the valid mask as places where the data is not missing, # nonfinite, or ignored @@ -133,6 +142,14 @@ class DataObject (object) : # set our masks self.masks = BasicMaskSetObject(self.masks.ignore_mask, valid_mask, non_finite_mask, missing_mask) + + def select_fill_value (self) : + """ + choose the fill value to use + """ + toReturn = self.fill_value if self.override_fill_value else self.default_fill_value + + return toReturn class DiffInfoObject (object) : """ @@ -248,8 +265,8 @@ class DiffInfoObject (object) : # get our shared data type and fill value sharedType, fill_data_value = DiffInfoObject._get_shared_type_and_fill_value(aDataObject.data, bDataObject.data, - aDataObject.fill_value, - bDataObject.fill_value) + aDataObject.select_fill_value(), + bDataObject.select_fill_value()) # construct our diff'ed data set raw_diff = np.zeros(shape, dtype=sharedType) diff --git a/pyglance/glance/gui_controller.py b/pyglance/glance/gui_controller.py index 2ccd7317c4ab71dccefe835c568f19e3b593b19b..d123eef0a8c0e06abcd0208b1af74f92978e4870 100644 --- a/pyglance/glance/gui_controller.py +++ b/pyglance/glance/gui_controller.py @@ -107,6 +107,13 @@ class GlanceGUIController (object) : self.model.updateSettingsDataSelection(newImageType=new_image_type) + def userRequestsStats (self) : + """ + the user has asked for stats information + """ + + self.model.sendStatsInfo() # TODO, should a different object handle this? + def userRequestsPlot (self) : """ the user has asked for a plot diff --git a/pyglance/glance/gui_model.py b/pyglance/glance/gui_model.py index 15d5a2c50963cb68b05ec58537d4f3451f2df0da..512eb81024df80f8a29f8558de6365f81d670728 100644 --- a/pyglance/glance/gui_model.py +++ b/pyglance/glance/gui_model.py @@ -23,6 +23,7 @@ import numpy as np import glance.data as dataobjects import glance.figures as figures import glance.io as io +import glance.stats as stats LOG = logging.getLogger(__name__) @@ -68,18 +69,15 @@ class _FileModelData (object) : self.file - the FileInfo object representing this file, can be used to load more information later self.variable - the name of the selected variable - self.doOverride - false if the default fill value should be used, true otherwise - self.fillValue - the fill value (may be different than the default), should be used when override is true - self.defaultFillValue - the default fill value, should be used when override is false + self.var_data_cache - a cache of all the variable data that has been loaded for this file + (stored in dataobjects.DataObject objects), keyed by variable name + self.var_attrs_cache - a cache of variable attributes (keyed by variable name), each set of attributes is a dictionary, + keyed with the attribute names and containing their values self.ALL_VARIABLES - a list of all the variable names in the file - self.var_data - the data contained in the currently selected variable (or None if no variable is selected) - self.var_attrs - a dictionary of the variable attributes, keyed with the attribute names and containing their values - - TODO, eventually replace this with a better data object and/or some combination of data and settings objects? """ def __init__(self, file_object=None, variable_selection=None, do_override=False, fill_value=None, default_fill_value=None, - variables_list=None, variable_data=None, variable_attributes={ }) : + variables_list=None, variable_data=None, variable_attributes=None) : """ create a set of model data, using the data passed in @@ -87,14 +85,16 @@ class _FileModelData (object) : """ self.file = file_object - self.variable = variable_selection - self.doOverride = do_override - self.fillValue = fill_value - self.defaultFillValue = default_fill_value + self.variable = str(variable_selection) self.ALL_VARIABLES = variables_list - self.var_data = variable_data - self.var_attrs = variable_attributes + self.var_data_cache = { } + self.var_attrs_cache = { } + if variable_selection is not None : + self.var_data_cache[variable_selection] = dataobjects.DataObject(variable_data, fillValue= fill_value, + overrideFillValue=do_override, + defaultFillValue=default_fill_value) + self.var_attrs_cache[variable_selection] = variable_attributes class GlanceGUIModel (object) : """ @@ -156,30 +156,51 @@ class GlanceGUIModel (object) : if newFile is None : return + # reset our caches + self._resetCaches(filePrefix) + # get the list of variables, and pick one variableList = sorted(newFile.file_object()) # gets a list of all the variables in the file - tempVariable = variableList[0] - fillValue = newFile.file_object.missing_value(tempVariable) + tempVariable = str(variableList[0]) # save all of the data related to this file for later use self.fileData[filePrefix].file = newFile self.fileData[filePrefix].variable = tempVariable - self.fileData[filePrefix].doOverride = False - self.fileData[filePrefix].defaultFillValue = fillValue - self.fileData[filePrefix].fillValue = fillValue self.fileData[filePrefix].ALL_VARIABLES = variableList - # get the size of the currently selected variable TODO, is it possible to do this without loading the variable? - tempShape = self._load_variable_data(filePrefix, str(self.fileData[filePrefix].variable)) + # load info on the current variable + tempDataObj = self._load_variable_data(filePrefix, tempVariable) - # get the variable's attributes TODO, does this work on all types of files? (FIXME no, make a general method in io!) - self.fileData[filePrefix].var_attrs = newFile.file_object.get_variable_object(str(tempVariable)).attributes() + # get the variable's attributes + tempAttrs = self._load_variable_attributes (filePrefix, tempVariable) # Now tell our data listeners that the file data changed for dataListener in self.dataListeners : LOG.debug("Sending update for file " + filePrefix + " with loaded data.") - dataListener.fileDataUpdate(filePrefix, newFile.path, tempVariable, False, fillValue, str(tempShape), - variable_list=variableList, attribute_list=self.fileData[filePrefix].var_attrs) + dataListener.fileDataUpdate(filePrefix, newFile.path, tempVariable, tempDataObj.override_fill_value, + self._select_fill_value(filePrefix), str(tempDataObj.data.shape), + variable_list=variableList, attribute_list=tempAttrs) + + def _load_variable_attributes (self, file_prefix, variable_name) : + """ + Load the attributes for for a given file name, saving them to the + file data structure as appropriate + + return the loaded attributes for convenience + """ + + variable_name = str(variable_name) + + tempAttrs = None + # if we can load the attributes from the cache, do that otherwise get them from the file + if variable_name in self.fileData[file_prefix].var_attrs_cache.keys() : + tempAttrs = self.fileData[file_prefix].var_attrs_cache[variable_name] + else : + tempAttrs = self.fileData[file_prefix].file.file_object.get_variable_attributes(variable_name) + # cache these for later use + self.fileData[file_prefix].var_attrs_cache[variable_name] = tempAttrs + + return tempAttrs def _load_variable_data (self, file_prefix, variable_name) : """ @@ -189,8 +210,32 @@ class GlanceGUIModel (object) : TODO, can this be handled as a background task in the future? """ - self.fileData[file_prefix].var_data = self.fileData[file_prefix].file.file_object[variable_name] - return self.fileData[file_prefix].var_data.shape + variable_name = str(variable_name) + + tempData = None + # if we have a cached version of this variable, use that, otherwise, load it from the file + if variable_name in self.fileData[file_prefix].var_data_cache.keys() : + LOG.debug ("Loading " + str(file_prefix) + " file cached variable: " + str(variable_name)) + tempData = self.fileData[file_prefix].var_data_cache[variable_name] + else : + LOG.debug ("Loading " + str(file_prefix) + " file variable from file: " + str(variable_name)) + tempRawData = self.fileData[file_prefix].file.file_object[variable_name] + tempFillVal = self.fileData[file_prefix].file.file_object.missing_value(variable_name) + tempOverride = False + # also save this new data in our cache TODO, this won't save the other data we need? + tempData = dataobjects.DataObject(tempRawData, fillValue=tempFillVal, + overrideFillValue=tempOverride, + defaultFillValue=tempFillVal) + self.fileData[file_prefix].var_data_cache[variable_name] = tempData + + return tempData + + def _resetCaches (self, file_prefix) : + """ + Clear the two internal caches + """ + self.fileData[file_prefix].var_data_cache = { } + self.fileData[file_prefix].var_attrs_cache = { } def sendGeneralSettingsData (self) : """ @@ -215,44 +260,44 @@ class GlanceGUIModel (object) : if (newVariableText is not None) and (newVariableText != self.fileData[file_prefix].variable) : if newVariableText in self.fileData[file_prefix].ALL_VARIABLES : LOG.debug("Setting file " + file_prefix + " variable selection to: " + newVariableText) - self.fileData[file_prefix].variable = newVariableText + self.fileData[file_prefix].variable = str(newVariableText) didUpdate = True # load the data for this new variable self._load_variable_data(file_prefix, str(newVariableText)) - # get the variable's attributes TODO, does this work on all types of files? (FIXME nope, make a general method in io!) - self.fileData[file_prefix].var_attrs = self.fileData[file_prefix].file.file_object.get_variable_object(str(newVariableText)).attributes() - - # the new fill value should be loaded and the override should be cleared - self.fileData[file_prefix].doOverride = False - self.fileData[file_prefix].defaultFillValue = self.fileData[file_prefix].file.file_object.missing_value(str(newVariableText)) - self.fileData[file_prefix].fillValue = self.fileData[file_prefix].defaultFillValue + # get the variable's attributes + self._load_variable_attributes (file_prefix, str(newVariableText)) + + # for convenience hang on to this + tempVariableName = self.fileData[file_prefix].variable # update the override selection if needed - if newOverrideValue is not None : + if (newOverrideValue is not None) and (tempVariableName in self.fileData[file_prefix].var_data_cache.keys()) : LOG.debug("Setting file " + file_prefix + " override selection to: " + str(newOverrideValue)) - self.fileData[file_prefix].doOverride = newOverrideValue + self.fileData[file_prefix].var_data_cache[self.fileData[file_prefix].variable].override_fill_value = newOverrideValue didUpdate = True # update the fill value if needed - if newFillValue is not np.nan : + if (newFillValue is not np.nan) and (tempVariableName in self.fileData[file_prefix].var_data_cache.keys()) : LOG.debug("Setting file " + file_prefix + " fill value to: " + str(newFillValue)) - self.fileData[file_prefix].fillValue = newFillValue + self.fileData[file_prefix].var_data_cache[self.fileData[file_prefix].variable].fill_value = newFillValue didUpdate = True # let our data listeners know about any changes if didUpdate : + tempDataObject = self.fileData[file_prefix].var_data_cache[tempVariableName] + tempAttrsList = self.fileData[file_prefix].var_attrs_cache[tempVariableName] for listener in self.dataListeners : - listener.fileDataUpdate(file_prefix, self.fileData[file_prefix].file.path, self.fileData[file_prefix].variable, - self.fileData[file_prefix].doOverride, self._select_fill_value(file_prefix), - str(self.fileData[file_prefix].var_data.shape), attribute_list=self.fileData[file_prefix].var_attrs) + listener.fileDataUpdate(file_prefix, self.fileData[file_prefix].file.path, tempVariableName, + tempDataObject.override_fill_value, self._select_fill_value(file_prefix), + str(tempDataObject.data.shape), attribute_list=tempAttrsList) def _select_fill_value (self, file_prefix) : """ which fill value should currently be used? """ - return self.fileData[file_prefix].fillValue if self.fileData[file_prefix].doOverride else self.fileData[file_prefix].defaultFillValue + return self.fileData[file_prefix].var_data_cache[self.fileData[file_prefix].variable].select_fill_value() def updateSettingsDataSelection (self, newEpsilonValue=np.nan, newImageType=None) : """ @@ -282,6 +327,43 @@ class GlanceGUIModel (object) : listener.updateEpsilon(self.epsilon) listener.updateImageTypes(self.imageType) + def sendStatsInfo (self) : + """ + our data listeners should be sent statistics information for a comparison + of the currently selected variables (if possible) + """ + + tempVarA = self.fileData["A"].variable + tempVarB = self.fileData["B"].variable + aData = self.fileData["A"].var_data_cache[tempVarA] if tempVarA in self.fileData["A"].var_data_cache else None + bData = self.fileData["B"].var_data_cache[tempVarB] if tempVarB in self.fileData["B"].var_data_cache else None + + # check the minimum validity of our data + message = None + if (aData is None) or (bData is None) : + message = ("Data for requested files was not available. " + + "Please load or reload files and try again.") + # check to see if the two variables have the same shape of data + elif aData.data.shape != bData.data.shape : + message = (tempVarA + ' / ' + tempVarB + ' ' + + 'could not be compared because the data for these variables does not match in shape ' + + 'between the two files (file A data shape: ' + str(aData.data.shape) + '; file B data shape: ' + + str(bData.data.shape) + ').') + # if the data isn't valid, stop now + if message is not None : + for errorHandler in self.errorHandlers : + errorHandler.handleWarning(message) + # we can't make any stats from this data, so just return + return + + LOG.info ("Constructing statistics") + + tempAnalysis = stats.StatisticalAnalysis.withDataObjects(aData, bData, epsilon=self.epsilon) + + # tell my listeners to show the stats data we've collected + for listener in self.dataListeners : + listener.displayStatsData(tempVarA, tempVarB, tempAnalysis) # TODO, do we need a different set of data here? + def spawnPlotWithCurrentInfo (self) : """ create a matplotlib plot using the current model information @@ -289,10 +371,17 @@ class GlanceGUIModel (object) : TODO, move this into some sort of figure manager model object/module? """ + LOG.debug ("Variable A cache entries: " + str(self.fileData["A"].var_data_cache.keys())) + LOG.debug ("Variable B cache entries: " + str(self.fileData["B"].var_data_cache.keys())) + LOG.info ("Preparing variable data for plotting...") - aData = self.fileData["A"].var_data - bData = self.fileData["B"].var_data + tempVarA = self.fileData["A"].variable + tempVarB = self.fileData["B"].variable + + # TODO, move to taking advantage of the whole data objects + aData = self.fileData["A"].var_data_cache[tempVarA].data if tempVarA in self.fileData["A"].var_data_cache else None + bData = self.fileData["B"].var_data_cache[tempVarB].data if tempVarB in self.fileData["B"].var_data_cache else None message = None @@ -303,7 +392,7 @@ class GlanceGUIModel (object) : "Please load or reload files and try again.") # check to see if the two variables have the same shape of data elif aData.shape != bData.shape : - message = (self.fileData["A"].variable + ' / ' + self.fileData["B"].variable + ' ' + + message = (tempVarA + ' / ' + tempVarB + ' ' + 'could not be compared because the data for these variables does not match in shape ' + 'between the two files (file A data shape: ' + str(aData.shape) + '; file B data shape: ' + str(bData.shape) + ').') @@ -328,8 +417,8 @@ class GlanceGUIModel (object) : rawDiffDataClean = bDataClean - aDataClean # pull the units information - aUnits = self.fileData["A"].file.file_object.get_attribute(str(self.fileData["A"].variable), io.UNITS_CONSTANT) - bUnits = self.fileData["B"].file.file_object.get_attribute(str(self.fileData["B"].variable), io.UNITS_CONSTANT) + aUnits = self.fileData["A"].file.file_object.get_attribute(self.fileData["A"].variable, io.UNITS_CONSTANT) + bUnits = self.fileData["B"].file.file_object.get_attribute(self.fileData["B"].variable, io.UNITS_CONSTANT) LOG.info("Spawning plot window: " + self.imageType) diff --git a/pyglance/glance/gui_view.py b/pyglance/glance/gui_view.py index d30fda869568bbf1068a7b4681c9ee2c236f585c..331da79e64a6ac98f1ca75f95899a42f59d889c4 100644 --- a/pyglance/glance/gui_view.py +++ b/pyglance/glance/gui_view.py @@ -74,6 +74,10 @@ class GlanceGUIView (QtGui.QWidget) : # we will use this to remember were the user wanted to load files from last # TODO, can we remember this between program runs? something like preferences? self.lastFilePath = './' + + # hang on to stats windows so they don't vanish + self.statsWindows = { } + self.statsCounter = 1 def _add_file_related_controls (self, file_prefix, grid_layout, currentRow) : """ @@ -155,6 +159,9 @@ class GlanceGUIView (QtGui.QWidget) : # now set up the input of the fill value that will be used grid_layout.addWidget(QtGui.QLabel("fill value:"), currentRow+1, 1) fillValue = QtGui.QLineEdit() + tempValidator = QtGui.QDoubleValidator(fillValue) + tempValidator.setNotation(QtGui.QDoubleValidator.StandardNotation) + fillValue.setValidator(tempValidator) fillValue.setDisabled(True) if file_prefix is "A" : fillValue.editingFinished.connect(self.fillValueEditedA) @@ -185,6 +192,10 @@ class GlanceGUIView (QtGui.QWidget) : # set up the epsilon input box layoutToUse.addWidget(QtGui.QLabel("epsilon:"), currentRow, 0) self.epsilonWidget = QtGui.QLineEdit() + tempValidator = QtGui.QDoubleValidator(self.epsilonWidget) + tempValidator.setBottom(0.0) # only accept positive epsilons + tempValidator.setNotation(QtGui.QDoubleValidator.StandardNotation) + self.epsilonWidget.setValidator(tempValidator) self.epsilonWidget.editingFinished.connect(self.reportEpsilonChanged) layoutToUse.addWidget(self.epsilonWidget, currentRow, 1, 1, 2) @@ -200,7 +211,10 @@ class GlanceGUIView (QtGui.QWidget) : # TODO should I add a drop down to select how to visualize the data? (ie 2D, line, on a map, etc?) - # TODO add a button that shows stats + # set up a button that shows stats + self.statsButton = QtGui.QPushButton("Display Statistics") + self.statsButton.clicked.connect(self.reportDisplayStatsClicked) + layoutToUse.addWidget(self.statsButton, currentRow, 1, 1, 2) # set up the button at the bottom that creates plots self.displayButton = QtGui.QPushButton("Display Plot") @@ -295,6 +309,9 @@ class GlanceGUIView (QtGui.QWidget) : """ newFillValue = self.widgetInfo[file_prefix]['fillValue'].text() + # it's still possible for this to not be a number, so fix that + newFillValue = self._extra_number_validation(newFillValue) + self.widgetInfo[file_prefix]['fillValue'].setText(str(newFillValue)) # let our user update listeners know the fill value changed for listener in self.userUpdateListeners : @@ -306,6 +323,9 @@ class GlanceGUIView (QtGui.QWidget) : """ newEpsilon = self.epsilonWidget.text() + # it's still possible for epsilon to not be a number, so fix that + newEpsilon = self._extra_number_validation(newEpsilon) + self.epsilonWidget.setText(str(newEpsilon)) # let our user update listeners know the epsilon changed for listener in self.userUpdateListeners : @@ -322,6 +342,18 @@ class GlanceGUIView (QtGui.QWidget) : for listener in self.userUpdateListeners : listener.userSelectedImageType(newImageType) + def reportDisplayStatsClicked (self) : + """ + the user clicked the display stats button + """ + + # make sure the focus isn't in a line-edit box + self.statsButton.setFocus() + + # now report to our listeners that the user wants stats + for listener in self.userUpdateListeners : + listener.userRequestsStats() + def reportDisplayPlotClicked (self) : """ the user clicked the display plot button @@ -334,10 +366,42 @@ class GlanceGUIView (QtGui.QWidget) : for listener in self.userUpdateListeners : listener.userRequestsPlot() + def _extra_number_validation (self, string_that_should_be_a_number) : + """ + try to validate the string that should be a number + """ + + toReturn = None + + try : + toReturn = int(string_that_should_be_a_number) + except ValueError : + try : + toReturn = float(string_that_should_be_a_number) + except ValueError : + pass # in this case we can't convert it, so just toss it + + return toReturn + ################# end methods related to user input ################# ################# start data model update related methods ################# + def displayStatsData (self, aVariableName, bVariableName, statsAnalysis) : + """ + given the names of the two variables and the statistical analysis, + display this to the user + """ + + tempID = self.statsCounter + self.statsCounter = self.statsCounter + 1 + + # I don't like this solution, but it would allow me to show multiple sets of stats at a time + self.statsWindows[tempID] = StatisticsDisplayWindow(tempID, + aVariableName, variable_name_b=bVariableName, + statsTextToDisplay=str(statsAnalysis.dictionary_form()), stored_in=self.statsWindows) + #TODO, this is a terrible way to display this info, but shows that it is there + def fileDataUpdate (self, file_prefix, file_path, selected_variable, use_fill_override, new_fill_value, variable_dimensions, variable_list=None, attribute_list=None) : """ @@ -386,7 +450,7 @@ class GlanceGUIView (QtGui.QWidget) : update the comparison epsilon displayed to the user """ - self.epsilonWidget.setText(str(epsilon)) # TODO, this needs to be a float? + self.epsilonWidget.setText(str(epsilon)) def updateImageTypes (self, imageType, list=None) : @@ -423,6 +487,53 @@ class GlanceGUIView (QtGui.QWidget) : if objectToRegister not in self.userUpdateListeners : self.userUpdateListeners.append(objectToRegister) +class StatisticsDisplayWindow (QtGui.QWidget) : + """ + This class represents a window that displays a statistics comparison between two variables. + This window is intended to be displayed in a non-modal manner. + """ + + def __init__ (self, id_number, variable_name_a, variable_name_b=None, + statsTextToDisplay="", stored_in=None, parent=None) : + """ + set up a window to display stats + """ + + QtGui.QWidget.__init__(self, parent) + + self.id = id_number + self.stored = stored_in + + # build and set the window title + tempTitle = "Statistics Comparing " + str(variable_name_a) + if (variable_name_b is not None) and (variable_name_b != variable_name_a) : + tempTitle = tempTitle + " / " + str(variable_name_b) + tempTitle = tempTitle + " data" + self.setWindowTitle(tempTitle) + + # create the layout and set up some of the overall record keeping + layoutToUse = QtGui.QGridLayout() + + # set up the button at the bottom that creates plots + self.statsText = QtGui.QTextEdit(statsTextToDisplay) + layoutToUse.addWidget(self.statsText, 1, 1) + + # set up the overall window geometry + self.setLayout(layoutToUse) + self.setGeometry(400, 400, 500, 500) + + self.show() + + def closeEvent (self, event) : + """ + we need to clean some stuff up when the window wants to close + """ + + if self.stored is not None : + del self.stored[self.id] + + event.accept() + if __name__=='__main__': import doctest doctest.testmod() diff --git a/pyglance/glance/imapp_plot.py b/pyglance/glance/imapp_plot.py new file mode 100644 index 0000000000000000000000000000000000000000..4db9aa4d0a11f1f35b6b74ece76bf065cefe4dce --- /dev/null +++ b/pyglance/glance/imapp_plot.py @@ -0,0 +1,360 @@ +#!/usr/bin/env python +# encoding: utf-8 +""" +Plot IMAPP IDEA data. + +Created by evas Oct 2011. +Copyright (c) 2011 University of Wisconsin SSEC. All rights reserved. +""" + +import sys, os, logging + +# these first two lines must stay before the pylab import +import matplotlib +matplotlib.use('Agg') # use the Anti-Grain Geometry rendering engine + +from pylab import * + +import matplotlib.cm as cm +import matplotlib.pyplot as plt +import matplotlib.colors as colors +from mpl_toolkits.basemap import Basemap + +import numpy as np + +import glance.data as dataobj + +LOG = logging.getLogger(__name__) + +defaultValues = { + 'longitudeVar': 'xtraj', + 'latitudeVar': 'ytraj', + 'initAODVar': 'aod_traj', + 'trajPressVar': 'ptraj', + 'timeVar': 'time', + 'nePiece': 'NE_', + 'swPiece': 'SW_', + 'latPiece': 'LAT', + 'lonPiece': 'LON', + 'figureName': 'frame.png', + 'figureDPI': 200 + } + +# a custom colormap or the Trajectory Pressures +color_data = { + 'red' : ( (0.0, 1.0, 1.0), + (0.75, 1.0, 1.0), + (0.9, 1.0, 1.0), + (1.0, 0.0, 0.0) ), + 'green' : ( (0.0, 1.0, 1.0), + (0.75, 1.0, 1.0), + (0.9, 0.08, 0.08), + (1.0, 0.0, 0.0) ), + 'blue' : ( (0.0, 1.0, 1.0), + (0.75, 1.0, 1.0), + (0.9, 0.58, 0.58), + (1.0, 0.0, 0.0) ) + } +dark_trajectory_pressure_color_map = matplotlib.colors.LinearSegmentedColormap('darkTrajPressCM', color_data, 256) +color_data = { + 'red' : ( (0.0, 0.0, 0.0), + (0.75, 0.0, 0.0), + (0.9, 1.0, 1.0), + (1.0, 1.0, 1.0) ), + 'green' : ( (0.0, 0.0, 0.0), + (0.75, 0.0, 0.0), + (0.9, 0.08, 0.08), + (1.0, 1.0, 1.0) ), + 'blue' : ( (0.0, 0.0, 0.0), + (0.75, 0.0, 0.0), + (0.9, 0.58, 0.58), + (1.0, 1.0, 1.0) ) + } +light_trajectory_pressure_color_map = matplotlib.colors.LinearSegmentedColormap('lightTrajPressCM', color_data, 256) + +def _create_imapp_figure (initAODdata, initLongitudeData, initLatitudeData, + pressureData=None, pressLongitudeData=None, pressLatitudeData=None, + baseMapInstance=None, figureTitle="MODIS AOD & AOD Trajectories", + useDarkBackground=True, latRange=None, lonRange=None) : + """ + TODO, I'm still deciding how this funciton works so it will probably change drastically + """ + + # build the plot + figure = plt.figure() + tempColor = 'k' if useDarkBackground else 'w' # either use a black or white background + axes = figure.add_subplot(111, axisbg=tempColor) + + # build extra info to go to the map plotting function + kwargs = { } + + # draw the basic physical and geopolitical features + tempColor='w' if useDarkBackground else 'k' # either draw black or white lines + if baseMapInstance is not None : + baseMapInstance.drawcoastlines( color=tempColor, linewidth=0.5) + baseMapInstance.drawcountries( color=tempColor, linewidth=0.5) + baseMapInstance.drawstates( color=tempColor, linewidth=0.5) + baseMapInstance.drawmapboundary(color=tempColor, linewidth=0.5) + # draw the parallels and meridians + if latRange is not None : + parallels = arange(-80., 90., latRange / 4.0) + baseMapInstance.drawparallels(parallels,labels=[1,0,0,1], color=tempColor, linewidth=0.5) + if lonRange is not None : + meridians = arange(0., 360., lonRange / 4.0) + baseMapInstance.drawmeridians(meridians,labels=[1,0,0,1], color=tempColor, linewidth=0.5) + + # translate the longitude and latitude data sets into map coordinates + pressX, pressY = None, None + if pressureData is not None : + pressX, pressY = baseMapInstance(pressLongitudeData, pressLatitudeData) + initX, initY = baseMapInstance(initLongitudeData, initLatitudeData) + + + color_map_to_use = dark_trajectory_pressure_color_map if useDarkBackground else light_trajectory_pressure_color_map + # if we have pressure data plot that + if pressureData is not None : + # I'm taking advantage of the fact that I can remove the black edge line with lw=0 (line width = 0) and scale the marker down with s=0.5 + baseMapInstance.scatter(pressX, pressY, s=0.5, c=pressureData, marker='o', cmap=color_map_to_use, vmin=0, vmax=1000, lw=0) + + # create the pressure colorbar + cbar1 = colorbar(format='%.3g', orientation='horizontal', shrink=0.25) + cbar1.ax.set_position([0.1, -0.16, 0.25, 0.25]) + for tempText in cbar1.ax.get_xticklabels(): + #print(tempText) + tempText.set_fontsize(5) + cbar1.set_label("Trajectory Pressure (mb)") + + # plot the origin points after the pressure so they'll appear on top, I'm assuming we will always have this data + baseMapInstance.scatter(initX, initY, s=10, c=initAODdata, marker='o', cmap=cm.jet, vmin=0.0, vmax=1.0, lw=0.5) + + # make a color bar + cbar2 = colorbar(format='%.3g', orientation='horizontal', shrink=0.25) + cbar2.ax.set_position([0.4, -0.16, 0.25, 0.25]) + for tempText in cbar2.ax.get_xticklabels(): + #print(tempText) + tempText.set_fontsize(5) + cbar2.set_label("MODIS AOD") # TODO, how do I get a second Trajectory Pressure colorbar in the right place? + + # now that we've moved everything around, make sure our main image is in the right place + axes.set_position([0.1, 0.15, 0.8, 0.8]) # why was this method so hard to find? + + # set up the figure title + # TODO compose the figure title with time/date info? + axes.set_title(figureTitle) + + return figure + +def main(): + import optparse + usage = """ +%prog [options] +run "%prog help" to list commands +examples: + +python -m glance.imapp_plot plot A.nc + +""" + # the following represent options available to the user on the command line: + + parser = optparse.OptionParser(usage) + + # logging output options + parser.add_option('-q', '--quiet', dest="quiet", + action="store_true", default=False, help="only error output") + parser.add_option('-v', '--verbose', dest="verbose", + action="store_true", default=False, help="enable more informational output") + parser.add_option('-w', '--debug', dest="debug", + action="store_true", default=False, help="enable debug output") + + # file generation related options TODO, make this work + parser.add_option('-p', '--outpath', dest="outpath", type='string', default='./', + help="set path to output directory") + + # file variable settings + parser.add_option('-o', '--longitude', dest="longitudeVar", type='string', + help="set name of longitude variable") + parser.add_option('-a', '--latitude', dest="latitudeVar", type='string', + help="set name of latitude variable") + + # time related settings + parser.add_option('-s', '--startTime', dest="startTime", type='int', + default=0, help="set first time to process") + parser.add_option('-e', '--endTime', dest="endTime", type='int', + help="set last time to process") + parser.add_option('-f', '--futureWindow', dest="futureWindow", type='int', + default=6, help="set number of hours of future pressures to show") + + parser.add_option('-t', '--test', dest="self_test", + action="store_true", default=False, help="run internal unit tests") + + # TODO will add this once we're out of alpha + #parser.add_option('-n', '--version', dest='version', + # action="store_true", default=False, help="view the glance version") + + + # parse the uers options from the command line + options, args = parser.parse_args() + if options.self_test: + import doctest + doctest.testmod() + sys.exit(2) + + # set up the logging level based on the options the user selected on the command line + lvl = logging.WARNING + if options.debug: lvl = logging.DEBUG + elif options.verbose: lvl = logging.INFO + elif options.quiet: lvl = logging.ERROR + logging.basicConfig(level = lvl) + + # TODO display the version + #if options.version : + # print (_get_version_string() + '\n') + + commands = {} + prior = None + prior = dict(locals()) + + """ + The following functions represent available menu selections. + """ + + def plot(*args): + """plot trajectory frames + Given a file with trajectory possitions and pressures over time, plot out + images of these trajectories on the Earth and save it to disk. + """ + + LOG.debug("startTime: " + str(options.startTime)) + LOG.debug("endTime: " + str(options.endTime)) + LOG.debug("futureWindow: " + str(options.futureWindow)) + + # setup the output directory now + if not (os.path.isdir(options.outpath)) : + LOG.info("Specified output directory (" + options.outpath + ") does not exist.") + LOG.info("Creating output directory.") + os.makedirs(options.outpath) + + # open the file + LOG.info("Opening trajectory data file.") + trajectoryFilePath = args[0] + trajectoryFileObject = dataobj.FileInfo(trajectoryFilePath) + if trajectoryFileObject is None: + LOG.warn("Trajectory file (" + trajectoryFilePath + ") could not be opened.") + LOG.warn("Aborting attempt to plot trajectories.") + sys.exit(1) + + # load the required variables + # TODO, allow the user control over the names? + LOG.info("Loading variable data from trajectory data file.") + initialAODdata = trajectoryFileObject.file_object[defaultValues['initAODVar']] + trajectoryPressureData = trajectoryFileObject.file_object[defaultValues['trajPressVar']] + latitudeData = trajectoryFileObject.file_object[defaultValues['latitudeVar']] + longitudeData = trajectoryFileObject.file_object[defaultValues['longitudeVar']] + trajectoryTimeData = trajectoryFileObject.file_object[defaultValues['timeVar']] + + # get information on where we should display the data + northeastLon = trajectoryFileObject.file_object.get_global_attribute( defaultValues['nePiece'] + defaultValues['lonPiece'] ) + northeastLat = trajectoryFileObject.file_object.get_global_attribute( defaultValues['nePiece'] + defaultValues['latPiece'] ) + southwestLon = trajectoryFileObject.file_object.get_global_attribute( defaultValues['swPiece'] + defaultValues['lonPiece'] ) + southwestLat = trajectoryFileObject.file_object.get_global_attribute( defaultValues['swPiece'] + defaultValues['latPiece'] ) + latRange = abs(northeastLat - southwestLat) + lonRange = abs(southwestLon - northeastLon) + + # build a basemap + LOG.info("Building basemap object.") + projectionName = 'merc' # use the Mercator Projection + basemapObject = Basemap (projection=projectionName,llcrnrlat=southwestLat,urcrnrlat=northeastLat, + llcrnrlon=southwestLon, urcrnrlon=northeastLon, lat_ts=20, resolution='l') # TODO do I need to use lat_ts=20, ? + + # sort out the times we're using + futureWindow = options.futureWindow + startTime = options.startTime + endTime = options.endTime if options.endTime is not None else trajectoryTimeData[-1] + # as a sanity check, it's not really productive to make frames for times after our data ends + if endTime > trajectoryTimeData[-1] : + endTime = trajectoryTimeData[-1] + + # loop over time to create each frame + initAODFlat = initialAODdata.ravel() + initLonData = longitudeData[0].ravel() + initLatData = latitudeData[0].ravel() + for currentTime in range (startTime, endTime + 1) : + + # select only the window of trajectory data we need + alreadyHappenedTimeIndex = 0 + tooFarInFutureTimeIndex = trajectoryTimeData.size + for tempIndex in range (0, trajectoryTimeData.size) : + + # first find the time that corresponds to the "most recently happened" index + if trajectoryTimeData[tempIndex] <= currentTime : + # TODO if time doesn't always increase, this needs another check + alreadyHappenedTimeIndex = tempIndex + + # then figure out how much further we can go before we pass the futureWindow's edge + if (trajectoryTimeData[tempIndex] > (currentTime + futureWindow)) : + # TODO, if time data doesn't always increase I also need & (trajectoryTimeData[tempIndex] < trajectoryTimeData[tooFarInFutureTimeIndex]) + tooFarInFutureTimeIndex = tempIndex + break # TODO, this assumes time data always increases; also breaks suck so eventually take this out + + # only get data to plot the trajectory points if there is some available + thisFramePressures = None + thisFramePressLon = None + thisFramePressLat = None + if alreadyHappenedTimeIndex + 1 < tooFarInFutureTimeIndex : + + LOG.debug("Already happened index: " + str(alreadyHappenedTimeIndex)) + LOG.debug("Too far future index: " + str(tooFarInFutureTimeIndex)) + + # now pull out the pressure data and related lon/lat info for plotting + thisFramePressures = trajectoryPressureData[alreadyHappenedTimeIndex + 1 : tooFarInFutureTimeIndex].ravel() + thisFramePressLon = longitudeData[alreadyHappenedTimeIndex + 1 : tooFarInFutureTimeIndex].ravel() + thisFramePressLat = latitudeData[alreadyHappenedTimeIndex + 1 : tooFarInFutureTimeIndex].ravel() + + # make the plot + LOG.info("Creating trajectory plot for time " + str(float(currentTime)) + ".") + tempFigure = _create_imapp_figure (initAODFlat, initLonData, initLatData, + pressureData=thisFramePressures, pressLongitudeData=thisFramePressLon, pressLatitudeData=thisFramePressLat, + baseMapInstance=basemapObject, latRange=latRange, lonRange=lonRange) + + # save the plot to disk + LOG.info("Saving plot to disk.") + tempFigure.savefig(os.path.join(options.outpath, str(currentTime) + defaultValues['figureName']), dpi=defaultValues['figureDPI']) + + # get rid of the figure + plt.close(tempFigure) + del(tempFigure) + + def help(command=None): + """print help for a specific command or list of commands + e.g. help stats + """ + if command is None: + # print first line of docstring + for cmd in commands: + ds = commands[cmd].__doc__.split('\n')[0] + print "%-16s %s" % (cmd,ds) + else: + print commands[command].__doc__ + + # def test(): + # "run tests" + # test1() + # + + # all the local public functions are considered part of this program, collect them up + commands.update(dict(x for x in locals().items() if x[0] not in prior)) + + # if what the user asked for is not one of our existing functions, print the help + if (not args) or (args[0] not in commands): + parser.print_help() + help() + return 9 + else: + # call the function the user named, given the arguments from the command line + locals()[args[0]](*args[1:]) + + return 0 + + +if __name__=='__main__': + sys.exit(main()) \ No newline at end of file diff --git a/pyglance/glance/io.py b/pyglance/glance/io.py index 464ad91d5e81a389fca168accc7ddc00f43eb26e..8f35d59d14726fd9835521587f1ea167cd8b5a22 100644 --- a/pyglance/glance/io.py +++ b/pyglance/glance/io.py @@ -192,14 +192,19 @@ class hdf(SD): return + def get_variable_attributes (self, variableName) : + """ + returns all the attributes associated with a variable name + """ + + return self.get_variable_object(variableName).attributes() + def get_attribute(self, variableName, attributeName) : """ returns the value of the attribute if it is available for this variable, or None """ toReturn = None - - variable_object = self.get_variable_object(variableName) - temp_attributes = variable_object.attributes() + temp_attributes = self.get_variable_attributes(variableName) if attributeName in temp_attributes : toReturn = temp_attributes[attributeName] @@ -365,14 +370,20 @@ class nc(CDF): return + def get_variable_attributes (self, variableName) : + """ + returns all the attributes associated with a variable name + """ + + return self.get_variable_object(variableName).attributes() + def get_attribute(self, variableName, attributeName) : """ returns the value of the attribute if it is available for this variable, or None """ toReturn = None - variable_object = self.get_variable_object(variableName) - temp_attributes = variable_object.attributes() + temp_attributes = self.get_variable_attributes(variableName) if attributeName in temp_attributes : toReturn = temp_attributes[attributeName] @@ -525,15 +536,23 @@ class h5(object): return + def get_variable_attributes (self, variableName) : + """ + returns all the attributes associated with a variable name + """ + + return self.get_variable_object(variableName).attrs + def get_attribute(self, variableName, attributeName) : """ returns the value of the attribute if it is available for this variable, or None """ toReturn = None - variable_object = self.get_variable_object(variableName) - if (attributeName in variable_object.attrs) : - toReturn = variable_object.attrs[attributeName] + temp_attrs = self.get_variable_attributes(variableName) + + if (attributeName in temp_attrs) : + toReturn = temp_attrs[attributeName] return toReturn @@ -625,6 +644,17 @@ class aeri(object): return + def get_variable_attributes (self, variableName) : + """ + returns all the attributes associated with a variable name + """ + toReturn = { } + + # TODO + LOG.warn('Glance does not yet support attribute retrieval in AERI files. None will be used.') + + return toReturn + def get_attribute(self, variableName, attributeName) : """ returns the value of the attribute if it is available for this variable, or None @@ -717,6 +747,17 @@ class jpss_adl(object): return + def get_variable_attributes (self, variableName) : + """ + returns all the attributes associated with a variable name + """ + toReturn = { } + + # TODO + LOG.warn('Glance does not yet support attribute retrieval in JPSS ADL files. None will be used.') + + return toReturn + def get_attribute(self, variableName, attributeName) : """ returns the value of the attribute if it is available for this variable, or None diff --git a/pyglance/glance/mainreport.txt b/pyglance/glance/mainreport.txt deleted file mode 100644 index 199505e9415df15e09700be0f0dc7f4c67dc8e69..0000000000000000000000000000000000000000 --- a/pyglance/glance/mainreport.txt +++ /dev/null @@ -1,269 +0,0 @@ -<%doc> -This Mako template is intended to create a summary report page for glance reportGen. - -Created by Eva Schiffer Jun 2009. -Copyright (c) 2009 University of Wisconsin SSEC. All rights reserved. -</%doc> - -<%! - import glance.report as report -%> - -<title>File Comparison</title> -</head> -<body> - - <h2>Comparison Summary</h2> - - <p> - % if 'version' in runInfo : - report produced with ${runInfo['version']} <br> - % endif - comparison generated ${runInfo['time']} by user ${runInfo['user']} on ${runInfo['machine']} - </p> - - ## show information on each of the files we're comparing (should only be two) - % for fileKey in sorted(list(files)) : - <% tempFileInfo = files[fileKey] %> - <p> - ${fileKey}: - % if tempFileInfo.has_key('displayName') : - ${tempFileInfo['displayName']} - % endif - <blockquote> - path: ${tempFileInfo['path']} <br> - md5sum for ${fileKey}: ${tempFileInfo['md5sum']} <br> - last modified: ${tempFileInfo['lastModifiedTime']} - </blockquote> - </p> - % endfor - - ## display info on the config file that was used, if one was - % if ('config_file_path' in runInfo) and (runInfo['config_file_path'] is not None) : - <p> - A configuration file was used to control the production report.<br> - Please see <a href="./${runInfo['config_file_name']}">this copy of the configuration file</a> - for details. - </p> - - <p> - ## display information about any data filtering on the lons/lats - <% wasFiltered = False %> - % if ('data_filter_function_lat_in_a' in runInfo) and (not (runInfo['data_filter_function_lat_in_a'] is None)) : - Note: The latitude in file A was filtered.<br> - <% wasFiltered = True %> - % endif - % if ('data_filter_function_lat_in_b' in runInfo) and (not (runInfo['data_filter_function_lat_in_b'] is None)) : - Note: The latitude in file B was filtered.<br> - <% wasFiltered = True %> - % endif - % if ('data_filter_function_lon_in_a' in runInfo) and (not (runInfo['data_filter_function_lon_in_a'] is None)) : - Note: The longitude in file A was filtered.<br> - <% wasFiltered = True %> - % endif - % if ('data_filter_function_lon_in_b' in runInfo) and (not (runInfo['data_filter_function_lon_in_b'] is None)) : - Note: The longitude in file B was filtered.<br> - <% wasFiltered = True %> - % endif - % if wasFiltered : - Please see the original configuration file to view any data filtering functions. - % endif - </p> - - % endif - - ## if the lon/lat variables exist, display info on them - %if ('latitude' in runInfo) and ('longitude' in runInfo) : - - ## display the latitude and longitude variable names - <p> - % if ('latitude_alt_name_in_b' in runInfo) : - latitude in A: ${runInfo['latitude']}<br> - latitude in B: ${runInfo['latitude_alt_name_in_b']}<br> - % else : - latitude: ${runInfo['latitude']} <br> - % endif - % if ('longitude_alt_name_in_b' in runInfo) : - longitude in A: ${runInfo['longitude']}<br> - longitude in B: ${runInfo['longitude_alt_name_in_b']}<br> - % else : - longitude: ${runInfo['longitude']}<br> - % endif - - % if ('lon_lat_epsilon' in runInfo) and (runInfo['lon_lat_epsilon'] > 0.0) : - longitude/latitude comparison epsilon: ${runInfo['lon_lat_epsilon']}<br> - % endif - </p> - - ## if there is a problem with the longitude/latitude correlation between the two files, - ## make a nice big warning for the user - % if spatial.has_key('lon_lat_not_equal_points_count') and (spatial['lon_lat_not_equal_points_count'] > 0) : - <p> - WARNING: ${spatial['lon_lat_not_equal_points_count']} data points - (${report.make_formatted_display_string(spatial['lon_lat_not_equal_points_percent'])}% of all data) - show possible mismatch in values stored in file a - and file b longitude and latitude values. Depending on the degree of mismatch, some data value comparisons - in this report may be distorted or spatially nonsensical. Please consider re-running this report and including an - examination of your longitude and latitude variables with appropriate epsilons in order to analyze the significance - of the difference.<br> - ## if we're showing images, link to graphs showing the problem - % if runInfo['shouldIncludeImages'] : - <a href="./LonLatMismatch.A.png"> - View mismatching points in A's lon/lat system - </a><br> - <a href="./LonLatMismatch.B.png"> - View mismatching points in B's lon/lat system - </a> - % endif - </p> - % endif - - ## if we have some unique spatially invalid points, report them - ## are there any in file A? - %if spatial.has_key('file A') and spatial['file A'].has_key('numInvPts') and (spatial['file A']['numInvPts'] > 0) : - <% - fileAInfo = spatial['file A'] - %> - <p> - % if runInfo['shouldIncludeImages'] : - <a href="./SpatialMismatch.A.png"><img src="./SpatialMismatch.A.small.png"></a><br> - % endif - File A has ${fileAInfo['numInvPts']} valid data points not present in File B.<br> - File A has a total of ${report.make_formatted_display_string(fileAInfo['perInvPts'])}% invalid longitude/latitude data. <br> - </p> - % endif - ## are there any in file B? - %if spatial.has_key('file B') and spatial['file B'].has_key('numInvPts') and (spatial['file B']['numInvPts'] > 0) : - <% - fileBInfo = spatial['file B'] - %> - <p> - % if runInfo['shouldIncludeImages'] : - <a href="./SpatialMismatch.B.png"><img src="./SpatialMismatch.B.small.png"></a><br> - % endif - File B has ${fileBInfo['numInvPts']} valid data points not present in File A.<br> - File B has a total of ${report.make_formatted_display_string(fileBInfo['perInvPts'])}% longitude/latitude invalid data. <br> - </p> - % endif - ## report on shared spatial invalidity if there is any - % if spatial.has_key('perInvPtsInBoth') : - <% perInBoth = spatial['perInvPtsInBoth'] %> - % if perInBoth > 0 : - <p> - lon/lat data that is invalid in either file A or file B: ${report.make_formatted_display_string(perInBoth)}% - </p> - % endif - % endif - - ## end of the if to display lon/lat info - % endif - - % if len(variables.keys()) > 0 : - - <h3>Compared Variables</h3> - - ## report on all the variables that were compared and give a basic stat idea of how well they did - <blockquote> - <p> - ## TODO replace the table with floating boxes at some point - <table> - - % for variableKey in sorted(list(variables)) : - <% - # get some information about the variable, for convenience - tempVariableInfo = variables[variableKey] - tempVarRunInfo = tempVariableInfo['variable_run_info'] - - technicalName = tempVarRunInfo['variable_name'] - varDisplayName = technicalName - if 'display_name' in tempVarRunInfo : - varDisplayName = tempVarRunInfo['display_name'] - - rSquaredCorr = tempVariableInfo['r_squared_correlation'] - passPercent = tempVariableInfo['pass_epsilon_percent'] - finiteSimilarPercent = tempVariableInfo['finite_similar_percent'] - didPass = tempVarRunInfo['did_pass'] - %> - <tr> - <td> - % if not (didPass is None) : - %if didPass : - <img src="./pass.gif"> - %else : - <img src="./fail.gif"> - % endif - % endif - </td> - <td> - Variable: <a href="${tempVarRunInfo['variable_report_path_escaped']}">${varDisplayName}</a> <br> - Epsilon used: ${tempVarRunInfo['epsilon']} <br> - ## if there is an epsilon percent, also show that - % if ('epsilon_percent' in tempVarRunInfo) and (tempVarRunInfo['epsilon_percent'] is not None) : - Epsilon percent (of file A) used: ${tempVarRunInfo['epsilon_percent']}% <br> - % endif - ## if there's an r squared correlation value, also show that - % if rSquaredCorr is not None : - R-squared Correlation Coefficient: ${rSquaredCorr}<br> - % endif - Finite values within one epsilon of difference: - ${report.make_formatted_display_string(passPercent, '%.6g')}%<br> - Data that matched in finite-ness between the files: - ${report.make_formatted_display_string(finiteSimilarPercent, '%.6g')}% - </td> - </tr> - % endfor - </table> - </p> - </blockquote> - - % endif - - ## report the names of variables shared between the two files - <% sharedVars = varNames['sharedVars'] %> - % if len(sharedVars) > 0 : - <h3>Shared Variables</h3> - <p> - The following variables were common to both files: <br> - <blockquote> - % for varName in sharedVars : - ${varName} <br> - % endfor - </blockquote> - </p> - % endif - - ## check to see if there are any unique variables we need to report - <% - uniqueToAVars = varNames['uniqueToAVars'] - uniqueToBVars = varNames['uniqueToBVars'] - %> - % if (len(uniqueToAVars) > 0) or (len(uniqueToBVars) > 0) : - <h3>Unique Variables</h3> - - % if (len(uniqueToAVars) > 0) : - <p> - Variables only found in file A: - <blockquote> - %for varName in uniqueToAVars : - ${varName} <br> - % endfor - </blockquote> - </p> - % endif - - % if (len(uniqueToBVars) > 0) : - <p> - Variables only found in file B: - <blockquote> - %for varName in uniqueToBVars : - ${varName} <br> - % endfor - </blockquote> - </p> - % endif - - % endif - -</body> -</html> - diff --git a/pyglance/glance/report.py b/pyglance/glance/report.py index 2d33e721840c17a41e3ab691c16ee6340049134b..24189801bfb200c6a6be217f18a571eb6cb6647d 100644 --- a/pyglance/glance/report.py +++ b/pyglance/glance/report.py @@ -11,6 +11,7 @@ import sys, logging from pkg_resources import resource_string, resource_filename #, resource_stream from mako.template import Template +from mako.lookup import TemplateLookup import types as types import numpy as np import shutil as shutil @@ -29,8 +30,10 @@ formattingSettings = { # in the template into the kwargs def _make_and_save_page (fullFilePath, templateFileNameToUse, **kwargs) : + tempLookup = TemplateLookup(directories=[resource_filename(__name__, "stuff")]) + fileToWrite = open(fullFilePath, 'w') - tempTemplate = Template(resource_string(__name__, templateFileNameToUse)) + tempTemplate = Template(resource_string(__name__, templateFileNameToUse), lookup=tempLookup) fileToWrite.write(tempTemplate.render(**kwargs)) fileToWrite.close() @@ -139,11 +142,11 @@ def generate_and_save_summary_report(files, 'variables': variables } - _make_and_save_page((outputPath + "/" + reportFileName), 'mainreport.txt', **kwargs) + _make_and_save_page((outputPath + "/" + reportFileName), 'stuff/mainreport.txt', **kwargs) # copy the pass/fail images, TODO should I move this to a list input in the parameters? - passFile = resource_filename(__name__, 'pass.gif') # TODO, how can this be done without unzipping the egg? - failFile = resource_filename(__name__, 'fail.gif') # TODO, how can this be done without unzipping the egg? + passFile = resource_filename(__name__, 'stuff/pass.gif') # TODO, how can this be done without unzipping the egg? + failFile = resource_filename(__name__, 'stuff/fail.gif') # TODO, how can this be done without unzipping the egg? shutil.copy(passFile, outputPath) shutil.copy(failFile, outputPath) @@ -160,7 +163,7 @@ def generate_and_save_doc_page(definitions, outputPath) : """ kwargs = {'definitions': definitions } - _make_and_save_page(outputPath + "/doc.html", 'doc.txt', **kwargs) + _make_and_save_page(outputPath + "/doc.html", 'stuff/doc.txt', **kwargs) return @@ -258,7 +261,7 @@ def generate_and_save_variable_report(files, 'imageNames': imageNames } - _make_and_save_page((outputPath + "/" + reportFileName), 'variablereport.txt', **kwargs) + _make_and_save_page((outputPath + "/" + reportFileName), 'stuff/variablereport.txt', **kwargs) return diff --git a/pyglance/glance/stats.py b/pyglance/glance/stats.py index 6412011a0ccb363a6b83b6e0abf7ac9978b14cd8..af89e49f7a8270962721412a854e5ff58cbedd30 100644 --- a/pyglance/glance/stats.py +++ b/pyglance/glance/stats.py @@ -354,8 +354,8 @@ class GeneralStatistics (StatisticalData) : total_num_values = a_missing_mask.size # fill in our statistics - self.a_missing_value = diffInfoObject.a_data_object.fill_value - self.b_missing_value = diffInfoObject.b_data_object.fill_value + self.a_missing_value = diffInfoObject.a_data_object.select_fill_value() + self.b_missing_value = diffInfoObject.b_data_object.select_fill_value() self.epsilon = diffInfoObject.epsilon_value self.epsilon_percent = diffInfoObject.epsilon_percent self.max_a = delta.max_with_mask(diffInfoObject.a_data_object.data, good_in_a_mask) @@ -615,23 +615,50 @@ class StatisticalAnalysis (StatisticalData) : documentation of the statistics. """ - def __init__ (self, - a_data, b_data, - a_missing_value=None, b_missing_value=None, - a_ignore_mask=None, b_ignore_mask=None, - epsilon=0., epsilon_percent=None) : + def __init__ (self) : """ - do a full statistical analysis of the data + this is a blank constructor to support our new class method creation pattern """ - self.title = "Statistical Summary" + + @classmethod + def withSimpleData (in_class, + a_data, b_data, + a_missing_value=None, b_missing_value=None, + a_ignore_mask=None, b_ignore_mask=None, + epsilon=0., epsilon_percent=None) : + """ + do a full statistical analysis of the data, after building the data objects + """ + + new_object = in_class() aDataObject = dataobj.DataObject(a_data, fillValue=a_missing_value, ignoreMask=a_ignore_mask) bDataObject = dataobj.DataObject(b_data, fillValue=b_missing_value, ignoreMask=b_ignore_mask) + diffInfo = dataobj.DiffInfoObject(aDataObject, bDataObject, - epsilonValue=epsilon, epsilonPercent=epsilon_percent) + epsilonValue=epsilon, epsilonPercent=epsilon_percent) + + new_object._create_stats(diffInfo) + + return new_object + + @classmethod + def withDataObjects (in_class, + a_data_object, b_data_object, + epsilon=0., epsilon_percent=None) : + """ + do a full statistical analysis of the data, using the given data objects + """ + + new_object = in_class() + + diffInfo = dataobj.DiffInfoObject(a_data_object, b_data_object, + epsilonValue=epsilon, epsilonPercent=epsilon_percent) + + new_object._create_stats(diffInfo) - self._create_stats(diffInfo) + return new_object def _create_stats(self, diffInfoObject) : """ diff --git a/pyglance/glance/stuff/basereport.txt b/pyglance/glance/stuff/basereport.txt new file mode 100644 index 0000000000000000000000000000000000000000..98ad9ccb9f169ae5a8f1087da54e021795439475 --- /dev/null +++ b/pyglance/glance/stuff/basereport.txt @@ -0,0 +1,163 @@ +<%doc> +This Mako template builds the skeleton of html reports for glance. + +Created by Eva Schiffer Nov 2011. +Copyright (c) 2011 University of Wisconsin SSEC. All rights reserved. +</%doc> + +<%! + import glance.report as report +%> + +<%block name="htmlContent"> + +<html> + <head> + <title>${self.title()}</title> + </head> + + <body> + + <h1><%block name="title"/></h2> + + <%block name="runIdentification"> + + ## display information on the version, user, and machine that ran the report + <p> + % if 'version' in runInfo : + report produced with ${runInfo['version']} <br> + % endif + comparison generated ${runInfo['time']} by user ${runInfo['user']} on ${runInfo['machine']} + </p> + + </%block> + + <%block name="fileIdentification"> + + ## show information on each of the files we're comparing (should only be two) + % for fileKey in sorted(list(files)) : + <% tempFileInfo = files[fileKey] %> + <p> + ${fileKey}: + % if 'displayName' in tempFileInfo : + ${tempFileInfo['displayName']} + % endif + <blockquote> + path: ${tempFileInfo['path']} <br> + md5sum for ${fileKey}: ${tempFileInfo['md5sum']} <br> + last modified: ${tempFileInfo['lastModifiedTime']} + </blockquote> + </p> + % endfor + + </%block> + + <%block name="configInfo"> + + ## display info on the config file that was used, if one was + % if ('config_file_path' in runInfo) and (runInfo['config_file_path'] is not None) : + <p> + A configuration file was used to control the production report.<br> + <%block name="configFileLink"> + Please see <a href="./${runInfo['config_file_name']}">this copy of the configuration file</a> + </%block> + for details. + </p> + + <p> + <% wasFiltered = False %> + + <%block name="additionalFilterInfo"></%block> + + ## display information about any data filtering on the lons/lats + % if ('data_filter_function_lat_in_a' in runInfo) and (not (runInfo['data_filter_function_lat_in_a'] is None)) : + Note: The latitude in file A was filtered.<br> + <% wasFiltered = True %> + % endif + % if ('data_filter_function_lat_in_b' in runInfo) and (not (runInfo['data_filter_function_lat_in_b'] is None)) : + Note: The latitude in file B was filtered.<br> + <% wasFiltered = True %> + % endif + % if ('data_filter_function_lon_in_a' in runInfo) and (not (runInfo['data_filter_function_lon_in_a'] is None)) : + Note: The longitude in file A was filtered.<br> + <% wasFiltered = True %> + % endif + % if ('data_filter_function_lon_in_b' in runInfo) and (not (runInfo['data_filter_function_lon_in_b'] is None)) : + Note: The longitude in file B was filtered.<br> + <% wasFiltered = True %> + % endif + + ## show an additional message if there was any filtering + % if wasFiltered : + Please see the original configuration file to view any data filtering functions. + % endif + </p> + + % endif + + </%block> + + <%block name="lonlatInfo"> + + ## if the lon/lat variables exist, display info on them + %if ('latitude' in runInfo) and ('longitude' in runInfo) : + + ## display the latitude and longitude variable names + <p> + % if ('latitude_alt_name_in_b' in runInfo) : + latitude in A: ${runInfo['latitude']}<br> + latitude in B: ${runInfo['latitude_alt_name_in_b']}<br> + % else : + latitude: ${runInfo['latitude']} <br> + % endif + % if ('longitude_alt_name_in_b' in runInfo) : + longitude in A: ${runInfo['longitude']}<br> + longitude in B: ${runInfo['longitude_alt_name_in_b']}<br> + % else : + longitude: ${runInfo['longitude']}<br> + % endif + + % if ('lon_lat_epsilon' in runInfo) and (runInfo['lon_lat_epsilon'] > 0.0) : + longitude/latitude comparison epsilon: ${runInfo['lon_lat_epsilon']}<br> + % endif + </p> + + ## if there is a problem with the longitude/latitude correlation between the two files, + ## make a nice big warning for the user + % if spatial.has_key('lon_lat_not_equal_points_count') and (spatial['lon_lat_not_equal_points_count'] > 0) : + <p> + WARNING: ${spatial['lon_lat_not_equal_points_count']} data points + (${report.make_formatted_display_string(spatial['lon_lat_not_equal_points_percent'])}% of all data) + show possible mismatch in values stored in file a + and file b longitude and latitude values. Depending on the degree of mismatch, some data value comparisons + in this report may be distorted or spatially nonsensical. Please consider re-running this report and including an + examination of your longitude and latitude variables with appropriate epsilons in order to analyze the significance + of the difference.<br> + ## if we're showing images, link to graphs showing the problem + % if runInfo['shouldIncludeImages'] : + <%block name="lonlatInvalidImages"> + <a href="./LonLatMismatch.A.png"> + View mismatching points in A's lon/lat system + </a><br> + <a href="./LonLatMismatch.B.png"> + View mismatching points in B's lon/lat system + </a> + </%block> + % endif + </p> + % endif + + <%block name="spatialInvalidity"></%block> + + ## end of the if to display lon/lat info + % endif + + </%block> + + <%block name="comparedData"></%block> + + </body> + +</html> + +</%block> \ No newline at end of file diff --git a/pyglance/glance/doc.txt b/pyglance/glance/stuff/doc.txt similarity index 100% rename from pyglance/glance/doc.txt rename to pyglance/glance/stuff/doc.txt diff --git a/pyglance/glance/fail.gif b/pyglance/glance/stuff/fail.gif similarity index 100% rename from pyglance/glance/fail.gif rename to pyglance/glance/stuff/fail.gif diff --git a/pyglance/glance/stuff/mainreport.txt b/pyglance/glance/stuff/mainreport.txt new file mode 100644 index 0000000000000000000000000000000000000000..5f1732fc456645d34c40108b6ade22081e184a11 --- /dev/null +++ b/pyglance/glance/stuff/mainreport.txt @@ -0,0 +1,183 @@ +<%doc> +This Mako template is intended to create a summary report page for Glance. + +Created by Eva Schiffer Nov 2011. +Copyright (c) 2011 University of Wisconsin SSEC. All rights reserved. +</%doc> + +<%! + import glance.report as report +%> + +<%inherit file="basereport.txt"/> + +<%block name="title">File Comparison Summary</%block> + +## report on any points that are spatially invalid in one file and not the other +<%block name="spatialInvalidity"> + + ## if we have some unique spatially invalid points, report them + ## are there any in file A? + %if spatial.has_key('file A') and spatial['file A'].has_key('numInvPts') and (spatial['file A']['numInvPts'] > 0) : + <% + fileAInfo = spatial['file A'] + %> + <p> + % if runInfo['shouldIncludeImages'] : + <a href="./SpatialMismatch.A.png"><img src="./SpatialMismatch.A.small.png"></a><br> + % endif + File A has ${fileAInfo['numInvPts']} valid data points not present in File B.<br> + File A has a total of ${report.make_formatted_display_string(fileAInfo['perInvPts'])}% invalid longitude/latitude data. <br> + </p> + % endif + ## are there any in file B? + %if spatial.has_key('file B') and spatial['file B'].has_key('numInvPts') and (spatial['file B']['numInvPts'] > 0) : + <% + fileBInfo = spatial['file B'] + %> + <p> + % if runInfo['shouldIncludeImages'] : + <a href="./SpatialMismatch.B.png"><img src="./SpatialMismatch.B.small.png"></a><br> + % endif + File B has ${fileBInfo['numInvPts']} valid data points not present in File A.<br> + File B has a total of ${report.make_formatted_display_string(fileBInfo['perInvPts'])}% longitude/latitude invalid data. <br> + </p> + % endif + ## report on shared spatial invalidity if there is any + % if spatial.has_key('perInvPtsInBoth') : + <% perInBoth = spatial['perInvPtsInBoth'] %> + % if perInBoth > 0 : + <p> + lon/lat data that is invalid in either file A or file B: ${report.make_formatted_display_string(perInBoth)}% + </p> + % endif + % endif + +</%block> + +## this is the main body block that includes most of the comparison info +<%block name="comparedData"> + + ## list all of the variables that were compared (if any) + <%block name="comparedVariablesList"> + + % if len(variables.keys()) > 0 : + + <h3>Compared Variables</h3> + + ## report on all the variables that were compared and give a basic stat idea of how well they did + <blockquote> + <p> + ## TODO replace the table with floating boxes at some point + <table> + % for variableKey in sorted(list(variables)) : + <% + # get some information about the variable, for convenience + tempVariableInfo = variables[variableKey] + tempVarRunInfo = tempVariableInfo['variable_run_info'] + + technicalName = tempVarRunInfo['variable_name'] + varDisplayName = technicalName + if 'display_name' in tempVarRunInfo : + varDisplayName = tempVarRunInfo['display_name'] + + rSquaredCorr = tempVariableInfo['r_squared_correlation'] + passPercent = tempVariableInfo['pass_epsilon_percent'] + finiteSimilarPercent = tempVariableInfo['finite_similar_percent'] + didPass = tempVarRunInfo['did_pass'] + %> + <tr> + <td> + % if not (didPass is None) : + %if didPass : + <img src="./pass.gif"> + %else : + <img src="./fail.gif"> + % endif + % endif + </td> + <td> + Variable: <a href="${tempVarRunInfo['variable_report_path_escaped']}">${varDisplayName}</a> <br> + Epsilon used: ${tempVarRunInfo['epsilon']} <br> + ## if there is an epsilon percent, also show that + % if ('epsilon_percent' in tempVarRunInfo) and (tempVarRunInfo['epsilon_percent'] is not None) : + Epsilon percent (of file A) used: ${tempVarRunInfo['epsilon_percent']}% <br> + % endif + ## if there's an r squared correlation value, also show that + % if rSquaredCorr is not None : + R-squared Correlation Coefficient: ${rSquaredCorr}<br> + % endif + Finite values within one epsilon of difference: + ${report.make_formatted_display_string(passPercent, '%.6g')}%<br> + Data that matched in finite-ness between the files: + ${report.make_formatted_display_string(finiteSimilarPercent, '%.6g')}% + </td> + </tr> + % endfor + </table> + </p> + </blockquote> + + % endif + + </%block> + + + + ## report the names of variables shared between the two files, if any + <%block name="sharedVarNames"> + + <% sharedVars = varNames['sharedVars'] %> + % if len(sharedVars) > 0 : + <h3>Shared Variables</h3> + <p> + The following variables were common to both files: <br> + <blockquote> + % for varName in sharedVars : + ${varName} <br> + % endfor + </blockquote> + </p> + % endif + + </%block> + + ## report the names of variables unique to a or b, if any + <%block name="uniqueVarNames"> + + <% + uniqueToAVars = varNames['uniqueToAVars'] + uniqueToBVars = varNames['uniqueToBVars'] + %> + ## check to see if there are any unique variables we need to report + % if (len(uniqueToAVars) > 0) or (len(uniqueToBVars) > 0) : + <h3>Unique Variables</h3> + + % if (len(uniqueToAVars) > 0) : + <p> + Variables only found in file A: + <blockquote> + %for varName in uniqueToAVars : + ${varName} <br> + % endfor + </blockquote> + </p> + % endif + + % if (len(uniqueToBVars) > 0) : + <p> + Variables only found in file B: + <blockquote> + %for varName in uniqueToBVars : + ${varName} <br> + % endfor + </blockquote> + </p> + % endif + + % endif + + </%block> + +</%block> + diff --git a/pyglance/glance/pass.gif b/pyglance/glance/stuff/pass.gif similarity index 100% rename from pyglance/glance/pass.gif rename to pyglance/glance/stuff/pass.gif diff --git a/pyglance/glance/stuff/variablereport.txt b/pyglance/glance/stuff/variablereport.txt new file mode 100644 index 0000000000000000000000000000000000000000..ea53421d1b5b01129e7fd2343a42b6f598e0252e --- /dev/null +++ b/pyglance/glance/stuff/variablereport.txt @@ -0,0 +1,169 @@ +<%doc> +This Mako template is intended to create a variable specific report page for Glance. + +Created by Eva Schiffer Nov 2011. +Copyright (c) 2011 University of Wisconsin SSEC. All rights reserved. +</%doc> + +<%! + import glance.report as report +%> + +<%inherit file="basereport.txt"/> + +<%block name="title"> + <% + variableName = runInfo['variable_name'] + variableDisplayName = variableName + if (runInfo.has_key('display_name')): + variableDisplayName = runInfo['display_name'] + %> + ${variableDisplayName} Variable Comparison +</%block> + +## this version of the config file path is escaped to be down in the variable report directory +<%block name="configFileLink"> + Please see <a href="${runInfo['config_file_path']}">this copy of the configuration file</a> +</%block> + +## additional filtering information specific to the variable being analyzed +<%block name="additionalFilterInfo"> + ## display info about the basic data filters if the data was filtered + % if ('data_filter_function_a' in runInfo) and (not (runInfo['data_filter_function_a'] is None)) : + Note: The data for this variable from file A was filtered.<br> + <% wasFiltered = True %> + % endif + % if ('data_filter_function_b' in runInfo) and (not (runInfo['data_filter_function_b'] is None)) : + Note: The data for this variable from file B was filtered.<br> + <% wasFiltered = True %> + % endif +</%block> + +## these need to be linked out further to allow for variable directories; TODO, this won't work with deeper directories :( +<%block name="lonlatInvalidImages"> + <a href="../LonLatMismatch.A.png"> + View mismatching points in A's lon/lat system + </a><br> + <a href="../LonLatMismatch.B.png"> + View mismatching points in B's lon/lat system + </a> +</%block> + +## this is the main body block that includes most of the comparison info +<%block name="comparedData"> + + ## show the original data images if we have any + <%block name="originalImages"> + + % if (len(imageNames['original']) > 0) : + <h3>Original Data</h3> + <p> + % for image in imageNames['original']: + <a href="./${image}"><img src="small.${image}"></a> + % endfor + </p> + % endif + + </%block> + + ## this is the summary of how the data was compared + <%block name="comparisonSummary"> + + <h3>Comparison Information</h3> + + ## comparison parameters + <p> + ## display the variable names + % if 'alternate_name_in_B' in runInfo : + variable name in A: ${runInfo['variable_name']} <br> + variable name in B: ${runInfo['alternate_name_in_B']} <br> + % else : + variable name: ${runInfo['variable_name']} <br> + % endif + + ## display the epsilon + epsilon value: ${runInfo['epsilon']} <br> + + ## if there is an epsilon percent, also show that + % if ('epsilon_percent' in runInfo) and (runInfo['epsilon_percent'] is not None) : + epsilon percent: ${runInfo['epsilon_percent']}% <br> + % endif + + ## display the missing value + % if ('missing_value_alt_in_b' in runInfo) and (not (runInfo['missing_value_alt_in_b'] is runInfo['missing_value'])) : + "missing" data value in A: ${str(runInfo['missing_value'])}<br> + "missing" data value in B: ${str(runInfo['missing_value_alt_in_b'])}<br> + % else : + "missing" data value: ${str(runInfo['missing_value'])}<br> + % endif + + ## if there are units available, display those + % if ('units_a' in runInfo) and (runInfo['units_a'] is not None) : + units in A: ${str(runInfo['units_a'])}<br> + % endif + % if ('units_b' in runInfo) and (runInfo['units_b'] is not None) : + units in B: ${str(runInfo['units_b'])}<br> + % endif + </p> + + </%block> + + ## show the comparison statistics + <%block name="stats"> + + <h3>Statistical Summary</h3> + + ## list out all of the statistics groups we have + <dl> + % for setName in sorted(list(statGroups)) : + <% dataSet = statGroups[setName] %> + <dt> + ${setName} + </dt> + <dd> + % for statName, statValue in sorted(list(dataSet.items())) : + ${statName}<%block name="statDocLink"><a href="${runInfo['doc_path']}">*</a></%block>: ${report.make_formatted_display_string(statValue)} <br> + % endfor + <br> + <dd> + % endfor + </dl> + + </%block> + + ## display any comparison images we have, if appropriate + <%block name="comparisonImages"> + + <% + hideComparisonImages = ('short_circuit_diffs' in runInfo) and runInfo['short_circuit_diffs'] + %> + % if (len(imageNames['compared']) > 0) and (not hideComparisonImages) : + <p> + <% inSubSet = False %> + % for image in imageNames['compared'] : + ## if we have a plain image, display it + % if (type(image) is str) : + % if inSubSet : + <hr> + <% inSubSet = False %> + % endif + <a href="./${image}"><img src="./small.${image}"></a> + + ## if we have a subset of images, separate it from the rest + % elif (type(image) is list) : + <% inSubSet = True %> + <hr> + <blockquote> + % for subImage in image : + <a href="./${subImage}"><img src="./small.${subImage}"></a> + % endfor + </blockquote> + % endif + + % endfor + </p> + % endif + + </%block> + +</%block> diff --git a/pyglance/glance/variablereport.txt b/pyglance/glance/variablereport.txt deleted file mode 100644 index 18196955d5434faaea72da172eb7ef6a9427cdf4..0000000000000000000000000000000000000000 --- a/pyglance/glance/variablereport.txt +++ /dev/null @@ -1,242 +0,0 @@ -<%doc> -This Mako template is intended to create a variable specific report page for glance reportGen. - -Created by Eva Schiffer Jun 2009. -Copyright (c) 2009 University of Wisconsin SSEC. All rights reserved. -</%doc> - -<%! - import glance.report as report -%> - -## we will use this stuff often, so hang on to it -<% - variableName = runInfo['variable_name'] - variableDisplayName = variableName - if (runInfo.has_key('display_name')): - variableDisplayName = runInfo['display_name'] - hideComparisonImages = ('short_circuit_diffs' in runInfo) and runInfo['short_circuit_diffs'] -%> - -<title>${variableDisplayName} Variable Comparison</title> -</head> -<body> - - <h1>${variableDisplayName} Variable Comparison</h1> - - <p> - % if 'version' in runInfo : - report produced with ${runInfo['version']} <br> - % endif - comparison generated ${runInfo['time']} by user ${runInfo['user']} on ${runInfo['machine']} - </p> - - ## show information on each of the files we're comparing (should only be two) - % for fileKey in sorted(list(files)) : - <% tempFileInfo = files[fileKey] %> - <p> - ${fileKey}: - % if 'displayName' in tempFileInfo : - ${tempFileInfo['displayName']} - % endif - <blockquote> - path: ${tempFileInfo['path']} <br> - md5sum for ${fileKey}: ${tempFileInfo['md5sum']} <br> - last modified: ${tempFileInfo['lastModifiedTime']} - </blockquote> - </p> - % endfor - - ## display info on the config file that was used, if one was - % if ('config_file_path' in runInfo) and (runInfo['config_file_path'] is not None) : - <p> - A configuration file was used to control the production report.<br> - Please see <a href="${runInfo['config_file_path']}">this copy of the configuration file</a> - for details. - </p> - - <p> - <% wasFiltered = False %> - - ## display info about the basic data filters if the data was filtered - % if ('data_filter_function_a' in runInfo) and (not (runInfo['data_filter_function_a'] is None)) : - Note: The data for this variable from file A was filtered.<br> - <% wasFiltered = True %> - % endif - % if ('data_filter_function_b' in runInfo) and (not (runInfo['data_filter_function_b'] is None)) : - Note: The data for this variable from file B was filtered.<br> - <% wasFiltered = True %> - % endif - - ## display information about any data filtering on the lons/lats - % if ('data_filter_function_lat_in_a' in runInfo) and (not (runInfo['data_filter_function_lat_in_a'] is None)) : - Note: The latitude in file A was filtered.<br> - <% wasFiltered = True %> - % endif - % if ('data_filter_function_lat_in_b' in runInfo) and (not (runInfo['data_filter_function_lat_in_b'] is None)) : - Note: The latitude in file B was filtered.<br> - <% wasFiltered = True %> - % endif - % if ('data_filter_function_lon_in_a' in runInfo) and (not (runInfo['data_filter_function_lon_in_a'] is None)) : - Note: The longitude in file A was filtered.<br> - <% wasFiltered = True %> - % endif - % if ('data_filter_function_lon_in_b' in runInfo) and (not (runInfo['data_filter_function_lon_in_b'] is None)) : - Note: The longitude in file B was filtered.<br> - <% wasFiltered = True %> - % endif - - ## show an additional message if there was any filtering - % if wasFiltered : - Please see the original configuration file to view any data filtering functions. - % endif - </p> - - % endif - - ## if the lon/lat variables exist, display info on them - %if ('latitude' in runInfo) and ('longitude' in runInfo) : - - ## display the latitude and longitude variable names - <p> - % if ('latitude_alt_name_in_b' in runInfo) : - latitude in A: ${runInfo['latitude']}<br> - latitude in B: ${runInfo['latitude_alt_name_in_b']}<br> - % else : - latitude: ${runInfo['latitude']} <br> - % endif - % if ('longitude_alt_name_in_b' in runInfo) : - longitude in A: ${runInfo['longitude']}<br> - longitude in B: ${runInfo['longitude_alt_name_in_b']}<br> - % else : - longitude: ${runInfo['longitude']}<br> - % endif - - % if ('lon_lat_epsilon' in runInfo) and (runInfo['lon_lat_epsilon'] > 0.0) : - longitude/latitude comparison epsilon: ${runInfo['lon_lat_epsilon']}<br> - % endif - </p> - - ## if there is a problem with the longitude/latitude correlation between the two files, - ## make a nice big warning for the user - % if spatial.has_key('lon_lat_not_equal_points_count') and (spatial['lon_lat_not_equal_points_count'] > 0) : - <p> - WARNING: ${spatial['lon_lat_not_equal_points_count']} data points - (${report.make_formatted_display_string(spatial['lon_lat_not_equal_points_percent'])}% of all data) - show possible mismatch in values stored in file a - and file b longitude and latitude values. Depending on the degree of mismatch, some data value comparisons - in this report may be distorted or spatially nonsensical. Please consider re-running this report and including an - examination of your longitude and latitude variables with appropriate epsilons in order to analyze the significance - of the difference.<br> - ## if we're showing images, link to graphs showing the problem - % if runInfo['shouldIncludeImages'] : - <a href="../LonLatMismatch.A.png"> - View mismatching points in A's lon/lat system - </a><br> - <a href="../LonLatMismatch.B.png"> - View mismatching points in B's lon/lat system - </a> - % endif - </p> - % endif - - ## end of the if to display lon/lat info - % endif - - % if (len(imageNames['original']) > 0) : - <h3>Original Data</h3> - <p> - % for image in imageNames['original']: - <a href="./${image}"><img src="small.${image}"></a> - % endfor - </p> - % endif - - <h3>Comparison Information</h3> - - ## comparison parameters - <p> - ## display the variable names - % if 'alternate_name_in_B' in runInfo : - variable name in A: ${variableName} <br> - variable name in B: ${runInfo['alternate_name_in_B']} <br> - % else : - variable name: ${variableName} <br> - % endif - - ## display the epsilon - epsilon value: ${runInfo['epsilon']} <br> - - ## if there is an epsilon percent, also show that - % if ('epsilon_percent' in runInfo) and (runInfo['epsilon_percent'] is not None) : - epsilon percent: ${runInfo['epsilon_percent']}% <br> - % endif - - ## display the missing value - % if ('missing_value_alt_in_b' in runInfo) and (not (runInfo['missing_value_alt_in_b'] is runInfo['missing_value'])) : - "missing" data value in A: ${str(runInfo['missing_value'])}<br> - "missing" data value in B: ${str(runInfo['missing_value_alt_in_b'])}<br> - % else : - "missing" data value: ${str(runInfo['missing_value'])}<br> - % endif - - ## if there are units available, display those - % if ('units_a' in runInfo) and (runInfo['units_a'] is not None) : - units in A: ${str(runInfo['units_a'])}<br> - % endif - % if ('units_b' in runInfo) and (runInfo['units_b'] is not None) : - units in B: ${str(runInfo['units_b'])}<br> - % endif - </p> - - <h3>Statistical Summary</h3> - - ## list out all of the statistics groups we have - <dl> - % for setName in sorted(list(statGroups)) : - <% dataSet = statGroups[setName] %> - <dt> - ${setName} - </dt> - <dd> - % for statName, statValue in sorted(list(dataSet.items())) : - ${statName}<a href="${runInfo['doc_path']}">*</a>: ${report.make_formatted_display_string(statValue)} <br> - % endfor - <br> - <dd> - % endfor - </dl> - - % if (len(imageNames['compared']) > 0) and (not hideComparisonImages) : - <p> - <% inSubSet = False %> - - % for image in imageNames['compared'] : - - ## if we have a plain image, display it - % if (type(image) is str) : - % if inSubSet : - <hr> - <% inSubSet = False %> - % endif - - <a href="./${image}"><img src="./small.${image}"></a> - - ## if we have a subset of images, separate it from the rest - % elif (type(image) is list) : - <% inSubSet = True %> - <hr> - <blockquote> - % for subImage in image : - <a href="./${subImage}"><img src="./small.${subImage}"></a> - % endfor - </blockquote> - % endif - - % endfor - </p> - % endif - -</body> -</html> - diff --git a/pyglance/setup.py b/pyglance/setup.py index 5de675a866edb4e8a9990128a9e3de440e13d959..020caeb80567f99086f378000d8ff16ebbb9f5f9 100644 --- a/pyglance/setup.py +++ b/pyglance/setup.py @@ -22,11 +22,11 @@ easy_install -d $HOME/Library/Python -vi http://larch.ssec.wisc.edu/eggs/repos g from setuptools import setup, find_packages setup( name="glance", - version="0.2.6.29", - zip_safe = True, + version="0.2.7.01", + zip_safe = False, entry_points = { 'console_scripts': [ 'glance = glance.compare:main' ] }, packages = find_packages('.'), install_requires=[ 'numpy', 'matplotlib' ], - package_data = {'': ['*.txt', '*.gif'], } + package_data = {'stuff': ['*.txt', '*.gif'], } )