diff --git a/pyglance/glance/compare.py b/pyglance/glance/compare.py index 6b28ee2ad6919f880e8a76f7ccb89bbb0c4e51da..4b80bf6f27d1abe4e0c1a0ccfdc06d8cdd131e74 100644 --- a/pyglance/glance/compare.py +++ b/pyglance/glance/compare.py @@ -1831,6 +1831,7 @@ def reportGen_library_call (a_path, b_path, var_list=[ ], if not do_not_test_with_lon_lat : mask_a_to_use = lon_lat_data['a']['inv_mask'] mask_b_to_use = lon_lat_data['b']['inv_mask'] + LOG.debug("Analyzing " + displayName + " statistically.") variable_stats = statistics.StatisticalAnalysis.withSimpleData(aData, bData, varRunInfo['missing_value'], varRunInfo['missing_value_alt_in_b'], mask_a_to_use, mask_b_to_use, @@ -2122,10 +2123,10 @@ def inspect_stats_library_call (afn, var_list=[ ], options_set={ }, do_document= dict_data for each_stat in sorted(list(dict_data)): print >> output_channel, ' %s: %s' % (each_stat, dict_data[each_stat]) - if doc_each: print >> output_channel, (' ' + statistics.StatisticalAnalysis.doc_strings()[each_stat]) + if doc_each: print >> output_channel, (' ' + statistics.StatisticalInspectionAnalysis.doc_strings()[each_stat]) print >> output_channel, '' if doc_atend: - print >> output_channel, ('\n\n' + statistics.STATISTICS_DOC_STR) + print >> output_channel, ('\n\n' + statistics.INSP_STATISTICS_DOC_STR) def main(): import optparse diff --git a/pyglance/glance/gui_controller.py b/pyglance/glance/gui_controller.py index 2068942cdc4b9eac64d767397df916b501893359..4f5281091ff68d52b2b47dd135f9b314b4c9fb9f 100644 --- a/pyglance/glance/gui_controller.py +++ b/pyglance/glance/gui_controller.py @@ -145,6 +145,13 @@ class GlanceGUIController (object) : self.model.updateSettingsDataSelection(newImageType=new_image_type) + def userSelectedColormap (self, new_colormap) : + """ + the user has selected a new colormap + """ + + self.model.updateSettingsDataSelection(newColormap=new_colormap) + def userSelectedDataForm (self, new_data_form) : """ the user has selected a new data form @@ -152,6 +159,13 @@ class GlanceGUIController (object) : self.model.updateSettingsDataSelection(newDataForm=new_data_form) + def userToggledSharedRange(self, should_use_shared_range) : + """ + the user has toggled whether or not the original data should use a shared range + """ + + self.model.updateSettingsDataSelection(useSharedRangeForOriginals=should_use_shared_range) + def userRequestsStats (self) : """ the user has asked for stats information diff --git a/pyglance/glance/gui_figuremanager.py b/pyglance/glance/gui_figuremanager.py index ecdf333e8ddad9326d1fa93f575813a020b2e08b..21b5624ede9d227e34a16927c2a294dcf9bf4d9f 100644 --- a/pyglance/glance/gui_figuremanager.py +++ b/pyglance/glance/gui_figuremanager.py @@ -24,13 +24,14 @@ import numpy as np import glance.data as dataobjects import glance.figures as figures import glance.gui_model as model +from glance.gui_constants import * LOG = logging.getLogger(__name__) -# the number of bins to use for histograms -DEFAULT_NUM_BINS = 50 - -NO_DATA_MESSAGE = "Requested data was not available or did not exist." +# colormaps that are available in the GUI +# TODO, if this changes the list of colormap names in the constants module needs to be kept up +AVAILABLE_COLORMAPS = {CM_RAINBOW: cm.jet, + CM_GRAY: cm.gray} class GlanceGUIFigures (object) : """ diff --git a/pyglance/glance/gui_model.py b/pyglance/glance/gui_model.py index 76e92666d495d8b2000a53a64ec728c7a14fd57c..f4c02b695437e14dc42f5e544291daedd68c196c 100644 --- a/pyglance/glance/gui_model.py +++ b/pyglance/glance/gui_model.py @@ -12,6 +12,7 @@ import numpy as np import glance.data as dataobjects import glance.io as io +from glance.gui_constants import * LOG = logging.getLogger(__name__) @@ -25,50 +26,6 @@ errors. It's expected that error handlers will manage either logging or displayi errors appropriately. """ -# constants for the possible image types -ORIGINAL_A = "Original A Data" -ORIGINAL_B = "Original B Data" -ABS_DIFF = "Abs. Difference" -RAW_DIFF = "Raw Difference" -HISTOGRAM = "Histogram" -MISMATCH = "Mismatch Areas" -SCATTER = "Scatter Plot" -HEX_PLOT = "Hex Plot" - -# a list of all the image types, for convenience -IMAGE_TYPES = [ORIGINAL_A, - ORIGINAL_B, - ABS_DIFF, - RAW_DIFF, - HISTOGRAM, - MISMATCH, - SCATTER, - HEX_PLOT - ] - -# a list of image types that require both the A and B data -COMPARISON_IMAGES = [ABS_DIFF, - RAW_DIFF, - HISTOGRAM, - MISMATCH, - SCATTER, - HEX_PLOT - ] - -# constants for possible types of data handling -SIMPLE_2D = "Simple Two Dimensional" -MAPPED_2D = "Mapped Two Dimensional" -ONLY_1D = "One Dimensional" - -# a list of data handling types, for conveinence -DATA_FORMS = [SIMPLE_2D, - MAPPED_2D, - ONLY_1D] - -# the default names that the model will try to select for the latitude and longitude -DEFAULT_LONGITUDE = 'pixel_longitude' -DEFAULT_LATITUDE = 'pixel_latitude' - class UnableToReadFile(Exception): """ An exception to be used when glance could not read a file @@ -133,7 +90,11 @@ class GlanceGUIModel (object) : self.epsilonPercent - the epsilon percent value for comparison self.llEpsilon - the epsilon used for judging the longitude and latitude data self.imageType - the image type that should be created when files are compared + self.colormap - the colormap to use for plotting (if one is needed) self.dataForm - the form the data should be considered to be + self.useSharedRange - True if images for the original data should be displayed in a range + that includeds the data from both files, False if not + self.dataListeners - objects that want to be notified when data changes self.errorHandlers - objects that want to be notified when there's a serious error """ @@ -153,7 +114,9 @@ class GlanceGUIModel (object) : self.epsilonPercent = None self.llEpsilon = 0.0 self.imageType = IMAGE_TYPES[0] + self.colormap = COLORMAP_NAMES[0] self.dataForm = SIMPLE_2D + self.useSharedRange = False # this represents all the people who want to hear about data updates # these people can register and will get data related messages @@ -293,7 +256,9 @@ class GlanceGUIModel (object) : dataListener.updateEpsilonPercent(self.epsilonPercent) dataListener.updateLLEpsilon(self.llEpsilon) dataListener.updateImageTypes(self.imageType, list=IMAGE_TYPES) + dataListener.updateColormaps(self.colormap, list=COLORMAP_NAMES) dataListener.updateDataForms(self.dataForm, list=DATA_FORMS) + dataListener.updateUseSharedRange(self.useSharedRange) def updateFileDataSelection (self, file_prefix, newVariableText=None, newOverrideValue=None, newFillValue=np.nan) : """ @@ -348,7 +313,8 @@ class GlanceGUIModel (object) : return self.fileData[file_prefix].var_data_cache[self.fileData[file_prefix].variable].select_fill_value() def updateSettingsDataSelection (self, newEpsilonValue=np.nan, newImageType=None, newDataForm=None, - newEpsilonPercent=np.nan, newllEpsilon=np.nan) : + newEpsilonPercent=np.nan, newllEpsilon=np.nan, + useSharedRangeForOriginals=None, newColormap=None) : """ someone has changed one or more of the general settings related data values @@ -382,6 +348,13 @@ class GlanceGUIModel (object) : self.imageType = newImageType didUpdate = True + # update the colormap if needed + if (newColormap is not None) and (newColormap != self.colormap) : + if newColormap in COLORMAP_NAMES : + LOG.debug("Setting colormap to: " + newColormap) + self.colormap = newColormap + didUpdate = True + # update the data form if needed if (newDataForm is not None) and (newDataForm != self.dataForm) : if newDataForm in DATA_FORMS : @@ -389,6 +362,13 @@ class GlanceGUIModel (object) : self.dataForm = newDataForm didUpdate = True + # update the shared range settings if needed + if (useSharedRangeForOriginals is not None) and (useSharedRangeForOriginals != self.useSharedRange) : + if useSharedRangeForOriginals is True or useSharedRangeForOriginals is False : + LOG.debug("Setting use shared range for originals to: " + str(useSharedRangeForOriginals)) + self.useSharedRange = useSharedRangeForOriginals + didUpdate = True + # let our data listeners know about any changes if didUpdate : for listener in self.dataListeners : @@ -396,7 +376,9 @@ class GlanceGUIModel (object) : listener.updateEpsilonPercent(self.epsilonPercent) listener.updateLLEpsilon(self.llEpsilon) listener.updateImageTypes(self.imageType) + listener.updateColormaps(self.colormap) listener.updateDataForms(self.dataForm) + listener.updateUseSharedRange(self.useSharedRange) def updateLonLatSelections (self, file_prefix, new_latitude_name=None, new_longitude_name=None) : """ diff --git a/pyglance/glance/gui_view.py b/pyglance/glance/gui_view.py index d584028b68aed3e0925058cc2f5463a8d0c2c0f1..2e7b6578b7e45b2ba9f164a9385cca61b249e29b 100644 --- a/pyglance/glance/gui_view.py +++ b/pyglance/glance/gui_view.py @@ -24,6 +24,39 @@ The GUI's most complex responsibility is keeping the user from entering garbage data into the UI. """ +class _DoubleOrNoneValidator (QtGui.QDoubleValidator) : + """ + This validator accepts doubles like it's parent class or None. + """ + + def __init__(self, parent=None) : + super(self.__class__, self).__init__(parent) + + def validate (self, value, pos ) : + """ + override our parent's validate method + """ + + if (value == "") or (value is None) or (value == "None") : + return (QtGui.QValidator.Acceptable, pos) + if (value == "N") or (value == "No") or (value == "Non") : + return (QtGui.QValidator.Intermediate, pos) + + return super(self.__class__, self).validate(value, pos) + + def fixup (self, value) : + """ + correct the input if needed + """ + + value = QtCore.QString(value) + + if (value == "") or (value == "N") or (value == "No") or (value == "Non") : + value = QtCore.QString("None") + + super(self.__class__, self).fixup(value) + + class GlanceGUIView (QtGui.QWidget) : """ The main view object that will create the gui that's visible to the user. @@ -74,11 +107,11 @@ class GlanceGUIView (QtGui.QWidget) : tempWidget.setLayout(self._build_data_tab()) self.tabWidget.addTab(tempWidget, "basic") - # TODO uncomment this to work on the settings tab # add a tab that allows more detailed, optional settings - #tempWidget = QtGui.QWidget() - #tempWidget.setLayout(self._build_settings_tab()) - #self.tabWidget.addTab(tempWidget, "settings") + tempWidget = QtGui.QWidget() + tempWidget.setLayout(self._build_settings_tab()) + self.tempHangOnToTab = tempWidget # TODO, remove when line below is uncommented + #self.tabWidget.addTab(tempWidget, "settings") #TODO, uncomment this when this tab is ready to go tempLayout = QtGui.QGridLayout() tempLayout.addWidget(self.tabWidget) @@ -115,7 +148,8 @@ class GlanceGUIView (QtGui.QWidget) : layoutToUse.addWidget(QtGui.QLabel("epsilon:"), currentRow, 0) self.epsilonWidget = QtGui.QLineEdit() self.epsilonWidget.setToolTip("Maximum acceptible difference between the variable data in the two files.") - tempValidator = QtGui.QDoubleValidator(self.epsilonWidget) + tempValidator = _DoubleOrNoneValidator(self.epsilonWidget) + #tempValidator = QtGui.QDoubleValidator(self.epsilonWidget) tempValidator.setBottom(0.0) # only accept positive epsilons tempValidator.setNotation(QtGui.QDoubleValidator.StandardNotation) self.epsilonWidget.setValidator(tempValidator) @@ -128,7 +162,8 @@ class GlanceGUIView (QtGui.QWidget) : layoutToUse.addWidget(QtGui.QLabel("epsilon percent:"), currentRow, 0) self.epsilonPerWidget = QtGui.QLineEdit() self.epsilonPerWidget.setToolTip("Maximum acceptible difference between the variable data in terms of % of each data point in the A file.") - tempValidator = QtGui.QDoubleValidator(self.epsilonPerWidget) + tempValidator = _DoubleOrNoneValidator(self.epsilonPerWidget) + #tempValidator = QtGui.QDoubleValidator(self.epsilonPerWidget) tempValidator.setBottom(0.0) # only accept positive epsilon percents tempValidator.setNotation(QtGui.QDoubleValidator.StandardNotation) self.epsilonPerWidget.setValidator(tempValidator) @@ -252,9 +287,13 @@ class GlanceGUIView (QtGui.QWidget) : layoutToUse = QtGui.QGridLayout() currentRow = 0 - # add the filter entry areas - currentRow = self._add_filter_controls("A", layoutToUse, currentRow) - currentRow = self._add_filter_controls("B", layoutToUse, currentRow) + # add the drop down for selecting a custom color map + layoutToUse.addWidget(QtGui.QLabel("Color map:"), currentRow, 0) + self.colormapDropDown = QtGui.QComboBox() + self.colormapDropDown.activated.connect(self.colormapSelected) + layoutToUse.addWidget(self.colormapDropDown, currentRow, 1, 1, 2) + + currentRow = currentRow + 1 # add the drop down for selecting the data display form layoutToUse.addWidget(QtGui.QLabel("Data Form:"), currentRow, 0) @@ -269,7 +308,7 @@ class GlanceGUIView (QtGui.QWidget) : showOriginalsInSameRange.setToolTip("Check to constrain the colorbar range of the Original A and " + "Original B data plots to be the same.\nThe range will include all data in both A and B.") showOriginalsInSameRange.setDisabled(False) - #showOriginalsInSameRange.stateChanged.connect(TODO) + showOriginalsInSameRange.stateChanged.connect(self.reportOriginalsRangeToggled) self.useSameRangeWidget = showOriginalsInSameRange layoutToUse.addWidget(showOriginalsInSameRange, currentRow, 1, 1, 2) @@ -281,23 +320,23 @@ class GlanceGUIView (QtGui.QWidget) : currentRow = self._add_lon_lat_controls("A", layoutToUse, currentRow) currentRow = self._add_lon_lat_controls("B", layoutToUse, currentRow) + # add the filtering related controls + currentRow = self._add_filter_controls("A", layoutToUse, currentRow) + currentRow = self._add_filter_controls("B", layoutToUse, currentRow) + # add box to enter lon/lat epsilon layoutToUse.addWidget(QtGui.QLabel("lon/lat epsilon:"), currentRow, 0) llepsilonWidget = QtGui.QLineEdit() self.llepsilonWidget = llepsilonWidget llepsilonWidget.setToolTip("Maximum acceptible difference between the longitudes or latitudes in the two files.") - tempValidator = QtGui.QDoubleValidator(llepsilonWidget) + tempValidator = _DoubleOrNoneValidator(llepsilonWidget) + #tempValidator = QtGui.QDoubleValidator(llepsilonWidget) tempValidator.setBottom(0.0) # only accept positive epsilons tempValidator.setNotation(QtGui.QDoubleValidator.StandardNotation) llepsilonWidget.setValidator(tempValidator) llepsilonWidget.editingFinished.connect(self.reportLLEpsilonChanged) layoutToUse.addWidget(llepsilonWidget, currentRow, 1, 1, 2) - # TODO, possible enhancements for the future - # add filters for lon/lat? - # allow lon/lat to be loaded from a different file? - # add buttons to let you plot lon/lat specific errors? - return layoutToUse def _add_filter_controls (self, file_prefix, grid_layout, current_row) : @@ -308,25 +347,60 @@ class GlanceGUIView (QtGui.QWidget) : return the next free current_row number when finished adding widgets """ - # add the entry to name the specific function - grid_layout.addWidget(QtGui.QLabel(str(file_prefix) + " filter function:"), current_row, 0) - functionEntry = QtGui.QLineEdit() - functionEntry.setToolTip("Enter python function to use for filtering " + str(file_prefix) + " data here." - + "\nThis function should work in the form function_name(data_to_filter)" - + "\n and you should enter only the function_name here.") - #functionEntry.editingFinished.connect(TODO) - grid_layout.addWidget(functionEntry, current_row, 1, 1, 3) - self.widgetInfo[file_prefix]["filter_function"] = functionEntry + # label the specific section + grid_layout.addWidget(QtGui.QLabel(str(file_prefix) + " filtering:"), current_row, 0) + + current_row = current_row + 1 + + # add something to give range restrictions + restrictToRangeCheckbox = QtGui.QCheckBox("restrict data to range:") + restrictToRangeCheckbox.setToolTip("When checked data outside the entered range will be treated as the fill value.") + restrictToRangeCheckbox.setDisabled(False) + restrictToRangeCheckbox.stateChanged.connect(partial(self.reportRestrictRangeToggled, file_prefix=file_prefix)) + self.widgetInfo[file_prefix]["doRestrictRangeCheckbox"] = restrictToRangeCheckbox + grid_layout.addWidget(restrictToRangeCheckbox, current_row, 1, 1, 2) + + current_row = current_row + 1 + + # add the areas to enter the range boundaries + + # first add the min + minRangeValue = QtGui.QLineEdit() + minRangeValue.setToolTip("Minimum acceptable data value for " + str(file_prefix) + " data") + tempValidator = QtGui.QDoubleValidator(minRangeValue) + # at some point the bottom and top of the ranges will need to be set on this validator + # do I need to save it or can I recover it from the widget? + tempValidator.setNotation(QtGui.QDoubleValidator.StandardNotation) + minRangeValue.setValidator(tempValidator) + minRangeValue.editingFinished.connect(partial(self.reportMinRangeValChanged, file_prefix=file_prefix)) + self.widgetInfo[file_prefix]["minRangeRestriction"] = minRangeValue + grid_layout.addWidget(minRangeValue, current_row, 1) #, 1, 2) + + # now a label to clarify the relationship between the entry areas + grid_layout.addWidget(QtGui.QLabel("to"), current_row, 2) + + # first add the min + maxRangeValue = QtGui.QLineEdit() + maxRangeValue.setToolTip("Maximum acceptable data value for " + str(file_prefix) + " data") + tempValidator = QtGui.QDoubleValidator(maxRangeValue) + # at some point the bottom and top of the ranges will need to be set on this validator + # do I need to save it or can I recover it from the widget? + tempValidator.setNotation(QtGui.QDoubleValidator.StandardNotation) + maxRangeValue.setValidator(tempValidator) + maxRangeValue.editingFinished.connect(partial(self.reportMaxRangeValChanged, file_prefix=file_prefix)) + self.widgetInfo[file_prefix]["maxRangeRestriction"] = maxRangeValue + grid_layout.addWidget(maxRangeValue, current_row, 3) #1, 1, 2) current_row = current_row + 1 - grid_layout.addWidget(QtGui.QLabel(str(file_prefix) + " filter setup: "), current_row, 0) - setupCodeEntry = QtGui.QTextEdit() - setupCodeEntry.setToolTip("Enter additional python code needed to set up the filter function for " + str(file_prefix) + ".") - setupCodeEntry.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Preferred) # TODO, this is not working! - #setupCodeEntry.editingFinished.connect(TODO) - self.widgetInfo[file_prefix]["filter_setup"] = setupCodeEntry - grid_layout.addWidget(setupCodeEntry, current_row, 1, 1, 3) + # add a check box to filter AWIPS data + isAWIPSdata = QtGui.QCheckBox("correct for AWIPS data types") + isAWIPSdata.setToolTip("AWIPS files use signed numbers to hold unsigned data. " + + "Check this box if you are plotting an AWIPS file and would like your data plotted in the original, unsigned form.") + isAWIPSdata.setDisabled(False) + isAWIPSdata.stateChanged.connect(partial(self.reportIsAWIPSToggled, file_prefix=file_prefix)) + self.widgetInfo[file_prefix]["isAWIPScheckbox"] = isAWIPSdata + grid_layout.addWidget(isAWIPSdata, current_row, 1, 1, 2) current_row = current_row + 1 @@ -501,6 +575,17 @@ class GlanceGUIView (QtGui.QWidget) : for listener in self.userUpdateListeners : listener.userSelectedImageType(newImageType) + def colormapSelected (self) : + """ + the user selected a colormap, let our user update listeners know + """ + + newColormap = self.colormapDropDown.currentText() + + # let the listeners know + for listener in self.userUpdateListeners : + listener.userSelectedColormap(newColormap) + def reportDataFormSelected (self) : """ the user selected a new data form, so let our user update listeners know that @@ -512,6 +597,51 @@ class GlanceGUIView (QtGui.QWidget) : for listener in self.userUpdateListeners : listener.userSelectedDataForm(newDataForm) + def reportOriginalsRangeToggled (self) : + """ + the user toggled whether or not they want their original plots to be + displayed in a shared range + """ + + # this must be recorded before we tamper with the focus, because that will + # trigger other events that may erase this information temporarily + shouldUseSharedRange = self.useSameRangeWidget.isChecked() + + # first we need to clean up focus in case it's in one of the line-edit boxes + self.setFocus() + + # let our listeners know the user changed an overload setting + for listener in self.userUpdateListeners : + listener.userToggledSharedRange(shouldUseSharedRange) + + def reportRestrictRangeToggled (self, file_prefix=None) : + """ + the user toggled the "restrict data to range" check box + """ + + print ("*** restrict range checkbox toggled TODO " + file_prefix) + + def reportMinRangeValChanged (self, file_prefix=None) : + """ + the user changed the minimum value for the acceptable range + """ + + print ("*** minimum range value changed TODO " + file_prefix) + + def reportMaxRangeValChanged (self, file_prefix=None) : + """ + the user changed the maximum value for the acceptable range + """ + + print ("*** maximum range value changed TODO " + file_prefix) + + def reportIsAWIPSToggled (self, file_prefix=None) : + """ + the user toggled the "correct for AWIPS data types" check box + """ + + print ("*** AWIPS check box toggled TODO " + file_prefix) + def reportDisplayStatsClicked (self) : """ the user clicked the display stats button @@ -622,8 +752,6 @@ class GlanceGUIView (QtGui.QWidget) : if a list is given, then replace the list of options that are being displayed for that file. """ - """ TODO, uncomment once that tab is set up - # if we got a new list, set up the appropriate drop down lists if lonlatList is not None : @@ -640,8 +768,6 @@ class GlanceGUIView (QtGui.QWidget) : # set the selected longitude tempPosition = self.widgetInfo[filePrefix]['lonName'].findText(newLongitude) self.widgetInfo[filePrefix]['lonName'].setCurrentIndex(tempPosition) - - """ def updateEpsilon (self, epsilon) : """ @@ -662,7 +788,7 @@ class GlanceGUIView (QtGui.QWidget) : update the epsilon for longitude and latitude displayed to the user """ - #self.llepsilonWidget.setText(str(newLonLatEpsilon)) TODO, uncomment once that tab is set up + self.llepsilonWidget.setText(str(newLonLatEpsilon)) def updateImageTypes (self, imageType, list=None) : """ @@ -679,13 +805,27 @@ class GlanceGUIView (QtGui.QWidget) : tempPosition = self.imageSelectionDropDown.findText(imageType) self.imageSelectionDropDown.setCurrentIndex(tempPosition) + def updateColormaps(self, colormap, list=None) : + """ + update the colormap that's selected, + if the list is given, clear and reset the list of possible colormaps + """ + + # replace the list if needed + if list is not None : + self.colormapDropDown.clear() + self.colormapDropDown.addItems(list) + + #change the currently selected colormap + tempPosition = self.colormapDropDown.findText(colormap) + self.colormapDropDown.setCurrentIndex(tempPosition) + def updateDataForms(self, dataForm, list=None) : """ update the data form that's selected, if the list is given, clear and reset the list of possible data forms """ - """ TODO, uncomment once that tab is set up # replace the list if needed if list is not None : self.dataDisplayFormDropDown.clear() @@ -694,7 +834,13 @@ class GlanceGUIView (QtGui.QWidget) : # change the currently selected data form tempPosition = self.dataDisplayFormDropDown.findText(dataForm) self.dataDisplayFormDropDown.setCurrentIndex(tempPosition) + + def updateUseSharedRange(self, doUseSharedRange) : """ + update whether or not a shared range should be used + """ + + self.useSameRangeWidget.setChecked(doUseSharedRange) ################# end data model update related methods ################# diff --git a/pyglance/glance/io.py b/pyglance/glance/io.py index 4f24b823eedc32977aba5a548964f84527529444..a3b57821a77eb14e059e49c49a6a5faec4ddd0fd 100644 --- a/pyglance/glance/io.py +++ b/pyglance/glance/io.py @@ -57,6 +57,10 @@ UNITS_CONSTANT = "units" fillValConst1 = '_FillValue' fillValConst2 = 'missing_value' +ADD_OFFSET_STR = 'add_offset' +SCALE_FACTOR_STR = 'scale_factor' +SCALE_METHOD_STR = 'scaling_method' + class IOUnimplimentedError(Exception): """ The exception raised when a requested io operation is not yet available. @@ -68,24 +72,128 @@ class IOUnimplimentedError(Exception): def __str__(self): return self.msg -class hdf(SD): +class CaseInsensitiveAttributeCache (object) : + """ + A cache of attributes for a single file and all of it's variables. + This cache is considered uncased, it will store all attributes it caches + in lower case and will lower case any strings it is asked to search for + in the cache. + When variable or global attribute sets are not yet loaded and something + from that part of the file is requested the cache will transparently load + attributes from the file behind the scenes and build the cache for that + part of the file. + """ + + def __init__(self, fileObject) : + """ + set up the empty cache and hang on to the file object we'll be caching + """ + + self.fileToCache = fileObject + self.globalAttributesLower = None + self.variableAttributesLower = { } + + def _load_global_attributes_if_needed (self) : + """ + load up the global attributes if they need to be cached + """ + + # load the attributes from the file if they aren't cached + if self.globalAttributesLower is None : + LOG.debug ("Loading file global attributes into case-insensitive cache.") + tempAttrs = self.fileToCache.get_global_attributes(caseInsensitive=False) + self.globalAttributesLower = dict((k.lower(), v) for k, v in tempAttrs.items()) + + def _load_variable_attributes_if_needed (self, variableName) : + """ + load up the variable attributes if they need to be cached + """ + + # make a lower cased version of the variable name + tempVariableName = variableName.lower() + + # load the variable's attributes from the file if they aren't cached + if tempVariableName not in self.variableAttributesLower.keys() : + LOG.debug ("Loading attributes for variable \"" + variableName + "\" into case-insensitive cache.") + tempAttrs = self.fileToCache.get_variable_attributes(variableName, caseInsensitive=False) + # now if there are any attributes, make a case insensitive version + self.variableAttributesLower[tempVariableName] = dict((k.lower(), v) for k, v in tempAttrs.items()) + + def get_variable_attribute (self, variableName, attributeName) : + """ + get the specified attribute for the specified variable, + if this variable's attributes have not yet been loaded + they will be loaded and cached + """ + + self._load_variable_attributes_if_needed(variableName) + + toReturn = None + tempVariableName = variableName.lower() + tempAttributeName = attributeName.lower() + if (tempVariableName in self.variableAttributesLower) and (tempAttributeName in self.variableAttributesLower[tempVariableName]) : + toReturn = self.variableAttributesLower[tempVariableName][tempAttributeName] + else: + LOG.debug ("Attribute \"" + attributeName + "\" was not present for variable \"" + variableName + "\".") + + return toReturn + + def get_variable_attributes (self, variableName) : + """ + get the variable attributes for the variable name given + """ + + self._load_variable_attributes_if_needed(variableName) + + toReturn = self.variableAttributesLower[variableName.lower()] if (variableName.lower() in self.variableAttributesLower) else None + + return toReturn + + def get_global_attribute (self, attributeName) : + """ + get a global attribute with the given name + """ + + self._load_global_attributes_if_needed() + + toReturn = self.globalAttributesLower[attributeName.lower()] if (attributeName.lower() in self.globalAttributesLower) else None + + return toReturn + + def get_global_attributes (self) : + """ + get the global attributes, + """ + + self._load_global_attributes_if_needed() + + toReturn = self.globalAttributesLower + + return toReturn + +class hdf (object): """wrapper for HDF4 dataset for comparison __call__ yields sequence of variable names __getitem__ returns individual variables ready for slicing to numpy arrays """ + _hdf = None + def __init__(self, filename, allowWrite=False): + if pyhdf is None: LOG.error('pyhdf is not installed and is needed in order to read hdf4 files') assert(pyhdf is not None) mode = SDC.READ if allowWrite: mode = mode | SDC.WRITE - super(self.__class__,self).__init__(filename, mode) + + self._hdf = SD(filename, mode) + self.attributeCache = CaseInsensitiveAttributeCache(self) def __call__(self): "yield names of variables to be compared" - return self.datasets().keys() + return self._hdf.datasets().keys() # this returns a numpy array with a copy of the full, scaled # data for this variable, if the data type must be changed to allow @@ -98,32 +206,26 @@ class hdf(SD): data_type = None scaling_method = None - #print ("***** getting " + name + " from file") - # get the variable object and use it to # get our raw data and scaling info variable_object = self.get_variable_object(name) - #print ("****** variable object gotten") raw_data_copy = variable_object[:] - #print ("****** raw data loaded") try : # TODO, this currently won't work with geocat data, work around it for now scale_factor, scale_factor_error, add_offset, add_offset_error, data_type = SDS.getcal(variable_object) except HDF4Error: - # load just the scale factor and add offset - temp_attributes = variable_object.attributes() - if ('add_offset' in temp_attributes) : - add_offset = temp_attributes['add_offset'] + # load just the scale factor and add offset information by hand + temp = self.attributeCache.get_variable_attributes(name) + if ADD_OFFSET_STR in temp.keys() : + add_offset = temp[ADD_OFFSET_STR] data_type = np.dtype(type(add_offset)) - if ('scale_factor' in temp_attributes) : - scale_factor = temp_attributes['scale_factor'] + if SCALE_FACTOR_STR in temp.keys() : + scale_factor = temp[SCALE_FACTOR_STR] data_type = np.dtype(type(scale_factor)) - if ('scaling_method' in temp_attributes) : - scaling_method = temp_attributes['scaling_method'] + if SCALE_METHOD_STR in temp.keys() : + scaling_method = temp[SCALE_METHOD_STR] SDS.endaccess(variable_object) - #print ("***** scaling information loaded") - # don't do lots of work if we don't need to scale things if (scale_factor == 1.0) and (add_offset == 0.0) : return raw_data_copy @@ -151,24 +253,17 @@ class hdf(SD): missing_mask[raw_data_copy == missing_val] = True # create the scaled version of the data - scaled_data_copy = np.array(raw_data_copy, dtype=data_type) - #scaled_data_copy[~missing_mask] = (scaled_data_copy[~missing_mask] - add_offset) * scale_factor #TODO, type truncation issues? + scaled_data_copy = np.array(raw_data_copy, dtype=data_type) scaled_data_copy[~missing_mask] = (scaled_data_copy[~missing_mask] * scale_factor) + add_offset #TODO, type truncation issues? return scaled_data_copy def get_variable_object(self, name): - return self.select(name) + return self._hdf.select(name) def missing_value(self, name): - variable_object = self.select(name) - to_return = None - if hasattr(variable_object, fillValConst1) : - to_return = getattr(variable_object, fillValConst1, None) - SDS.endaccess(variable_object) - - return to_return + return self.get_attribute(name, fillValConst1) def create_new_variable(self, variablename, missingvalue=None, data=None, variabletocopyattributesfrom=None): """ @@ -193,59 +288,88 @@ class hdf(SD): return - def get_variable_attributes (self, variableName) : + def get_variable_attributes (self, variableName, caseInsensitive=True) : """ returns all the attributes associated with a variable name """ - return self.get_variable_object(variableName).attributes() + toReturn = None + if caseInsensitive : + toReturn = self.attributeCache.get_variable_attributes(variableName) + else : + toReturn = self.get_variable_object(variableName).attributes() + + return toReturn - def get_attribute(self, variableName, attributeName) : + def get_attribute(self, variableName, attributeName, caseInsensitive=True) : """ returns the value of the attribute if it is available for this variable, or None """ toReturn = None - temp_attributes = self.get_variable_attributes(variableName) - if attributeName in temp_attributes : - toReturn = temp_attributes[attributeName] + if caseInsensitive : + toReturn = self.attributeCache.get_variable_attribute(variableName, attributeName) + else : + temp_attributes = self.get_variable_attributes(variableName, caseInsensitive=False) + + if attributeName in temp_attributes : + toReturn = temp_attributes[attributeName] return toReturn - def get_global_attribute(self, attributeName) : + def get_global_attributes(self, caseInsensitive=True) : + """ + get a list of all the global attributes for this file or None + """ + + toReturn = None + + if caseInsensitive : + self.attributeCache.get_global_attributes() + else : + toReturn = self._hdf.attributes() + + return toReturn + + def get_global_attribute(self, attributeName, caseInsensitive=True) : """ returns the value of a global attribute if it is available or None """ toReturn = None - if attributeName in self.attributes() : - toReturn = self.attributes()[attributeName] + if caseInsensitive : + toReturn = self.attributeCache.get_global_attribute(attributeName) + else : + if attributeName in self._hdf.attributes() : + toReturn = self._hdf.attributes()[attributeName] return toReturn -class nc(CDF): +class nc (object): """wrapper for NetCDF3/4/opendap dataset for comparison __call__ yields sequence of variable names __getitem__ returns individual variables ready for slicing to numpy arrays """ + _nc = None + def __init__(self, filename, allowWrite=False): if pycdf is None: LOG.error('pycdf is not installed and is needed in order to read NetCDF files') assert(pycdf is not None) - mode = NC.NOWRITE if allowWrite : mode = NC.WRITE - super(self.__class__,self).__init__(filename, mode) + self._nc = CDF(filename, mode) + self.attributeCache = CaseInsensitiveAttributeCache(self) def __call__(self): "yield names of variables to be compared" - return self.variables().keys() + return self._nc.variables().keys() # this returns a numpy array with a copy of the full, scaled # data for this variable, if the data type must be changed to allow @@ -262,11 +386,12 @@ class nc(CDF): variable_object = self.get_variable_object(name) raw_data_copy = variable_object[:] # load the scale factor and add offset - temp_attributes = variable_object.attributes() - if ('scale_factor' in temp_attributes) : - scale_factor = temp_attributes['scale_factor'] - if ('add_offset' in temp_attributes) : - add_offset = temp_attributes['add_offset'] + + temp = self.attributeCache.get_variable_attributes(name) + if SCALE_FACTOR_STR in temp.keys() : + scale_factor = temp[SCALE_FACTOR_STR] + if ADD_OFFSET_STR in temp.keys() : + add_offset = temp[ADD_OFFSET_STR] # todo, does cdf have an equivalent of endaccess to close the variable? # don't do lots of work if we don't need to scale things @@ -280,17 +405,27 @@ class nc(CDF): # create the scaled version of the data scaled_data_copy = np.array(raw_data_copy, dtype=data_type) - #scaled_data_copy[~missing_mask] = (scaled_data_copy[~missing_mask] - add_offset) * scale_factor #TODO, type truncation issues? scaled_data_copy[~missing_mask] = (scaled_data_copy[~missing_mask] * scale_factor) + add_offset #TODO, type truncation issues? return scaled_data_copy def get_variable_object(self, name): - return self.var(name) + return self._nc.var(name) def missing_value(self, name): - variable_object = self.var(name) + toReturn = None + + temp = self.attributeCache.get_variable_attribute(name, fillValConst1) + if temp is not None : + toReturn = temp + else : + temp = self.attributeCache.get_variable_attribute(name, fillValConst2) + if temp is not None : + toReturn = temp + + """ todo, why was the getattr method being used with 3 params? I can't find this documented anywhere... + variable_object = self._nc.var(name) to_return = None if hasattr(variable_object, fillValConst1) \ @@ -298,8 +433,9 @@ class nc(CDF): hasattr(variable_object, fillValConst2) : to_return = getattr(variable_object, fillValConst1, getattr(variable_object, fillValConst2, None)) + """ - return to_return + return toReturn def create_new_variable(self, variablename, missingvalue=None, data=None, variabletocopyattributesfrom=None): """ @@ -310,10 +446,10 @@ class nc(CDF): be created """ - self.redef() + self._nc.redef() # if the variable already exists, stop with a warning - if variablename in self.variables().keys() : + if variablename in self._nc.variables().keys() : LOG.warn("New variable name requested (" + variablename + ") is already present in file. " + "Skipping generation of new variable.") return None @@ -340,14 +476,14 @@ class nc(CDF): dimensions = [ ] dimensionNum = 0 for dimSize in data.shape : - dimensions.append(self.def_dim(variablename + '-index' + str(dimensionNum), dimSize)) + dimensions.append(self._nc.def_dim(variablename + '-index' + str(dimensionNum), dimSize)) dimensionNum = dimensionNum + 1 # create the new variable #print('variable name: ' + variablename) #print('data type: ' + str(dataType)) #print('dimensions: ' + str(dimensions)) - newVariable = self.def_var(variablename, dataType, tuple(dimensions)) + newVariable = self._nc.def_var(variablename, dataType, tuple(dimensions)) # if a missing value was given, use that if missingvalue is not None : @@ -360,7 +496,7 @@ class nc(CDF): for attribute in attributes.keys() : newVariable.__setattr__(attribute, attributes[attribute]) - self.enddef() + self._nc.enddef() # if data was given, use that if data is not None : @@ -375,47 +511,76 @@ class nc(CDF): """ variableObject = self.get_variable_object(variableName) - self.redef() + self._nc.redef() variableObject.__setattr__(newAttributeName, newAttributeValue) + # TODO, this will cause our attribute cache to be wrong! + # TODO, for now, brute force clear the cache + self.attributeCache = CaseInsensitiveAttributeCache(self) - self.enddef() + self._nc.enddef() return - def get_variable_attributes (self, variableName) : + def get_variable_attributes (self, variableName, caseInsensitive=True) : """ returns all the attributes associated with a variable name """ - return self.get_variable_object(variableName).attributes() + toReturn = None + + if caseInsensitive : + toReturn = self.attributeCache.get_variable_attributes(variableName) + else : + toReturn = self.get_variable_object(variableName).attributes() + + return toReturn - def get_attribute(self, variableName, attributeName) : + def get_attribute(self, variableName, attributeName, caseInsensitive=True) : """ returns the value of the attribute if it is available for this variable, or None """ toReturn = None - temp_attributes = self.get_variable_attributes(variableName) + if caseInsensitive : + toReturn = self.attributeCache.get_variable_attribute(variableName, attributeName) + else : + temp_attributes = self.get_variable_attributes(variableName, caseInsensitive=False) + + if attributeName in temp_attributes : + toReturn = temp_attributes[attributeName] + + return toReturn + + def get_global_attributes(self, caseInsensitive=True) : + """ + get a list of all the global attributes for this file or None + """ + + toReturn = None - if attributeName in temp_attributes : - toReturn = temp_attributes[attributeName] + if caseInsensitive : + self.attributeCache.get_global_attributes() + else : + toReturn = self._nc.attributes() return toReturn - def get_global_attribute(self, attributeName) : + def get_global_attribute(self, attributeName, caseInsensitive=True) : """ returns the value of a global attribute if it is available or None """ toReturn = None - if attributeName in self.attributes() : - toReturn = self.attributes()[attributeName] + if caseInsensitive : + toReturn = self.attributeCache.get_global_attribute(attributeName) + else : + if attributeName in self._nc.attributes() : + toReturn = self._nc.attributes()[attributeName] return toReturn - nc4 = nc cdf = nc @@ -428,6 +593,8 @@ class h5(object): _h5 = None def __init__(self, filename, allowWrite=False): + self.attributeCache = CaseInsensitiveAttributeCache(self) + mode = 'r' if allowWrite : mode = 'r+' @@ -484,10 +651,11 @@ class h5(object): #print ('*************************') # load the scale factor and add offset - if ('scale_factor' in variable_object.attrs) : - scale_factor = variable_object.attrs['scale_factor'] - if ('add_offset' in variable_object.attrs) : - add_offset = variable_object.attrs['add_offset'] + temp = self.attributeCache.get_variable_attributes(name) + if (SCALE_FACTOR_STR in temp.keys()) : + scale_factor = temp[SCALE_FACTOR_STR] + if (ADD_OFFSET_STR in temp.keys()) : + add_offset = temp[ADD_OFFSET_STR] # todo, does cdf have an equivalent of endaccess to close the variable? # don't do lots of work if we don't need to scale things @@ -501,7 +669,6 @@ class h5(object): # create the scaled version of the data scaled_data_copy = np.array(raw_data_copy, dtype=data_type) - #scaled_data_copy[~missing_mask] = (scaled_data_copy[~missing_mask] - add_offset) * scale_factor #TODO, type truncation issues? scaled_data_copy[~missing_mask] = (scaled_data_copy[~missing_mask] * scale_factor) + add_offset #TODO, type truncation issues? return scaled_data_copy @@ -547,41 +714,66 @@ class h5(object): return - def get_variable_attributes (self, variableName) : + def get_variable_attributes (self, variableName, caseInsensitive=True) : """ returns all the attributes associated with a variable name """ - return self.get_variable_object(variableName).attrs + toReturn = None + + if caseInsensitive : + toReturn = self.attributeCache.get_variable_attributes(variableName) + else : + toReturn = self.get_variable_object(variableName).attrs + + return toReturn - def get_attribute(self, variableName, attributeName) : + def get_attribute(self, variableName, attributeName, caseInsensitive=True) : """ returns the value of the attribute if it is available for this variable, or None """ toReturn = None - temp_attrs = self.get_variable_attributes(variableName) + if caseInsensitive : + toReturn = self.attributeCache.get_variable_attribute(variableName, attributeName) + else : + temp_attrs = self.get_variable_attributes(variableName, caseInsensitive=False) + + if (attributeName in temp_attrs) : + toReturn = temp_attrs[attributeName] + + return toReturn + + def get_global_attributes(self, caseInsensitive=True) : + """ + get a list of all the global attributes for this file or None + """ + + toReturn = None - if (attributeName in temp_attrs) : - toReturn = temp_attrs[attributeName] + if caseInsensitive : + self.attributeCache.get_global_attributes() + else : + toReturn = self._h5.attrs return toReturn - def get_global_attribute(self, attributeName) : + def get_global_attribute(self, attributeName, caseInsensitive=True) : """ returns the value of a global attribute if it is available or None """ toReturn = None - if attributeName in self._h5.attrs : - toReturn = self._h5.attrs[attributeName] + if caseInsensitive : + toReturn = self.attributeCache.get_global_attribute(attributeName) + else : + if attributeName in self._h5.attrs : + toReturn = self._h5.attrs[attributeName] return toReturn - - class aeri(object): """wrapper for AERI RNC/SUM/CXS/etc datasets """ @@ -667,7 +859,7 @@ class aeri(object): return - def get_variable_attributes (self, variableName) : + def get_variable_attributes (self, variableName, caseInsensitive=True) : """ returns all the attributes associated with a variable name """ @@ -678,7 +870,7 @@ class aeri(object): return toReturn - def get_attribute(self, variableName, attributeName) : + def get_attribute(self, variableName, attributeName, caseInsensitive=True) : """ returns the value of the attribute if it is available for this variable, or None """ @@ -689,7 +881,19 @@ class aeri(object): return toReturn - def get_global_attribute(self, attributeName) : + def get_global_attributes(self, caseInsensitive=True) : + """ + get a list of all the global attributes for this file or None + """ + + toReturn = None + + # TODO + LOG.warn('Glance does not yet support attribute retrieval in AERI files. None will be used.') + + return toReturn + + def get_global_attribute(self, attributeName, caseInsensitive=True) : """ returns the value of a global attribute if it is available or None """ @@ -782,7 +986,7 @@ class jpss_adl(object): return - def get_variable_attributes (self, variableName) : + def get_variable_attributes (self, variableName, caseInsensitive=True) : """ returns all the attributes associated with a variable name """ @@ -793,7 +997,7 @@ class jpss_adl(object): return toReturn - def get_attribute(self, variableName, attributeName) : + def get_attribute(self, variableName, attributeName, caseInsensitive=True) : """ returns the value of the attribute if it is available for this variable, or None """ @@ -804,7 +1008,7 @@ class jpss_adl(object): return toReturn - def get_global_attribute(self, attributeName) : + def get_global_attribute(self, attributeName, caseInsensitive=True) : """ returns the value of a global attribute if it is available or None """ @@ -815,6 +1019,18 @@ class jpss_adl(object): LOG.warn('Glance does not yet support attribute retrieval in JPSS ADL files. None will be used.') return toReturn + + def get_global_attributes(self, caseInsensitive=True) : + """ + get a list of all the global attributes for this file or None + """ + + toReturn = None + + # TODO + LOG.warn('Glance does not yet support attribute retrieval in JPSS ADL files. None will be used.') + + return toReturn diff --git a/pyglance/glance/plotcreatefns.py b/pyglance/glance/plotcreatefns.py index 0e8c65143739d1781f0f06727544cb1a4b93fc33..14904c7643c9984dadff98f2ed067ddfe1026def 100644 --- a/pyglance/glance/plotcreatefns.py +++ b/pyglance/glance/plotcreatefns.py @@ -1388,7 +1388,7 @@ class InspectMappedContourPlotFunctionFactory (PlottingFunctionFactory) : assert(lonLatDataDict is not None) assert(goodInAMask is not None) - print ("lon lat dictionary form: " + str(lonLatDataDict)) + #print ("lon lat dictionary form: " + str(lonLatDataDict)) # figure out which part of the earth is visible and construct a basemap using that fullAxis, baseMapInstance = _make_axis_and_basemap({'a':lonLatDataDict}, diff --git a/pyglance/glance/stats.py b/pyglance/glance/stats.py index ed53d999669e583137cbace461dfcf1574c1b2f9..2838a015651dc7f3ce06e45c79de1f9f01233cf8 100644 --- a/pyglance/glance/stats.py +++ b/pyglance/glance/stats.py @@ -1058,7 +1058,8 @@ class StatisticalInspectionAnalysis (StatisticalData) : # -------------------------- documentation ----------------------------- # TODO, can this be moved? -STATISTICS_DOC_STR = '\n'.join( '%s:\n %s' % x for x in sorted(list(StatisticalAnalysis.doc_strings().items())) ) + '\n' +STATISTICS_DOC_STR = '\n'.join( '%s:\n %s' % x for x in sorted(list( StatisticalAnalysis.doc_strings().items())) ) + '\n' +INSP_STATISTICS_DOC_STR = '\n'.join( '%s:\n %s' % x for x in sorted(list(StatisticalInspectionAnalysis.doc_strings().items())) ) + '\n' if __name__=='__main__': import doctest