diff --git a/pyglance/glance/compare.py b/pyglance/glance/compare.py
index fd149dd1f004a923f199b906cd83056ff21679bc..01669a602b34f4e9712772fbf8f3684cf619434d 100644
--- a/pyglance/glance/compare.py
+++ b/pyglance/glance/compare.py
@@ -1189,9 +1189,15 @@ def stats_library_call(afn, bfn, var_list=None,
LOG.warn(name + " is of a type that cannot be loaded using current file handling libraries included with Glance." +
" Skipping " + name + ".")
continue
-
- aData = aFile[name]
- bData = bFile[name]
+
+ try :
+ aData = aFile[name]
+ bData = bFile[name]
+ except io.IONonnumericalTypeError as bad_data_error :
+ LOG.error("Skipping variable %s because it is of a non-numerical type "
+ "(may indicate array of variable-length strings): %s" % (name, repr(bad_data_error)))
+ continue
+
if missing is None:
amiss = aFile.missing_value(name)
bmiss = bFile.missing_value(name)
diff --git a/pyglance/glance/gui_controller.py b/pyglance/glance/gui_controller.py
index afb6345742a751dabf85b24da6c0068e4f7c21c7..2c4ab36de66216e534c6ef211dad0a24f81f356a 100644
--- a/pyglance/glance/gui_controller.py
+++ b/pyglance/glance/gui_controller.py
@@ -18,6 +18,7 @@ import glance.gui_figuremanager as gui_figs
from glance.gui_constants import A_CONST, B_CONST
from glance.data import IncompatableDataObjects
+from glance.io import IONonnumericalTypeError
LOG = logging.getLogger(__name__)
@@ -85,8 +86,8 @@ class GlanceGUIController (object) :
try :
self.model.loadNewFile(file_prefix, new_file_path)
- except (gui_model.UnableToReadFile, ValueError) as utrf :
- self.handleWarning(str(utrf))
+ except (gui_model.UnableToReadFile, ValueError, IONonnumericalTypeError) as bad :
+ self.handleWarning(str(bad))
def userSelectedVariable (self, file_prefix, newSelection) :
"""
@@ -95,8 +96,8 @@ class GlanceGUIController (object) :
try :
self.model.updateFileDataSelection(file_prefix, newVariableText=newSelection)
- except ValueError as ve :
- self.handleWarning(str(ve))
+ except (ValueError, IONonnumericalTypeError) as bad :
+ self.handleWarning(str(bad))
def userChangedOverload (self, file_prefix, new_override_value) :
"""
diff --git a/pyglance/glance/gui_model.py b/pyglance/glance/gui_model.py
index 281eb061390ecfbef33e9709455e742ea3e08bcc..7afbc283057dbc362a4e159fa5ee8deeab554aca 100644
--- a/pyglance/glance/gui_model.py
+++ b/pyglance/glance/gui_model.py
@@ -162,7 +162,39 @@ class GlanceGUIModel (object) :
# that the user should know about
# these people can register and will get error related messages
self.errorHandlers = [ ]
-
+
+ _AVOID_LOADING_VARIABLES_PREFIXES = ["nwp_"]
+ def _pick_variable_to_load_on_file_load (self, all_variable_names, other_file_variable_name,
+ avoid_prefixes=_AVOID_LOADING_VARIABLES_PREFIXES,) :
+ """
+ When we load a new file, select our best guess at a variable to load by default.
+
+ This logic will try to select the same variable as the file that is already loaded if possible.
+ """
+
+ # start out by sorting our variables, and pick a matching one if we have one to match
+ variable_list = sorted(all_variable_names)
+ selected_var = None
+ found_ok_one = False
+ if other_file_variable_name in set(variable_list) :
+ found_ok_one = True
+ selected_var = other_file_variable_name
+
+ # if we don't have a candidate, try to look for one
+ temp_index = -1
+ while not found_ok_one :
+ if selected_var is not None :
+ found_ok_one = True # tentatively assume this one is ok
+ # check to see if it has any of the prefixes we don't want
+ for prefix in avoid_prefixes :
+ if selected_var.find(prefix) >= 0 :
+ found_ok_one = False
+ if not found_ok_one :
+ temp_index += 1
+ selected_var = str(variable_list[temp_index])
+
+ return selected_var
+
def loadNewFile (self, filePrefix, newFilePath) :
"""
load up a new file based on the prefix and path given
@@ -187,52 +219,66 @@ class GlanceGUIModel (object) :
# get the list of variables, and pick one
variableList = sorted(newFile.file_object()) # gets a list of all the variables in the file
- tempVariable = str(variableList[0])
- tempIndex = 0
- # don't automatically select the nwp variables if possible
- while (tempVariable.find("nwp_") >= 0) :
- tempIndex = tempIndex + 1
- tempVariable = variableList[tempIndex]
- # if we have a variable selected in the other slot, try to match
- temp_to_match = None
- if ( (filePrefix == A_CONST) & (self.fileData[B_CONST].ALL_VARIABLES is not None) ) :
- temp_to_match = self.fileData[B_CONST].variable
- if ( (filePrefix == B_CONST) & (self.fileData[A_CONST].ALL_VARIABLES is not None) ) :
- temp_to_match = self.fileData[A_CONST].variable
- if temp_to_match is not None :
- if temp_to_match in variableList :
- tempVariable = temp_to_match
-
+ other_file_var_name = None
+ if ((filePrefix == A_CONST) & (self.fileData[B_CONST].ALL_VARIABLES is not None)):
+ other_file_var_name = self.fileData[B_CONST].variable
+ if ((filePrefix == B_CONST) & (self.fileData[A_CONST].ALL_VARIABLES is not None)):
+ other_file_var_name = self.fileData[A_CONST].variable
+ tempVariable = self._pick_variable_to_load_on_file_load(variableList, other_file_var_name,)
LOG.debug ("selected variable: " + str(tempVariable))
-
- # save all of the data related to this file for later use
- self.fileData[filePrefix].file = newFile
- self.fileData[filePrefix].variable = tempVariable
- self.fileData[filePrefix].ALL_VARIABLES = variableList
-
- # set the longitude and latitude names, using the defaults if they exist
- self.fileData[filePrefix].latitude = tempVariable
- self.fileData[filePrefix].longitude = tempVariable
- if DEFAULT_LATITUDE in variableList :
- self.fileData[filePrefix].latitude = DEFAULT_LATITUDE
- if DEFAULT_LONGITUDE in variableList :
- self.fileData[filePrefix].longitude = DEFAULT_LONGITUDE
- # make sure the longitude and latitude are loaded into our local cache
- # TODO, it would be good to background this task at some point
- _ = self._load_variable_data(filePrefix, self.fileData[filePrefix].latitude)
- _ = self._load_variable_data(filePrefix, self.fileData[filePrefix].longitude)
-
- # load info on the current variable
- tempDataObj = self._load_variable_data(filePrefix, tempVariable)
-
- # get the variable's attributes
- tempAttrs = self._load_variable_attributes (filePrefix, tempVariable)
+
+ # try to load up our variable, and pick a different one if we can't
+ no_var_loaded = True
+ temp_vars_set = set(variableList)
+ io_error = None
+ while no_var_loaded :
+ try :
+ # save all of the data related to this file for later use
+ self.fileData[filePrefix].file = newFile
+ self.fileData[filePrefix].variable = tempVariable
+ self.fileData[filePrefix].ALL_VARIABLES = variableList
+
+ # set the longitude and latitude names, using the defaults if they exist
+ self.fileData[filePrefix].latitude = tempVariable
+ self.fileData[filePrefix].longitude = tempVariable
+ if DEFAULT_LATITUDE in variableList :
+ self.fileData[filePrefix].latitude = DEFAULT_LATITUDE
+ if DEFAULT_LONGITUDE in variableList :
+ self.fileData[filePrefix].longitude = DEFAULT_LONGITUDE
+
+ # make sure the longitude and latitude are loaded into our local cache
+ # TODO, it would be good to background this task at some point
+ _ = self._load_variable_data(filePrefix, self.fileData[filePrefix].latitude)
+ _ = self._load_variable_data(filePrefix, self.fileData[filePrefix].longitude)
+
+ # load info on the current variable
+ tempDataObj = self._load_variable_data(filePrefix, tempVariable)
+
+ # get the variable's attributes
+ tempAttrs = self._load_variable_attributes (filePrefix, tempVariable)
+
+ # if we get here then we loaded the variable ok
+ no_var_loaded = False
+ except io.IONonnumericalTypeError as ex :
+ LOG.warn("Unable to load automatically selected variable. Details: " + str(ex))
+ io_error = ex
+
+ if no_var_loaded :
+ # try to pick a different variable removing the one we tried from our potential list
+ temp_vars_set -= set(tempVariable)
+ if len(temp_vars_set) <= 0 :
+ LOG.error("Could not load any variables.")
+ raise io_error
+ tempVariable = self._pick_variable_to_load_on_file_load(list(temp_vars_set), None,)
+ io_error = None
# Now tell our data listeners that the file data changed
for dataListener in self.dataListeners :
LOG.debug("Sending update for file " + filePrefix + " with loaded data.")
- dataListener.fileDataUpdate(filePrefix, newFile.path, tempVariable, tempDataObj.override_fill_value,
- self._select_fill_value(filePrefix), tempDataObj.describe_shape(),
+ dataListener.fileDataUpdate(filePrefix, newFile.path, tempVariable,
+ tempDataObj.override_fill_value,
+ self._select_fill_value(filePrefix),
+ tempDataObj.describe_shape(),
variable_list=variableList, attribute_list=tempAttrs)
dataListener.updateSelectedLatLon(filePrefix,
self.fileData[filePrefix].latitude,
@@ -338,27 +384,33 @@ class GlanceGUIModel (object) :
newBVar = None # None if we don't need to update the B variable to match the A variable, or a variable name if we do
# update the variable selection if needed
+ io_error = None
if (newVariableText is not None) and (newVariableText != self.fileData[file_prefix].variable) :
if newVariableText in self.fileData[file_prefix].ALL_VARIABLES :
- # figure out if we need to update the B file variable selection to match the A file variable selection
- # Note: we will only do this if the user already had the same variable selected for both files and
- # file B has the new selection available
- previous_variable = self.fileData[file_prefix].variable
- if ( (file_prefix == A_CONST) & (self.fileData[B_CONST].variable == previous_variable) &
- (self.fileData[B_CONST].ALL_VARIABLES is not None) ) :
- if (newVariableText in self.fileData[B_CONST].ALL_VARIABLES) :
- newBVar = newVariableText
-
- LOG.debug("Setting file " + file_prefix + " variable selection to: " + newVariableText)
- self.fileData[file_prefix].variable = str(newVariableText)
- didUpdate = True
-
- # load the data for this new variable
- self._load_variable_data(file_prefix, str(newVariableText))
-
- # get the variable's attributes
- self._load_variable_attributes (file_prefix, str(newVariableText))
+ try :
+ # load the data for this new variable
+ self._load_variable_data(file_prefix, str(newVariableText))
+ except io.IONonnumericalTypeError as nne :
+ LOG.debug("Unable to load requested variable, reverting to previous variable.")
+ io_error = nne
+
+ if io_error is None :
+ # figure out if we need to update the B file variable selection to match the A file variable selection
+ # Note: we will only do this if the user already had the same variable selected for both files and
+ # file B has the new selection available
+ previous_variable = self.fileData[file_prefix].variable
+ if ( (file_prefix == A_CONST) & (self.fileData[B_CONST].variable == previous_variable) &
+ (self.fileData[B_CONST].ALL_VARIABLES is not None) ) :
+ if (newVariableText in self.fileData[B_CONST].ALL_VARIABLES) :
+ newBVar = newVariableText
+
+ LOG.debug("Setting file " + file_prefix + " variable selection to: " + newVariableText)
+ self.fileData[file_prefix].variable = str(newVariableText)
+ didUpdate = True
+
+ # get the variable's attributes
+ self._load_variable_attributes (file_prefix, str(newVariableText))
# for convenience hang on to this
tempVariableName = self.fileData[file_prefix].variable
@@ -376,7 +428,7 @@ class GlanceGUIModel (object) :
didUpdate = True
# let our data listeners know about any changes
- if didUpdate :
+ if didUpdate or io_error is not None :
tempDataObject = self.fileData[file_prefix].var_data_cache[tempVariableName]
tempAttrsList = self.fileData[file_prefix].var_attrs_cache[tempVariableName]
for listener in self.dataListeners :
@@ -388,6 +440,10 @@ class GlanceGUIModel (object) :
if newBVar is not None :
LOG.debug("Syncing file " + B_CONST + " variable selection to: " + newVariableText)
self.updateFileDataSelection(B_CONST, newVariableText=newVariableText)
+
+ # if we had an io issue, raise that
+ if io_error is not None :
+ raise io_error
def _select_fill_value (self, file_prefix) :
"""
diff --git a/pyglance/glance/io.py b/pyglance/glance/io.py
index 26b94535000cd06bf203dcbfd95a5afe212947ac..47853dd37357352a2bad52f438c0fdb40d30deef 100644
--- a/pyglance/glance/io.py
+++ b/pyglance/glance/io.py
@@ -94,6 +94,12 @@ class IOUnimplimentedError(Exception):
def __str__(self):
return self.msg
+class IONonnumericalTypeError(Exception):
+ """
+ A type was encountered that numpy doesn't know how to deal with - e.g. netCDF variable-length string arrays
+ """
+ pass
+
class CaseInsensitiveAttributeCache (object) :
"""
A cache of attributes for a single file and all of it's variables.
@@ -468,6 +474,18 @@ class nc (object):
variable_object.set_auto_maskandscale(False) # for now just do the darn calculations ourselves
temp_input_data = variable_object[:]
LOG.debug("Native input dtype: " + str(temp_input_data.dtype))
+ # if this is object data, stop because we can't run our regular analysis on that kind
+ if temp_input_data.dtype == object :
+ LOG.warn("Variable '" + name + "' has a data type of 'object'. This type of data cannot be analyzed by Glance. "
+ "This variable will not be analyzed.")
+ raise IONonnumericalTypeError("Variable '" + name + "' is of data type 'object'. "
+ "This program can't analyze non-numerical data.")
+ """
+ Note to self, if we ever do want to access data in a numpy array with dtype=object, for some
+ reason this library is packing that into a a zero dimensional tuple or something similar.
+ I was able to unpack the data using a construction like: temp_input_data = temp_input_data[()]
+ After that the array can be indexed into as normal for a numpy array.
+ """
dtype_to_use = _get_data_uptype(temp_input_data.dtype)
LOG.debug("Choosing dtype " + str(dtype_to_use) + " for our internal representation of this data.")
scaled_data_copy = numpy.array(temp_input_data, dtype=dtype_to_use,)
diff --git a/pyglance/glance/plotcreatefns.py b/pyglance/glance/plotcreatefns.py
index 97f561c2e4390b9b928b20989bd2e395ed331843..9856b4dc81ec3bbbc8dd6ff25a86812ab72f69b0 100644
--- a/pyglance/glance/plotcreatefns.py
+++ b/pyglance/glance/plotcreatefns.py
@@ -103,7 +103,9 @@ class PlottingFunctionFactory :
"""
def __init__(self) :
- LOG.error("Someone has instantiated PlottingFunctionFactory. This class is NOT meant to be instantiated.")
+ #LOG.error("Someone has instantiated PlottingFunctionFactory. This class is NOT meant to be instantiated.")
+ # note: somewhere in the transition to python 3 the internals started calling this method for child classes too
+ pass
def create_plotting_functions (