io.py

#!/usr/bin/env python
# encoding: utf-8
"""
I/O routines supporting reading a number of file formats.

Created by rayg Apr 2009.
Copyright (c) 2009 University of Wisconsin SSEC. All rights reserved.
"""

import os, logging
import numpy
from functools import reduce

LOG = logging.getLogger(__name__)

Loadable_Types = set()

try:
    import pyhdf
    from pyhdf.SD import SD,SDC, SDS, HDF4Error
    Loadable_Types.add("hdf")
except:
    LOG.info('no pyhdf module available for HDF4')
    pyhdf = None
    SD = SDC = SDS = object
    HDF4Error = EnvironmentError
    
try:
    import h5py
    from h5py import h5d
    Loadable_Types.add("h5")
except ImportError:
    LOG.info('no h5py module available for reading HDF5')
    h5py = None

# the newer netCDF library that replaced pycdf
try:
    import netCDF4
    Loadable_Types.update(["nc", "nc4", "cdf", ])
except:
    LOG.info("unable to import netcdf4 library")
    netCDF4 = None

try:
    import dmv as dmvlib
    LOG.info('loaded dmv module for AERI data file access')
    Loadable_Types.update(["cxs", "rnc", "cxv", "csv", "spc", "sum", "uvs", "aeri", ])
except ImportError:
    LOG.info('no AERI dmv data file format module')
    dmvlib = None

try:
    import adl_blob
    LOG.info('adl_blob module found for JPSS ADL data file access')
    # TODO, what is the loadable file extension?
except ImportError:
    LOG.info('no adl_blob format handler available')
    adl_blob = None

try :
    from osgeo import gdal
    LOG.info('loading osgeo module for GeoTIFF data file access')
    Loadable_Types.update(["tiff", "tif", "tifa", ])
except :
    LOG.info('no osgeo available for reading GeoTIFF data files')
    gdal = None

UNITS_CONSTANT = "units"

fillValConst1 = '_FillValue'
fillValConst2 = 'missing_value'

ADD_OFFSET_STR   = 'add_offset'
SCALE_FACTOR_STR = 'scale_factor'
SCALE_METHOD_STR = 'scaling_method'

UNSIGNED_ATTR_STR = "_unsigned"

SIGNED_TO_UNSIGNED_DTYPES = {
                                numpy.dtype(numpy.int8):    numpy.dtype(numpy.uint8),
                                numpy.dtype(numpy.int16):   numpy.dtype(numpy.uint16),
                                numpy.dtype(numpy.int32):   numpy.dtype(numpy.uint32),
                                numpy.dtype(numpy.int64):   numpy.dtype(numpy.uint64),
                            }

class IOUnimplimentedError(Exception):
    """
    The exception raised when a requested io operation is not yet available.
    
        msg  -- explanation of the problem
    """
    def __init__(self, msg):
        self.msg = msg
    def __str__(self):
        return self.msg

class CaseInsensitiveAttributeCache (object) :
    """
    A cache of attributes for a single file and all of it's variables.
    This cache is considered uncased, it will store all attributes it caches
    in lower case and will lower case any strings it is asked to search for
    in the cache.
    When variable or global attribute sets are not yet loaded and something
    from that part of the file is requested the cache will transparently load
    attributes from the file behind the scenes and build the cache for that
    part of the file.
    """
    
    def __init__(self, fileObject) :
        """
        set up the empty cache and hang on to the file object we'll be caching
        """
        
        self.fileToCache             = fileObject
        self.globalAttributesLower   = None
        self.variableAttributesLower = { }
    
    def _load_global_attributes_if_needed (self) :
        """
        load up the global attributes if they need to be cached
        """
        
        # load the attributes from the file if they aren't cached
        if self.globalAttributesLower is None :
            LOG.debug ("Loading file global attributes into case-insensitive cache.")
            tempAttrs                  = self.fileToCache.get_global_attributes(caseInsensitive=False)
            self.globalAttributesLower = dict((k.lower(), v) for k, v in tempAttrs.items())
    
    def _load_variable_attributes_if_needed (self, variableName) :
        """
        load up the variable attributes if they need to be cached
        """
        
        # make a lower cased version of the variable name
        tempVariableName = variableName.lower()
        
        # load the variable's attributes from the file if they aren't cached
        if tempVariableName not in self.variableAttributesLower :
            LOG.debug ("Loading attributes for variable \"" + variableName + "\" into case-insensitive cache.")
            tempAttrs = self.fileToCache.get_variable_attributes(variableName, caseInsensitive=False)
            # now if there are any attributes, make a case insensitive version
            self.variableAttributesLower[tempVariableName] = dict((k.lower(), v) for k, v in tempAttrs.items())
    
    def get_variable_attribute (self, variableName, attributeName) :
        """
        get the specified attribute for the specified variable,
        if this variable's attributes have not yet been loaded
        they will be loaded and cached
        """
        
        self._load_variable_attributes_if_needed(variableName)
        
        toReturn = None
        tempVariableName  =  variableName.lower()
        tempAttributeName = attributeName.lower()
        if (tempVariableName in self.variableAttributesLower) and (tempAttributeName in self.variableAttributesLower[tempVariableName]) :
            toReturn = self.variableAttributesLower[tempVariableName][tempAttributeName]
        else:
            LOG.debug ("Attribute \"" + attributeName + "\" was not present for variable \"" + variableName + "\".")
        
        return toReturn
    
    def get_variable_attributes (self, variableName) :
        """
        get the variable attributes for the variable name given
        """
        
        self._load_variable_attributes_if_needed(variableName)
        
        toReturn = self.variableAttributesLower[variableName.lower()] if (variableName.lower() in self.variableAttributesLower) else None
        
        return toReturn
    
    def get_global_attribute (self, attributeName) :
        """
        get a global attribute with the given name
        """
        
        self._load_global_attributes_if_needed()
        
        toReturn = self.globalAttributesLower[attributeName.lower()] if (attributeName.lower() in self.globalAttributesLower) else None
        
        return toReturn
    
    def get_global_attributes (self) :
        """
        get the global attributes,
        """
        
        self._load_global_attributes_if_needed()
        
        toReturn = self.globalAttributesLower
        
        return toReturn
    
    def is_loadable_type (self, name) :
        """
        check to see if the indicated variable is a type that can be loaded
        """
        
        # TODO, are there any bad types for these files?
        return True

def _get_data_uptype (input_dtype) :
    """
    Given an input data type, figure out what type we need to upcast it to.

    Note: Glance expects all it's data to get upcast into floats for the purposes of it's
    later math manipulations.
    """

    default_uptype = numpy.float32
    default_finfo  = numpy.finfo(default_uptype)
    input_info     = numpy.finfo(input_dtype) if  numpy.issubdtype(input_dtype, numpy.floating,) else numpy.iinfo(input_dtype)

    # if our input won't fit into the default, pick a bigger type
    if ( (default_finfo.min > input_info.min) or (default_finfo.max < input_info.max) ) :
        LOG.debug("Input data will not fit in default float32 data type, using larger type.")
        default_uptype = numpy.float64

    # FUTURE, if we reach a point where a float64 isn't big enough, this will need to be revisited

    return default_uptype

class hdf (object):
    """wrapper for HDF4 dataset for comparison
    __call__ yields sequence of variable names
    __getitem__ returns individual variables ready for slicing to numpy arrays
    """
    
    _hdf = None
    
    def __init__(self, filename, allowWrite=False):
        
        if pyhdf is None:
            LOG.error('pyhdf is not installed and is needed in order to read hdf4 files')
            assert(pyhdf is not None)
        mode = SDC.READ
        if allowWrite:
            mode = mode | SDC.WRITE
        
        self._hdf = SD(filename, mode)
        self.attributeCache = CaseInsensitiveAttributeCache(self)

    def __call__(self):
        "yield names of variables to be compared"
        return list(self._hdf.datasets())
    
    # this returns a numpy array with a copy of the full, scaled
    # data for this variable, if the data type must be changed to allow
    # for scaling it will be (so the return type may not reflect the
    # type found in the original file)
    def __getitem__(self, name):
        # defaults
        scale_factor = 1.0
        add_offset = 0.0
        data_type = None 
        scaling_method = None
        
        # get the variable object and use it to
        # get our raw data and scaling info
        variable_object = self.get_variable_object(name)
        raw_data_copy = variable_object[:]
        try :
            # TODO, this currently won't work with geocat data, work around it for now
            scale_factor, scale_factor_error, add_offset, add_offset_error, data_type = SDS.getcal(variable_object)
        except HDF4Error:
            # load just the scale factor and add offset information by hand
            temp = self.attributeCache.get_variable_attributes(name)
            if ADD_OFFSET_STR in temp :
                add_offset = temp[ADD_OFFSET_STR]
                data_type = numpy.dtype(type(add_offset))
            if SCALE_FACTOR_STR in temp :
                scale_factor = temp[SCALE_FACTOR_STR]
                data_type = numpy.dtype(type(scale_factor))
            if SCALE_METHOD_STR in temp :
                scaling_method = temp[SCALE_METHOD_STR]
        SDS.endaccess(variable_object)
        
        # don't do lots of work if we don't need to scale things
        if (scale_factor == 1.0) and (add_offset == 0.0) :
            return raw_data_copy
        
        # at the moment geocat has several scaling methods that don't match the normal standards for hdf
        """
        please see constant.f90 for a more up to date version of this information:
            INTEGER(kind=int1) :: NO_SCALE              ! 0
            INTEGER(kind=int1) :: LINEAR_SCALE          ! 1
            INTEGER(kind=int1) :: LOG_SCALE             ! 2
            INTEGER(kind=int1) :: SQRT_SCALE            ! 3 
        """
        if (scaling_method == 0) :
            return raw_data_copy
        if not ((scaling_method is None) or (int(scaling_method) <= 1)) :
            LOG.warn ('Scaling method of \"' + str(scaling_method) + '\" will be ignored in favor of hdf standard method. '
                      + 'This may cause problems with data consistency')
        
        # if we don't have a data type something strange has gone wrong
        assert(not (data_type is None))
        
        # get information about where the data is the missing value
        missing_val = self.missing_value(name)
        missing_mask = numpy.zeros(raw_data_copy.shape, dtype=numpy.bool)
        if missing_val is not None :
            missing_mask[raw_data_copy == missing_val] = True
        
        # create the scaled version of the data
        scaled_data_copy                = numpy.array(raw_data_copy, dtype=data_type)
        scaled_data_copy[~missing_mask] = (scaled_data_copy[~missing_mask] * scale_factor) + add_offset #TODO, type truncation issues?
        
        return scaled_data_copy 
    
    def get_variable_object(self, name):
        return self._hdf.select(name)
    
    def missing_value(self, name):
        
        return self.get_attribute(name, fillValConst1)
    
    def create_new_variable(self, variablename, missingvalue=None, data=None, variabletocopyattributesfrom=None):
        """
        create a new variable with the given name
        optionally set the missing value (fill value) and data to those given
        
        the created variable will be returned, or None if a variable could not
        be created
        """
        
        raise IOUnimplimentedError('Unable to create variable in hdf file, this functionality is not yet available.')
        
        return None
    
    def add_attribute_data_to_variable(self, variableName, newAttributeName, newAttributeValue) :
        """
        if the attribute exists for the given variable, set it to the new value
        if the attribute does not exist for the given variable, create it and set it to the new value
        """
        
        raise IOUnimplimentedError('Unable add attribute to hdf file, this functionality is not yet available.')
        
        return
    
    def get_variable_attributes (self, variableName, caseInsensitive=True) :
        """
        returns all the attributes associated with a variable name
        """
        
        toReturn = None
        if caseInsensitive :
            toReturn = self.attributeCache.get_variable_attributes(variableName)
        else :
            toReturn = self.get_variable_object(variableName).attributes()
        
        return toReturn
    
    def get_attribute(self, variableName, attributeName, caseInsensitive=True) :
        """
        returns the value of the attribute if it is available for this variable, or None
        """
        toReturn = None
        
        if caseInsensitive :
            toReturn = self.attributeCache.get_variable_attribute(variableName, attributeName)
        else :
            temp_attributes = self.get_variable_attributes(variableName, caseInsensitive=False)
            
            if attributeName in temp_attributes :
                toReturn = temp_attributes[attributeName]
        
        return toReturn
    
    def get_global_attributes(self, caseInsensitive=True) :
        """
        get a list of all the global attributes for this file or None
        """
        
        toReturn = None
        
        if caseInsensitive :
            toReturn = self.attributeCache.get_global_attributes()
        else :
            toReturn = self._hdf.attributes()
        
        return toReturn
    
    def get_global_attribute(self, attributeName, caseInsensitive=True) :
        """
        returns the value of a global attribute if it is available or None
        """
        
        toReturn = None
        
        if caseInsensitive :
            toReturn = self.attributeCache.get_global_attribute(attributeName)
        else :
            if attributeName in self._hdf.attributes() :
                toReturn = self._hdf.attributes()[attributeName]
        
        return toReturn
    
    def is_loadable_type (self, name) :
        """
        check to see if the indicated variable is a type that can be loaded
        """
        
        # TODO, are there any bad types for these files?
        return True

class nc (object):
    """wrapper for netcdf4-python data access for comparison
    __call__ yields sequence of variable names
    __getitem__ returns individual variables ready for slicing to numpy arrays
    """
    
    _nc = None
    _var_map = None

    # walk down through all groups and get variable names and objects
    def _walkgroups(self, start_at, prefix=None, ):
        # look through the variables that are here
        for var_name in start_at.variables:
            temp_name = var_name if prefix is None or len(prefix) <= 0 else prefix + "/" + var_name
            yield temp_name, start_at[var_name]
        # look through the groups that are here
        for group_name in start_at.groups:
            grp_str = group_name if prefix is None or len(prefix) <= 0 else prefix + "/" + group_name
            for more_var_name, more_var_obj in self._walkgroups(start_at.groups[group_name], prefix=grp_str):
                yield more_var_name, more_var_obj
    
    def __init__(self, filename, allowWrite=False):
        
        if netCDF4 is None:
            LOG.error('netCDF4 is not installed and is needed in order to read NetCDF files')
            assert(netCDF4 is not None)
        
        mode = 'r'
        if allowWrite :
            mode = 'w'
        
        self._nc = netCDF4.Dataset(filename, mode)
        self.attributeCache = CaseInsensitiveAttributeCache(self)
        self._var_map = { }
        for var_name, var_obj in self._walkgroups(self._nc,) :
            self._var_map[var_name] = var_obj

    def __call__(self):
        """
        yield names of variables in this file
        """

        return list(self._var_map)

    def __getitem__(self, name):
        """
        this returns a numpy array with a copy of the full, scaled
        data for this variable, if the data type must be changed to allow
        for scaling it will be (so the return type may not reflect the
        type found in the original file)
        """

        LOG.debug("loading variable data for: " + name)

        # get the variable object and use it to
        # get our raw data and scaling info
        variable_object = self.get_variable_object(name)

        # get our data, save the dtype, and make sure it's a more flexible dtype for now
        variable_object.set_auto_maskandscale(False)  # for now just do the darn calculations ourselves
        temp_input_data = variable_object[:]
        LOG.debug("Native input dtype: " + str(temp_input_data.dtype))
        dtype_to_use = _get_data_uptype(temp_input_data.dtype)
        LOG.debug("Choosing dtype " + str(dtype_to_use) + " for our internal representation of this data.")
        scaled_data_copy = numpy.array(temp_input_data, dtype=dtype_to_use,)

        # get the attribute cache so we can check on loading related attributes
        temp = self.attributeCache.get_variable_attributes(name)

        # get information about where the data is the missing value
        missing_val = self.missing_value(name)
        missing_mask = numpy.zeros(scaled_data_copy.shape, dtype=numpy.bool)
        if missing_val is not None:
            missing_mask[scaled_data_copy == missing_val] = True

        #***** just do the darn unsigned handling ourselves, ugh

        # if our data is labeled as being unsigned by the appropriately set attribute
        if UNSIGNED_ATTR_STR in temp and str(temp[UNSIGNED_ATTR_STR]).lower() == ("true"):
            LOG.debug("Correcting for unsigned values in variable data.")
            where_temp = (scaled_data_copy < 0.0) & ~missing_mask # where we have negative but not missing data
            scaled_data_copy[where_temp] += (numpy.iinfo(numpy.uint16).max + 1.0) # add the 2's complement

        #***** end of handling the unsigned attribute

        ###### the start of the scaling code
        # Note, I had to turn this back on because the netcdf4 library is behaving erratically when unsigned is set

        # get the scale factor and add offset from the attributes
        scale_factor = 1.0
        add_offset = 0.0
        if SCALE_FACTOR_STR in temp :
            scale_factor = temp[SCALE_FACTOR_STR]
        if ADD_OFFSET_STR in temp :
            add_offset = temp[ADD_OFFSET_STR]

        # don't do work if we don't need to unpack things
        if (scale_factor != 1.0) or (add_offset != 0.0) :

            LOG.debug("Manually applying scale (" + str(scale_factor) + ") and add offset (" + str(add_offset) + ").")

            # unpack the data
            scaled_data_copy[~missing_mask] = (scaled_data_copy[~missing_mask] * scale_factor) + add_offset

        ###### end of the scaling code

        """
        #TODO, this section was for when we had to do the unsigned correction after unpacking
        if UNSIGNED_ATTR_STR in temp and str(temp[UNSIGNED_ATTR_STR]).lower() == ( "true" ) :

            LOG.debug("fixing unsigned values in variable " + name)

            # load the scale factor and add offset
            scale_factor = 1.0
            add_offset = 0.0
            temp = self.attributeCache.get_variable_attributes(name)
            if SCALE_FACTOR_STR in temp :
                scale_factor = temp[SCALE_FACTOR_STR]
            if ADD_OFFSET_STR in temp :
                add_offset = temp[ADD_OFFSET_STR]

            # get the missing value and figure out the dtype of the original data
            missing_val  = self.missing_value(name)
            orig_dtype   = numpy.array([missing_val,]).dtype
            needed_dtype = SIGNED_TO_UNSIGNED_DTYPES[orig_dtype] if orig_dtype in SIGNED_TO_UNSIGNED_DTYPES else None

            if needed_dtype is not None :
                # now figure out where all the corrupted values are, and shift them up to be positive
                needs_fix_mask = (scaled_data_copy < add_offset) & (scaled_data_copy != missing_val)
                # we are adding the 2's complement, but first we're scaling it appropriately
                scaled_data_copy[needs_fix_mask] += ((numpy.iinfo(numpy.uint16).max + 1.0) * scale_factor)
        """

        return scaled_data_copy
    
    # TODO, this hasn't been supported in other file types
    def close (self) :
        self._nc.close()
        self._nc = None
        self._var_map = None

    def get_variable_object(self, name):

        return self._var_map[name]
    
    def missing_value(self, name):
        
        toReturn = None
        
        temp = self.attributeCache.get_variable_attribute(name, fillValConst1)
        if temp is not None :
            toReturn = temp
        else :
            temp = self.attributeCache.get_variable_attribute(name, fillValConst2)
            if temp is not None :
                toReturn = temp
        
        return toReturn

    def create_new_variable(self, variablename, missingvalue=None, data=None, variabletocopyattributesfrom=None):
        """
        create a new variable with the given name
        optionally set the missing value (fill value) and data to those given
        
        the created variable will be returned, or None if a variable could not
        be created
        """

        # TODO, this will not work with groups
        self._nc.nc_redef()
        
        # if the variable already exists, stop with a warning
        if variablename in self._nc.variables :
            LOG.warn("New variable name requested (" + variablename + ") is already present in file. " +
                     "Skipping generation of new variable.")
            return None
        # if we have no data we won't be able to determine the data type to create the variable
        if (data is None) or (len(data) <= 0) :
            LOG.warn("Data type for new variable (" + variablename + ") could not be determined. " +
                     "Skipping generation of new variable.")
            return None
        
        dataType = None
        if numpy.issubdtype(data.dtype, int) :
            dataType = numpy.int
            #print("Picked INT")
        # TODO, at the moment the fill type is forcing me to use a double, when sometimes I want a float
        #elif numpy.issubdtype(data.dtype, numpy.float32) :
        #    dataType = numpy.float
        #    print("Picked FLOAT")
        elif numpy.issubdtype(data.dtype, float) :
            dataType = numpy.float64
            #print("Picked DOUBLE")
        # what do we do if it's some other type?
        
        # create and set all the dimensions
        dimensions = [ ]
        dimensionNum = 0
        for dimSize in data.shape :
            dimensions.append(self._nc.createDimension(variablename + '-index' + str(dimensionNum), dimSize))
            dimensionNum = dimensionNum + 1
        
        # create the new variable
        #print('variable name: ' + variablename)
        #print('data type:     ' + str(dataType))
        #print('dimensions:    ' + str(dimensions))
        newVariable = self._nc.createVariable(variablename, dataType, tuple(dimensions))
        
        # if a missing value was given, use that
        if missingvalue is not None :
            newVariable._FillValue = missingvalue
        
        # if we have a variable to copy attributes from, do so
        if variabletocopyattributesfrom is not None :
            attributes = self.get_variable_attributes(variabletocopyattributesfrom, caseInsensitive=False)

            for attribute in attributes :
                setattr(newVariable, attribute, attributes[attribute])

        self._nc.nc_enddef()

        # if data was given, use that
        if data is not None :
            newVariable[:](data.tolist())

        return newVariable

    def add_attribute_data_to_variable(self, variableName, newAttributeName, newAttributeValue) :
        """
        if the attribute exists for the given variable, set it to the new value
        if the attribute does not exist for the given variable, create it and set it to the new value
        """
        # TODO, this will not work with groups

        variableObject = self.get_variable_object(variableName)
        
        self._nc.nc_redef()

        setattr(variableObject, newAttributeName, newAttributeValue)

        self._nc.nc_enddef()

        # TODO, this will cause our attribute cache to be wrong!
        # TODO, for now, brute force clear the cache
        self.attributeCache = CaseInsensitiveAttributeCache(self)
        
        return
    
    def get_variable_attributes (self, variableName, caseInsensitive=True) :
        """
        returns all the attributes associated with a variable name
        """
        
        toReturn = None
        
        if caseInsensitive :
            toReturn = self.attributeCache.get_variable_attributes(variableName)
        else :
            toReturn = { }
            tempVarObj   = self.get_variable_object(variableName)
            tempAttrKeys = tempVarObj.ncattrs()
            for attrKey in tempAttrKeys :
                toReturn[attrKey] = getattr(tempVarObj, attrKey)
        
        return toReturn
    
    def get_attribute(self, variableName, attributeName, caseInsensitive=True) :
        """
        returns the value of the attribute if it is available for this variable, or None
        """
        toReturn = None
        
        if caseInsensitive :
            toReturn = self.attributeCache.get_variable_attribute(variableName, attributeName)
        else :
            temp_attributes = self.get_variable_attributes(variableName, caseInsensitive=False)
            
            if attributeName in temp_attributes :
                toReturn = getattr(self.get_variable_object, attributeName)
        
        return toReturn
    
    def get_global_attributes(self, caseInsensitive=True) :
        """
        get a list of all the global attributes for this file or None
        """
        
        toReturn = None
        
        if caseInsensitive :
            toReturn = self.attributeCache.get_global_attributes()
        else :
            toReturn = { }
            tempAttrKeys = self._nc.ncattrs()
            for attrKey in tempAttrKeys :
                toReturn[attrKey] = getattr(self._nc, attrKey)

        return toReturn
    
    def get_global_attribute(self, attributeName, caseInsensitive=True) :
        """
        returns the value of a global attribute if it is available or None
        """
        
        toReturn = None
        
        if caseInsensitive :
            toReturn = self.attributeCache.get_global_attribute(attributeName)
        else :
            if attributeName in self._nc.ncattrs() :
                toReturn = getattr(self._nc, attributeName)
        
        return toReturn
    
    def is_loadable_type (self, name) :
        """
        check to see if the indicated variable is a type that can be loaded
        """

        return True

nc4 = nc
cdf = nc

# TODO remove
#FIXME_IDPS = [ '/All_Data/CrIS-SDR_All/ES' + ri + band for ri in ['Real','Imaginary'] for band in ['LW','MW','SW'] ] 

class h5(object):
    """wrapper for HDF5 datasets
    """
    _h5 = None
    
    def __init__(self, filename, allowWrite=False):
        self.attributeCache = CaseInsensitiveAttributeCache(self)
        
        mode = 'r'
        if allowWrite :
            mode = 'r+'
        if h5py is None:
            LOG.error('h5py module is not installed and is needed in order to read h5 files')
            assert(h5py is not None)
        self._h5 = h5py.File(filename, mode)
    
    def __call__(self):
        
        variableList = [ ]
        def testFn (name, obj) :
            #print ('checking name: ' + name)
            #print ('object: ' + str(obj))
            
            if isinstance(obj, h5py.Dataset) :
                try :
                    tempType = obj.dtype # this is required to provoke a type error for closed data sets
                    
                    #LOG.debug ('type: ' + str(tempType))
                    variableList.append(name)
                except TypeError :
                    LOG.debug('TypeError prevents the use of variable ' + name
                              + '. This variable will be ignored')
        
        self._h5.visititems(testFn)
        
        LOG.debug('variables from visiting h5 file structure: ' + str(variableList))
        
        return(variableList)
    
    @staticmethod
    def trav(h5,pth): 
        return reduce( lambda x,a: x[a] if a else x, pth.split('/'), h5)
        
    # this returns a numpy array with a copy of the full, scaled
    # data for this variable, if the data type must be changed to allow
    # for scaling it will be (so the return type may not reflect the
    # type found in the original file)
    def __getitem__(self, name):
        
        # defaults
        scale_factor = 1.0
        add_offset = 0.0
        
        # get the variable object and use it to
        # get our raw data and scaling info
        variable_object = self.get_variable_object(name)
        raw_data_copy = variable_object[:]

        # pick a data type to use internally
        data_type = _get_data_uptype(raw_data_copy.dtype)

        #print ('*************************')
        #print (dir (variable_object.id)) # TODO, is there a way to get the scale and offset through this?
        #print ('*************************')
        
        # load the scale factor and add offset
        temp = self.attributeCache.get_variable_attributes(name)
        if SCALE_FACTOR_STR in temp :
            scale_factor = temp[SCALE_FACTOR_STR]
        if ADD_OFFSET_STR in temp :
            add_offset = temp[ADD_OFFSET_STR]
        # todo, does cdf have an equivalent of endaccess to close the variable?
        
        # don't do lots of work if we don't need to scale things
        if (scale_factor == 1.0) and (add_offset == 0.0) :
            return raw_data_copy
        
        # get information about where the data is the missing value
        missing_val = self.missing_value(name)
        missing_mask = numpy.zeros(raw_data_copy.shape, dtype=numpy.bool)
        if missing_val is not None:
            missing_mask[raw_data_copy == missing_val] = True
        
        # create the scaled version of the data
        scaled_data_copy = numpy.array(raw_data_copy, dtype=data_type)
        scaled_data_copy[~missing_mask] = (scaled_data_copy[~missing_mask] * scale_factor) + add_offset #TODO, type truncation issues?
        
        return scaled_data_copy
    
    def get_variable_object(self,name):
        return h5.trav(self._h5, name)
    
    def missing_value(self, name):
        
        toReturn = None
        
        # get the missing value if it has been set
        variableObject = self.get_variable_object(name)
        pListObj = variableObject.id.get_create_plist()
        fillValueStatus = pListObj.fill_value_defined()
        if (h5d.FILL_VALUE_DEFAULT == fillValueStatus) or (h5d.FILL_VALUE_USER_DEFINED == fillValueStatus) :
            temp = numpy.array((1), dtype=variableObject.dtype)
            pListObj.get_fill_value(temp)
            toReturn = temp
        
        return toReturn
    
    def create_new_variable(self, variablename, missingvalue=None, data=None, variabletocopyattributesfrom=None):
        """
        create a new variable with the given name
        optionally set the missing value (fill value) and data to those given
        
        the created variable will be returned, or None if a variable could not
        be created
        """
        
        raise IOUnimplimentedError('Unable to create variable in hdf 5 file, this functionality is not yet available.')
        
        return None
    
    def add_attribute_data_to_variable(self, variableName, newAttributeName, newAttributeValue) :
        """
        if the attribute exists for the given variable, set it to the new value
        if the attribute does not exist for the given variable, create it and set it to the new value
        """
        
        raise IOUnimplimentedError('Unable to add attribute to hdf 5 file, this functionality is not yet available.')
        
        return
    
    def get_variable_attributes (self, variableName, caseInsensitive=True) :
        """
        returns all the attributes associated with a variable name
        """
        
        toReturn = None
        
        if caseInsensitive :
            toReturn = self.attributeCache.get_variable_attributes(variableName)
        else :
            toReturn = self.get_variable_object(variableName).attrs
        
        return toReturn
    
    def get_attribute(self, variableName, attributeName, caseInsensitive=True) :
        """
        returns the value of the attribute if it is available for this variable, or None
        """
        toReturn = None
        
        if caseInsensitive :
            toReturn = self.attributeCache.get_variable_attribute(variableName, attributeName)
        else :
            temp_attrs = self.get_variable_attributes(variableName, caseInsensitive=False)
            
            if (attributeName in temp_attrs) :
                toReturn = temp_attrs[attributeName]
        
        return toReturn
    
    def get_global_attributes(self, caseInsensitive=True) :
        """
        get a list of all the global attributes for this file or None
        """
        
        toReturn = None
        
        if caseInsensitive :
            toReturn = self.attributeCache.get_global_attributes()
        else :
            toReturn = self._h5.attrs
        
        return toReturn
    
    def get_global_attribute(self, attributeName, caseInsensitive=True) :
        """
        returns the value of a global attribute if it is available or None
        """
        
        toReturn = None
        
        if caseInsensitive :
            toReturn = self.attributeCache.get_global_attribute(attributeName)
        else :
            if attributeName in self._h5.attrs :
                toReturn = self._h5.attrs[attributeName]
        
        return toReturn
    
    def is_loadable_type (self, name) :
        """
        check to see if the indicated variable is a type that can be loaded
        """
        
        # TODO, are there any bad types for these files?
        return True


class aeri(object):
    """wrapper for AERI RNC/SUM/CXS/etc datasets
    """
    _dmv = None
    _vectors = { }
    _scalars = { }
    
    @staticmethod
    def _meta_mapping(fp):
        ids = fp.metaIDs()
        names = [fp.queryMetaDescString(1, id_, fp.SHORTNAME) for id_ in ids]
        assert len(ids) == len(names)
        return (dict((n, i) for n, i in zip(names, ids)))
    
    def _inventory(self):
        fp = self._dmv
        assert(fp is not None)
        # get list of vectors and scalars
        self._vectors = dict( (fp.queryVectorDescString(n,fp.SHORTNAME), n) for n in fp.vectorIDs() )
        self._scalars = self._meta_mapping(fp)

    def __init__(self, filename, allowWrite=False):
        assert(allowWrite==False)
        if dmvlib is None:
            LOG.error('cannot open AERI files without dmv module being available')
            return
        self._dmv = dmvlib.dmv()
        rc = self._dmv.openFile(filename)
        if rc!=0:
            LOG.error("unable to open file, rc=%d" % rc)
            self._dmv = None        
        else:
            self._inventory()
    
    def __call__(self):
        return list(self._vectors) + list(self._scalars)
        
    def __getitem__(self, name):
        fp = self._dmv
        assert(fp is not None)
        if 'DMV_RECORDS' in os.environ:
            nrecs = int(os.environ['DMV_RECORDS'])
            LOG.warning('overriding dmv record count to %d' % nrecs)
        else:
            nrecs = self._dmv.recordCount()
        recrange = list(range(1, nrecs+1))
        if name in self._vectors:
            vid = self._vectors[name]
            vdata = [ fp.vectorDepValues(rec, vid) for rec in recrange ]
            return numpy.array(vdata)
        elif name in self._scalars:
            vdata = fp.metaValueMatrix(recrange, [self._scalars[name]])
            return numpy.array(vdata)
        else:
            raise LookupError('cannot find variable %s' % name)
       
    def get_variable_object(self,name):
        return None
    
    def missing_value(self, name):
        return float('nan')
    
    def create_new_variable(self, variablename, missingvalue=None, data=None, variabletocopyattributesfrom=None):
        """
        create a new variable with the given name
        optionally set the missing value (fill value) and data to those given