From c18fda5d6031480b61f1cd8b0f852bc40a3ef7d5 Mon Sep 17 00:00:00 2001
From: "(no author)" <(no author)@8a9318a1-56ba-4d59-b755-99d26321be01>
Date: Thu, 14 Jan 2010 20:52:41 +0000
Subject: [PATCH] corrected loading of h5 missing values; fixed info variable
selection crash; invalid pts much more accurately detected and ignored during
colocation; set up to plot vector data using config file settings; fixed
remote file access for colocation copies
git-svn-id: https://svn.ssec.wisc.edu/repos/glance/trunk@100 8a9318a1-56ba-4d59-b755-99d26321be01
---
pyglance/glance/compare.py | 91 +++++++++++++++++++++++++++++++-------
pyglance/glance/delta.py | 13 +++---
pyglance/glance/figures.py | 2 +-
pyglance/glance/io.py | 39 +++++++++++++++-
4 files changed, 121 insertions(+), 24 deletions(-)
diff --git a/pyglance/glance/compare.py b/pyglance/glance/compare.py
index 736169e..6686428 100644
--- a/pyglance/glance/compare.py
+++ b/pyglance/glance/compare.py
@@ -555,7 +555,8 @@ def _handle_lon_lat_info (lon_lat_settings, a_file_object, b_file_object, output
error_msg = None
# if there is no lon/lat specified, stop now
- if ('longitude' not in lon_lat_settings) or ('latitude' not in lon_lat_settings) :
+ if ( ('longitude' not in lon_lat_settings) or ('latitude' not in lon_lat_settings)
+ or (('noLonLatVars' in lon_lat_settings) and lon_lat_settings['noLonLatVars']) ) :
return { }, spatialInfo, error_msg
# if we should not be comparing against the logitude and latitude, stop now
@@ -786,18 +787,45 @@ def _uri_needs_rsync(uri_to_check) :
"""
return not os.path.exists(uri_to_check)
-def rsync_or_copy_files (list_of_files, target_directory='.') :
+def _get_UV_info_from_magnitude_direction_info(fileObject, magnitudeName, directionName, invalidMask=None) :
+ """
+ If there are magnitude and direction names, load that information and calculate the u and v that correspond to it
+ """
+
+ # if we don't have magnitude and direction, we can't calculate the U and V values
+ if (magnitudeName is None) or (directionName is None) :
+ return None, None
+
+ # load the magnitude and direction data sets
+ magnitude = _load_variable_data(fileObject, magnitudeName)
+ direction = _load_variable_data(fileObject, directionName)
+
+ # convert the magnitude and direction data into u and v vectors
+ uData, vData = delta.convert_mag_dir_to_U_V_vector(magnitude, direction, invalidMask=invalidMask)
+
+ return uData, vData
+
+def rsync_or_copy_files (list_of_files, target_directory='.', additionalFileNameSuffix='') :
"""
If the files in the list are remote, rsync them, otherwise, just copy
them to the target directory
"""
+ newPaths = [ ]
+
for file_uri in list_of_files :
+ fileName = os.path.split(file_uri)[1]
+ baseFile, ext = os.path.splitext(fileName)
+ newPath = os.path.join(target_directory, baseFile + additionalFileNameSuffix + ext)
+ newPaths.append(newPath)
+
if _uri_needs_rsync(file_uri) :
- cmd = ['rsync', '-Cuav', file_uri, os.path.join(target_directory, os.path.split(file_uri)[1])]
+ cmd = ['rsync', '-Cuav', file_uri, newPath]
else :
- cmd = ['cp', os.path.abspath(file_uri), os.path.join(target_directory, os.path.split(file_uri)[1])]
+ cmd = ['cp', os.path.abspath(file_uri), newPath]
LOG.debug('running ' + ' '.join(cmd))
sh(cmd)
+
+ return newPaths
def colocateToFile_library_call(a_path, b_path, var_list=[ ],
options_set={ },
@@ -823,10 +851,10 @@ def colocateToFile_library_call(a_path, b_path, var_list=[ ],
LOG.info("Creating output directory.")
os.makedirs(pathsTemp['out'])
- # make copies of the input files for colocation
- rsync_or_copy_files ([pathsTemp['a'], pathsTemp['b']], target_directory=pathsTemp['out'])
- pathsTemp['a'] = os.path.join(pathsTemp['out'], os.path.split(pathsTemp['a'])[1])
- pathsTemp['b'] = os.path.join(pathsTemp['out'], os.path.split(pathsTemp['b'])[1])
+ # make copies of the input files for colocation TODO, fix paths
+ [pathsTemp['a'], pathsTemp['b']] = rsync_or_copy_files ([pathsTemp['a'], pathsTemp['b']],
+ target_directory=pathsTemp['out'],
+ additionalFileNameSuffix='-collocated')
# open the files
LOG.info("Processing File A:")
@@ -905,6 +933,10 @@ def colocateToFile_library_call(a_path, b_path, var_list=[ ],
# colocate the data for this variable if we have longitude/latitude data
if (len(lon_lat_data.keys()) > 0) and runInfo['doColocate'] :
+ # figure out the invalid masks
+ invalidA = lon_lat_data['a']['inv_mask'] | (aData == varRunInfo['missing_value'])
+ invalidB = lon_lat_data['b']['inv_mask'] | (bData == varRunInfo['missing_value_alt_in_b'])
+
# match up our points in A and B
(aData, bData, (numberOfMultipleMatchesInA, numberOfMultipleMatchesInB)), \
(aUnmatchedData, unmatchedALongitude, unmatchedALatitude), \
@@ -915,8 +947,8 @@ def colocateToFile_library_call(a_path, b_path, var_list=[ ],
missingData=varRunInfo['missing_value'],
altMissingDataInB=varRunInfo['missing_value_alt_in_b'],
# TODO, should missing data be considered?
- invalidAMask=lon_lat_data['a']['inv_mask'],
- invalidBMask=lon_lat_data['b']['inv_mask'])
+ invalidAMask=invalidA,
+ invalidBMask=invalidB)
LOG.debug(str(numberOfMultipleMatchesInA) + " data pairs contain A data points used for multiple matches.")
LOG.debug(str(numberOfMultipleMatchesInB) + " data pairs contain B data points used for multiple matches.")
@@ -924,15 +956,24 @@ def colocateToFile_library_call(a_path, b_path, var_list=[ ],
LOG.debug(str(len(bUnmatchedData)) + " B data points could not be matched.")
# save the colocated data information in the output files
+
+ # all the a file information
aFile.create_new_variable(technical_name + '-colocated', # TODO, how should this suffix be handled?
missingvalue = varRunInfo['missing'] if 'missing' in varRunInfo else None,
data = aData,
variabletocopyattributesfrom = technical_name)
+ aFile.add_attribute_data_to_variable(technical_name + '-colocated', 'number of multiple matches', numberOfMultipleMatchesInA)
+ aFile.add_attribute_data_to_variable(technical_name + '-colocated', 'number of unmatched points', len(aUnmatchedData))
+
+ # all the b file information
bFile.create_new_variable(b_variable_technical_name + '-colocated', # TODO, how should this suffix be handled?
missingvalue = varRunInfo['missing_value_alt_in_b'] if 'missing_value_alt_in_b' in varRunInfo else None,
data = bData,
variabletocopyattributesfrom = b_variable_technical_name)
- # TODO, save the unmatched data and info on multiple matches
+ bFile.add_attribute_data_to_variable(b_variable_technical_name + '-colocated', 'number of multiple matches', numberOfMultipleMatchesInB)
+ bFile.add_attribute_data_to_variable(b_variable_technical_name + '-colocated', 'number of unmatched points', len(bUnmatchedData))
+
+ # TODO, any additional statistics
else :
LOG.debug(explanationName + " was not selected for colocation and will be ignored.")
@@ -1061,7 +1102,8 @@ def reportGen_library_call (a_path, b_path, var_list=[ ],
do_not_test_with_lon_lat = (not include_images_for_this_variable) or (len(lon_lat_data.keys()) <= 0)
# handle vector data
- isVectorData = False # TODO actually figure out if we have vector data from user inputted settings
+ isVectorData = ( ('magnitudeName' in varRunInfo) and ('directionName' in varRunInfo) and
+ ('magnitudeBName' in varRunInfo) and ('directionBName' in varRunInfo) )
# check if this data can be displayed but
# don't compare lon/lat sizes if we won't be plotting
@@ -1142,6 +1184,18 @@ def reportGen_library_call (a_path, b_path, var_list=[ ],
else :
plotFunctionGenerationObjects.append(plotcreate.MappedContourPlotFunctionFactory())
+ # if there's magnitude and direction data, figure out the u and v, otherwise these will be None
+ aUData, aVData = _get_UV_info_from_magnitude_direction_info (aFile,
+ varRunInfo['magnitudeName'] if ('magnitudeName') in varRunInfo else None,
+ varRunInfo['directionName'] if ('directionName') in varRunInfo else None,
+ lon_lat_data['a']['inv_mask']
+ if ('a' in lon_lat_data) and ('inv_mask' in lon_lat_data['a']) else None)
+ bUData, bVData = _get_UV_info_from_magnitude_direction_info (bFile,
+ varRunInfo['magnitudeBName'] if ('magnitudeBName') in varRunInfo else None,
+ varRunInfo['directionBName'] if ('directionBName') in varRunInfo else None,
+ lon_lat_data['b']['inv_mask']
+ if ('b' in lon_lat_data) and ('inv_mask' in lon_lat_data['b']) else None)
+
# plot our lon/lat related info
image_names['original'], image_names['compared'] = \
plot.plot_and_save_comparison_figures \
@@ -1160,7 +1214,9 @@ def reportGen_library_call (a_path, b_path, var_list=[ ],
doFork=runInfo['doFork'],
shouldClearMemoryWithThreads=runInfo['useThreadsToControlMemory'],
shouldUseSharedRangeForOriginal=runInfo['useSharedRangeForOriginal'],
- doPlotSettingsDict = varRunInfo)
+ doPlotSettingsDict = varRunInfo,
+ aUData=aUData, aVData=aVData,
+ bUData=bUData, bVData=bVData,)
print("\tfinished creating figures for: " + explanationName)
@@ -1354,9 +1410,12 @@ python -m glance
List available variables for comparison.
"""
for fn in args:
- lal = list(io.open(fn)())
- lal.sort()
- print fn + ': ' + ('\n ' + ' '*len(fn)).join(lal)
+ try :
+ lal = list(io.open(fn)())
+ lal.sort()
+ print fn + ': ' + ('\n ' + ' '*len(fn)).join(lal)
+ except KeyError :
+ LOG.warn('Unable to open / process file selection: ' + fn)
def sdr_cris(*args):
"""compare sdr_cris output
diff --git a/pyglance/glance/delta.py b/pyglance/glance/delta.py
index fc14e56..38e5e2c 100644
--- a/pyglance/glance/delta.py
+++ b/pyglance/glance/delta.py
@@ -770,18 +770,19 @@ def create_colocated_data_with_lon_lat_colocation(listOfColocatedALonLat, listOf
[aLon, aLat, aIndex, aMatches] = listOfColocatedALonLat[aIndex]
tempMatches = 0
+ isInvalidA = invalidAMask[aIndex]
# for each point that matched to a given a point
for matchIndex in sorted(aMatches) :
[bLon, bLat, bIndex, bMatches] = listOfColocatedBLonLat[matchIndex]
+ isInvalidB = invalidBMask[bIndex]
# if either of our data points is invalid, then the data doesn't match
- if invalidBMask[matchIndex] or invalidAMask[aIndex] :
+ if isInvalidA or isInvalidB :
# fill in missing data in the matches
matchedAPoints[currentIndex] = missingData
matchedBPoints[currentIndex] = altMissingDataInB
-
else: # we have a valid match!
tempMatches = tempMatches + 1
matchedAPoints[currentIndex] = aData[aIndex]
@@ -792,7 +793,7 @@ def create_colocated_data_with_lon_lat_colocation(listOfColocatedALonLat, listOf
totalValidMatchedPairs = totalValidMatchedPairs + tempMatches
if tempMatches > 1 :
multipleMatchesInA = multipleMatchesInA + tempMatches
- elif tempMatches <= 0 :
+ elif (tempMatches <= 0) and (not isInvalidA) :
unmatchedAPoints.append(aData[aIndex])
unmatchedALongitude.append(aLon)
unmatchedALatitude.append(aLat)
@@ -803,14 +804,16 @@ def create_colocated_data_with_lon_lat_colocation(listOfColocatedALonLat, listOf
[bLon, bLat, bIndex, bMatches] = listOfColocatedBLonLat[bIndex]
tempMatches = 0
+ isInvalidB = invalidBMask[bIndex]
# for each point that matched to a given b point
for matchIndex in sorted(bMatches) :
[aLon, aLat, aIndex, aMatches] = listOfColocatedALonLat[matchIndex]
+ isInvalidA = invalidAMask[aIndex]
# if either of our data points is invalid, then the data doesn't match
- if invalidAMask[matchIndex] or invalidBMask[bIndex] :
+ if isInvalidB or isInvalidA :
# we've already built our matched data, so no need to missing it out
pass
else: # we have a valid match!
@@ -818,7 +821,7 @@ def create_colocated_data_with_lon_lat_colocation(listOfColocatedALonLat, listOf
if tempMatches > 1 :
multipleMatchesInB = multipleMatchesInB + tempMatches
- elif tempMatches <= 0 :
+ elif (tempMatches <= 0) and (not isInvalidB) :
unmatchedBPoints.append(bData[bIndex])
unmatchedBLongitude.append(bLon)
unmatchedBLatitude.append(bLat)
diff --git a/pyglance/glance/figures.py b/pyglance/glance/figures.py
index 0a7af9f..896cb1f 100644
--- a/pyglance/glance/figures.py
+++ b/pyglance/glance/figures.py
@@ -402,7 +402,7 @@ def create_quiver_mapped_figure(data, latitude, longitude, baseMapInstance, boun
# draw our data placed on a map
maps.draw_basic_features(baseMapInstance, boundingAxes)
- bMap, x, y = maps.show_quiver_plot (longitudeClean, latitudeClean, baseMapInstance, (uData, vData), colordata=colorData)
+ bMap, x, y = maps.show_quiver_plot (longitudeClean, latitudeClean, baseMapInstance, (uDataClean, vDataClean), colordata=colorData)
# show the title
axes.set_title(title)
diff --git a/pyglance/glance/io.py b/pyglance/glance/io.py
index 41d99ea..cdc1c6e 100644
--- a/pyglance/glance/io.py
+++ b/pyglance/glance/io.py
@@ -12,6 +12,7 @@ import os, sys, logging
from pyhdf.SD import SD,SDC, SDS, HDF4Error
try:
import h5py
+ from h5py import h5d
except ImportError:
pass
from pycdf import CDF, NC, strerror
@@ -248,6 +249,22 @@ class nc(CDF):
newVariable.put(data.tolist())
return newVariable
+
+ def add_attribute_data_to_variable(self, variableName, newAttributeName, newAttributeValue) :
+ """
+ if the attribute exists for the given variable, set it to the new value
+ if the attribute does not exist for the given variable, create it and set it to the new value
+ """
+ variableObject = self.get_variable_object(variableName)
+
+ self.redef()
+
+ variableObject.__setattr__(newAttributeName, newAttributeValue)
+
+ self.enddef()
+
+ return
+
nc4 = nc
cdf = nc
@@ -278,7 +295,7 @@ class h5(object):
try :
tempType = obj.dtype # this is required to provoke a type error for closed data sets
- LOG.debug ('type: ' + str(tempType))
+ #LOG.debug ('type: ' + str(tempType))
variableList.append(name)
except TypeError :
LOG.debug('TypeError prevents the use of variable ' + name
@@ -299,6 +316,7 @@ class h5(object):
# for scaling it will be (so the return type may not reflect the
# type found in the original file)
def __getitem__(self, name):
+
# defaults
scale_factor = 1.0
add_offset = 0.0
@@ -308,6 +326,11 @@ class h5(object):
# get our raw data and scaling info
variable_object = self.get_variable_object(name)
raw_data_copy = variable_object[:]
+
+ #print ('*************************')
+ #print (dir (variable_object.id)) # TODO, is there a way to get the scale and offset through this?
+ #print ('*************************')
+
# load the scale factor and add offset
if ('scale_factor' in variable_object.attrs) :
scale_factor = variable_object.attrs['scale_factor']
@@ -335,7 +358,19 @@ class h5(object):
return h5.trav(self._h5, name)
def missing_value(self, name):
- return None
+
+ toReturn = None
+
+ # get the missing value if it has been set
+ variableObject = self.get_variable_object(name)
+ pListObj = variableObject.id.get_create_plist()
+ fillValueStatus = pListObj.fill_value_defined()
+ if (h5d.FILL_VALUE_DEFAULT is fillValueStatus) or (h5d.FILL_VALUE_USER_DEFINED is fillValueStatus) :
+ temp = np.array((1), dtype=variableObject.dtype)
+ pListObj.get_fill_value(temp)
+ toReturn = temp
+
+ return toReturn
def open(pathname, allowWrite=False):
--
GitLab