Skip to content
Snippets Groups Projects
Commit e2064139 authored by (no author)'s avatar (no author)
Browse files

adding testing code to use fork to control memory use bloat when plotting;... 

adding testing code to use fork to control memory use bloat when plotting; changed to getting type of scaled data from loaded offset/scale

git-svn-id: https://svn.ssec.wisc.edu/repos/glance/trunk@76 8a9318a1-56ba-4d59-b755-99d26321be01
parent dbe7d517
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
pyglance/glance/io.py
+ 8
− 3
View file @ e2064139
......@@ -41,7 +41,7 @@ class hdf(SD):
# defaults
scale_factor = 1.0
add_offset = 0.0
data_type = np.float32 # TODO temporary
data_type = None
scaling_method = None
# get the variable object and use it to
......@@ -54,10 +54,12 @@ class hdf(SD):
except HDF4Error:
# load just the scale factor and add offset
temp_attributes = variable_object.attributes()
if ('scale_factor' in temp_attributes) :
scale_factor = temp_attributes['scale_factor']
if ('add_offset' in temp_attributes) :
add_offset = temp_attributes['add_offset']
data_type = np.dtype(type(add_offset))
if ('scale_factor' in temp_attributes) :
scale_factor = temp_attributes['scale_factor']
data_type = np.dtype(type(scale_factor))
if ('scaling_method' in temp_attributes) :
scaling_method = temp_attributes['scaling_method']
SDS.endaccess(variable_object)
......@@ -80,6 +82,9 @@ class hdf(SD):
LOG.warn ('Scaling method of \"' + str(scaling_method) + '\" will be ignored in favor of hdf standard method. '
+ 'This may cause problems with data consistency')
# if we don't have a data type something strange has gone wrong
assert(not (data_type is None))
# get information about where the data is the missing value
missing_val = self.missing_value(name)
missing_mask = np.zeros(raw_data_copy.shape, dtype=np.bool)
......
pyglance/glance/plot.py
+ 48
− 21
View file @ e2064139
......@@ -492,6 +492,10 @@ def plot_and_save_figure_comparison(aData, bData,
# for descriptive purposes if it is not defined
}
"""
# TODO Temporary for testing
testOverrideForMemoryThroughFork = False
# if we weren't given a variable display name,
# just use the standard variable name instead
variableName = variableRunInfo['variable_name']
......@@ -561,10 +565,13 @@ def plot_and_save_figure_comparison(aData, bData,
"\t\tsaving image of " + variableDisplayName + " for file a",
outputPath + "/" + variableName + ".A.png",
outputPath + "/" + variableName + ".A.small.png",
makeSmall, doFork)
makeSmall, doFork or testOverrideForMemoryThroughFork)
if not (pid is 0) :
childPids.append(pid)
LOG.debug ("Started child process (pid: " + str(pid) + ") to create file a image for " + variableDisplayName)
if doFork :
childPids.append(pid)
LOG.debug ("Started child process (pid: " + str(pid) + ") to create file a image for " + variableDisplayName)
else :
os.waitpid(pid, 0)
# the original B data
LOG.info("\t\tcreating image of " + variableDisplayName + " in file b")
......@@ -578,10 +585,13 @@ def plot_and_save_figure_comparison(aData, bData,
"\t\tsaving image of " + variableDisplayName + " for file b",
outputPath + "/" + variableName + ".B.png",
outputPath + "/" + variableName + ".B.small.png",
makeSmall, doFork)
makeSmall, doFork or testOverrideForMemoryThroughFork)
if not (pid is 0) :
childPids.append(pid)
LOG.debug ("Started child process (pid: " + str(pid) + ") to create file b image for " + variableDisplayName)
if doFork:
childPids.append(pid)
LOG.debug ("Started child process (pid: " + str(pid) + ") to create file b image for " + variableDisplayName)
else :
os.waitpid(pid, 0)
# make the data comparison figures
if not shortCircuitComparisons :
......@@ -596,10 +606,14 @@ def plot_and_save_figure_comparison(aData, bData,
"\t\tsaving image of the absolute value of difference for " + variableDisplayName,
outputPath + "/" + variableName + ".AbsDiff.png",
outputPath + "/" + variableName + ".AbsDiff.small.png",
makeSmall, doFork)
makeSmall, doFork or testOverrideForMemoryThroughFork)
if not (pid is 0) :
childPids.append(pid)
LOG.debug ("Started child process (pid: " + str(pid) + ") to create absolute value of difference image for " + variableDisplayName)
if doFork :
childPids.append(pid)
LOG.debug ("Started child process (pid: " + str(pid)
+ ") to create absolute value of difference image for " + variableDisplayName)
else :
os.waitpid(pid, 0)
# the subtraction of one data set from the other
LOG.info("\t\tcreating image of the difference in " + variableDisplayName)
......@@ -610,10 +624,13 @@ def plot_and_save_figure_comparison(aData, bData,
"\t\tsaving image of the difference in " + variableDisplayName,
outputPath + "/" + variableName + ".Diff.png",
outputPath + "/" + variableName + ".Diff.small.png",
makeSmall, doFork)
makeSmall, doFork or testOverrideForMemoryThroughFork)
if not (pid is 0) :
childPids.append(pid)
LOG.debug ("Started child process (pid: " + str(pid) + ") to create difference image for " + variableDisplayName)
if doFork :
childPids.append(pid)
LOG.debug ("Started child process (pid: " + str(pid) + ") to create difference image for " + variableDisplayName)
else :
os.waitpid(pid, 0)
# mark the trouble points
LOG.info("\t\tcreating image marking trouble data in " + variableDisplayName)
......@@ -632,10 +649,13 @@ def plot_and_save_figure_comparison(aData, bData,
"\t\tsaving image marking trouble data in " + variableDisplayName,
outputPath + "/" + variableName + ".Trouble.png",
outputPath + "/" + variableName + ".Trouble.small.png",
makeSmall, doFork)
makeSmall, doFork or testOverrideForMemoryThroughFork)
if not (pid is 0) :
childPids.append(pid)
LOG.debug ("Started child process (pid: " + str(pid) + ") to create trouble image for " + variableDisplayName)
if doFork :
childPids.append(pid)
LOG.debug ("Started child process (pid: " + str(pid) + ") to create trouble image for " + variableDisplayName)
else :
os.waitpid(pid, 0)
# a histogram of the values of fileA - file B
LOG.info("\t\tcreating histogram of the amount of difference in " + variableDisplayName)
......@@ -649,10 +669,14 @@ def plot_and_save_figure_comparison(aData, bData,
"\t\tsaving histogram of the amount of difference in " + variableDisplayName,
outputPath + "/" + variableName + ".Hist.png",
outputPath + "/" + variableName + ".Hist.small.png",
makeSmall, doFork)
makeSmall, doFork or testOverrideForMemoryThroughFork)
if not (pid is 0) :
childPids.append(pid)
LOG.debug ("Started child process (pid: " + str(pid) + ") to create difference histogram image for " + variableDisplayName)
if doFork :
childPids.append(pid)
LOG.debug ("Started child process (pid: " + str(pid)
+ ") to create difference histogram image for " + variableDisplayName)
else :
os.waitpid(pid, 0)
# scatter plot of file a vs file b values
LOG.info("\t\tcreating scatter plot of file a values vs file b values for " + variableDisplayName)
......@@ -665,10 +689,13 @@ def plot_and_save_figure_comparison(aData, bData,
"\t\tsaving scatter plot of file a values vs file b values in " + variableDisplayName,
outputPath + "/" + variableName + ".Scatter.png",
outputPath + "/" + variableName + ".Scatter.small.png",
makeSmall, doFork)
makeSmall, doFork or testOverrideForMemoryThroughFork)
if not (pid is 0) :
childPids.append(pid)
LOG.debug ("Started child process (pid: " + str(pid) + ") to create scatter plot image for " + variableDisplayName)
if doFork :
childPids.append(pid)
LOG.debug ("Started child process (pid: " + str(pid) + ") to create scatter plot image for " + variableDisplayName)
else :
os.waitpid(pid, 0)
# now we need to wait for all of our child processes to terminate before returning
if (isParent) : # just in case
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment