Skip to content
Snippets Groups Projects
Commit 90c052bb authored by (no author)'s avatar (no author)
Browse files

various refactoring to handle data without lon/lat; new image making methods... 

various refactoring to handle data without lon/lat; new image making methods for 2D data just shown as an image (not mapped) and for line plots; changed plotting module to use a pseudo-factory pattern for generating correct plot set; created new filter to make ipopp data into an image; created new filter to collapse multi-dimensional data

git-svn-id: https://svn.ssec.wisc.edu/repos/glance/trunk@92 8a9318a1-56ba-4d59-b755-99d26321be01
parent c8cf8b4f
No related branches found
No related tags found
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
pyglance/glance/compare.py
+ 143
− 64
View file @ 90c052bb
This diff is collapsed.
pyglance/glance/filters.py
+ 95
− 1
View file @ 90c052bb
......@@ -7,7 +7,7 @@ Created by Eva Schiffer August 2009.
Copyright (c) 2009 University of Wisconsin SSEC. All rights reserved.
"""
import numpy as np
import numpy as np
def trim_off_of_top (data, num_elements_to_trim) :
"""
......@@ -161,5 +161,99 @@ def set_to_value_between_bounds(data, value_to_set_to, bottom_bound_exclusive, t
return data
def collapse_to_index(data, index, collapsing_function=np.mean,
missing_value=None, ignore_below_exclusive=None, ignore_above_exclusive=None) :
"""
collapse the data given along the selected index, using the collapsing_function
"""
# figure the order to put the index we want first
new_index_order = range(len(data.shape))
del(new_index_order[index])
new_index_order = [index] + new_index_order
# copy our data and put the index we won't collapse first
new_data = data.copy()
new_data = new_data.transpose(new_index_order)
# find any bad points that we don't want to collapse
bad_mask = np.zeros(new_data.shape, dtype=bool)
if missing_value is not None :
bad_mask[new_data == missing_value] = True
if ignore_above_exclusive is not None :
bad_mask[new_data > ignore_above_exclusive] = True
if ignore_below_exclusive is not None :
bad_mask[new_data < ignore_below_exclusive] = True
# collapse any non bad data
final_num_pts = new_data.shape[0]
collapsed_data = np.zeros((final_num_pts), dtype=new_data.dtype)
for index_1_val in range(final_num_pts) :
values_to_use = new_data[index_1_val][~(bad_mask[index_1_val])]
if len(values_to_use) > 0 :
collapsed_data[index_1_val] = collapsing_function(values_to_use)
else:
collapsed_data[index_1_val] = missing_value
return collapsed_data
def organize_ipopp_data_into_image(original_ipopp_data, wave_number=None, missing_value=None,
propagate_partial_missing_values=False) :
"""
organize the ipopp data spatially into an 'image' of sorts
this basically consists of:
-> the 30 fields of regard
---------------
| | |
| | V
| | the 4 scan lines
| |
---------------
for each field of regard/scan line
_______
|_|_|_|
|_|_|_|
|_|_|_|
block of the 9 detectors (TODO order?
for the moment assuming increasing linear by line then field of regard)
for each detector point, if the wave_number was given
in the parameters, that specific interferogram data pt will be used
if no wave_number was given, the mean of the 717 pts will be used
"""
new_data_image = np.zeros((4 * 3, 30 * 3), dtype=original_ipopp_data.dtype)
# loop to the place in the old array to get each data point
for scan_line in range(4) :
for field_of_regard in range(30) :
for detector in range(9) :
# figure out the value we're moving
data_pt = None
data_array = original_ipopp_data[scan_line][field_of_regard][detector]
if (wave_number is not None):
data_pt = data_array[wave_number]
else:
if (propagate_partial_missing_values
and
sum(data_array == missing_value) > 0) :
data_pt = missing_value
else:
# remember to remove any remaining missing values
data_pt = np.mean(data_array[~(data_array == missing_value)])
# figure out where to put the value and put it there
index1 = scan_line * 3 + (detector / 3)
index2 = field_of_regard * 3 + (detector % 3)
new_data_image[index1][index2] = data_pt
return new_data_image
if __name__=='__main__':
sys.exit(main())
\ No newline at end of file
pyglance/glance/mainreport.txt
+ 98
− 92
View file @ 90c052bb
......@@ -47,107 +47,113 @@ Copyright (c) 2009 University of Wisconsin SSEC. All rights reserved.
</p>
% endif
## display the latitude and longitude variable names
<p>
% if ('latitude_alt_name_in_b' in runInfo) :
latitude in A: ${runInfo['latitude']}<br>
latitude in B: ${runInfo['latitude_alt_name_in_b']}<br>
% else :
latitude: ${runInfo['latitude']} <br>
% endif
% if ('longitude_alt_name_in_b' in runInfo) :
longitude in A: ${runInfo['longitude']}<br>
longitude in B: ${runInfo['longitude_alt_name_in_b']}<br>
% else :
longitude: ${runInfo['longitude']}<br>
% endif
% if ('lon_lat_epsilon' in runInfo) and (runInfo['lon_lat_epsilon'] > 0.0) :
longitude/latitude comparison epsilon: ${runInfo['lon_lat_epsilon']}<br>
% endif
<br>
## display information about any data filtering on the lons/lats
% if ('data_filter_function_lat_in_a' in runInfo) and (not (runInfo['data_filter_function_lat_in_a'] is None)) :
Note: The latitude in file A was filtered.
Please see <a href="${runInfo['config_file_name']}">this copy of
the original configuration file</a> to view the data filtering function.<br>
% endif
% if ('data_filter_function_lat_in_b' in runInfo) and (not (runInfo['data_filter_function_lat_in_b'] is None)) :
Note: The latitude in file B was filtered.
Please see <a href="${runInfo['config_file_name']}">this copy of
the original configuration file</a> to view the data filtering function.<br>
% endif
% if ('data_filter_function_lon_in_a' in runInfo) and (not (runInfo['data_filter_function_lon_in_a'] is None)) :
Note: The longitude in file A was filtered.
Please see <a href="${runInfo['config_file_name']}">this copy of
the original configuration file</a> to view the data filtering function.<br>
% endif
% if ('data_filter_function_lon_in_b' in runInfo) and (not (runInfo['data_filter_function_lon_in_b'] is None)) :
Note: The longitude in file B was filtered.
Please see <a href="${runInfo['config_file_name']}">this copy of
the original configuration file</a> to view the data filtering function.<br>
% endif
</p>
## if the lon/lat variables exist, display info on them
%if ('latitude' in runInfo) and ('longitude' in runInfo) :
## if there is a problem with the longitude/latitude correlation between the two files,
## make a nice big warning for the user
% if spatial.has_key('lon_lat_not_equal_points_count') and (spatial['lon_lat_not_equal_points_count'] > 0) :
## display the latitude and longitude variable names
<p>
WARNING: ${spatial['lon_lat_not_equal_points_count']} data points
(${report.make_formatted_display_string(spatial['lon_lat_not_equal_points_percent'])}% of all data)
show possible mismatch in values stored in file a
and file b longitude and latitude values. Depending on the degree of mismatch, some data value comparisons
in this report may be distorted or spatially nonsensical. Please consider re-running this report and including an
examination of your longitude and latitude variables with appropriate epsilons in order to analyze the significance
of the difference.<br>
## if we're showing images, link to graphs showing the problem
% if runInfo['shouldIncludeImages'] :
<a href="./LonLatMismatch.A.png">
View mismatching points in A's lon/lat system
</a><br>
<a href="./LonLatMismatch.B.png">
View mismatching points in B's lon/lat system
</a>
% if ('latitude_alt_name_in_b' in runInfo) :
latitude in A: ${runInfo['latitude']}<br>
latitude in B: ${runInfo['latitude_alt_name_in_b']}<br>
% else :
latitude: ${runInfo['latitude']} <br>
% endif
</p>
% endif
## if we have some unique spatially invalid points, report them
## are there any in file A?
%if spatial.has_key('file A') and spatial['file A'].has_key('numInvPts') and (spatial['file A']['numInvPts'] > 0) :
<%
fileAInfo = spatial['file A']
%>
<p>
% if runInfo['shouldIncludeImages'] :
<a href="./SpatialMismatch.A.png"><img src="./SpatialMismatch.A.small.png"></a><br>
% if ('longitude_alt_name_in_b' in runInfo) :
longitude in A: ${runInfo['longitude']}<br>
longitude in B: ${runInfo['longitude_alt_name_in_b']}<br>
% else :
longitude: ${runInfo['longitude']}<br>
% endif
File A has ${fileAInfo['numInvPts']} valid data points not present in File B.<br>
File A has a total of ${report.make_formatted_display_string(fileAInfo['perInvPts'])}% invalid longitude/latitude data. <br>
</p>
% endif
## are there any in file B?
%if spatial.has_key('file B') and spatial['file B'].has_key('numInvPts') and (spatial['file B']['numInvPts'] > 0) :
<%
fileBInfo = spatial['file B']
%>
<p>
% if runInfo['shouldIncludeImages'] :
<a href="./SpatialMismatch.B.png"><img src="./SpatialMismatch.B.small.png"></a><br>
% if ('lon_lat_epsilon' in runInfo) and (runInfo['lon_lat_epsilon'] > 0.0) :
longitude/latitude comparison epsilon: ${runInfo['lon_lat_epsilon']}<br>
% endif
<br>
## display information about any data filtering on the lons/lats
% if ('data_filter_function_lat_in_a' in runInfo) and (not (runInfo['data_filter_function_lat_in_a'] is None)) :
Note: The latitude in file A was filtered.
Please see <a href="${runInfo['config_file_name']}">this copy of
the original configuration file</a> to view the data filtering function.<br>
% endif
% if ('data_filter_function_lat_in_b' in runInfo) and (not (runInfo['data_filter_function_lat_in_b'] is None)) :
Note: The latitude in file B was filtered.
Please see <a href="${runInfo['config_file_name']}">this copy of
the original configuration file</a> to view the data filtering function.<br>
% endif
% if ('data_filter_function_lon_in_a' in runInfo) and (not (runInfo['data_filter_function_lon_in_a'] is None)) :
Note: The longitude in file A was filtered.
Please see <a href="${runInfo['config_file_name']}">this copy of
the original configuration file</a> to view the data filtering function.<br>
% endif
% if ('data_filter_function_lon_in_b' in runInfo) and (not (runInfo['data_filter_function_lon_in_b'] is None)) :
Note: The longitude in file B was filtered.
Please see <a href="${runInfo['config_file_name']}">this copy of
the original configuration file</a> to view the data filtering function.<br>
% endif
File B has ${fileBInfo['numInvPts']} valid data points not present in File A.<br>
File B has a total of ${report.make_formatted_display_string(fileBInfo['perInvPts'])}% longitude/latitude invalid data. <br>
</p>
% endif
## report on shared spatial invalidity if there is any
% if spatial.has_key('perInvPtsInBoth') :
<% perInBoth = spatial['perInvPtsInBoth'] %>
% if perInBoth > 0 :
## if there is a problem with the longitude/latitude correlation between the two files,
## make a nice big warning for the user
% if spatial.has_key('lon_lat_not_equal_points_count') and (spatial['lon_lat_not_equal_points_count'] > 0) :
<p>
lon/lat data that is invalid in either file A or file B: ${report.make_formatted_display_string(perInBoth)}%
WARNING: ${spatial['lon_lat_not_equal_points_count']} data points
(${report.make_formatted_display_string(spatial['lon_lat_not_equal_points_percent'])}% of all data)
show possible mismatch in values stored in file a
and file b longitude and latitude values. Depending on the degree of mismatch, some data value comparisons
in this report may be distorted or spatially nonsensical. Please consider re-running this report and including an
examination of your longitude and latitude variables with appropriate epsilons in order to analyze the significance
of the difference.<br>
## if we're showing images, link to graphs showing the problem
% if runInfo['shouldIncludeImages'] :
<a href="./LonLatMismatch.A.png">
View mismatching points in A's lon/lat system
</a><br>
<a href="./LonLatMismatch.B.png">
View mismatching points in B's lon/lat system
</a>
% endif
</p>
% endif
## if we have some unique spatially invalid points, report them
## are there any in file A?
%if spatial.has_key('file A') and spatial['file A'].has_key('numInvPts') and (spatial['file A']['numInvPts'] > 0) :
<%
fileAInfo = spatial['file A']
%>
<p>
% if runInfo['shouldIncludeImages'] :
<a href="./SpatialMismatch.A.png"><img src="./SpatialMismatch.A.small.png"></a><br>
% endif
File A has ${fileAInfo['numInvPts']} valid data points not present in File B.<br>
File A has a total of ${report.make_formatted_display_string(fileAInfo['perInvPts'])}% invalid longitude/latitude data. <br>
</p>
% endif
## are there any in file B?
%if spatial.has_key('file B') and spatial['file B'].has_key('numInvPts') and (spatial['file B']['numInvPts'] > 0) :
<%
fileBInfo = spatial['file B']
%>
<p>
% if runInfo['shouldIncludeImages'] :
<a href="./SpatialMismatch.B.png"><img src="./SpatialMismatch.B.small.png"></a><br>
% endif
File B has ${fileBInfo['numInvPts']} valid data points not present in File A.<br>
File B has a total of ${report.make_formatted_display_string(fileBInfo['perInvPts'])}% longitude/latitude invalid data. <br>
</p>
% endif
## report on shared spatial invalidity if there is any
% if spatial.has_key('perInvPtsInBoth') :
<% perInBoth = spatial['perInvPtsInBoth'] %>
% if perInBoth > 0 :
<p>
lon/lat data that is invalid in either file A or file B: ${report.make_formatted_display_string(perInBoth)}%
</p>
% endif
% endif
## end of the if to display lon/lat info
% endif
% if len(variables.keys()) > 0 :
......
pyglance/glance/plot.py
+ 515
− 213
View file @ 90c052bb
This diff is collapsed.
pyglance/glance/variablereport.txt
+ 66
− 60
View file @ 90c052bb
......@@ -57,70 +57,76 @@ Copyright (c) 2009 University of Wisconsin SSEC. All rights reserved.
</p>
% endif
## display the latitude and longitude variable names
<p>
% if ('latitude_alt_name_in_b' in runInfo) :
latitude in A: ${runInfo['latitude']}<br>
latitude in B: ${runInfo['latitude_alt_name_in_b']}<br>
% else :
latitude: ${runInfo['latitude']} <br>
% endif
% if ('longitude_alt_name_in_b' in runInfo) :
longitude in A: ${runInfo['longitude']}<br>
longitude in B: ${runInfo['longitude_alt_name_in_b']}<br>
% else :
longitude: ${runInfo['longitude']}<br>
% endif
% if ('lon_lat_epsilon' in runInfo) and (runInfo['lon_lat_epsilon'] > 0.0) :
longitude/latitude comparison epsilon: ${runInfo['lon_lat_epsilon']}<br>
% endif
## if the lon/lat variables exist, display info on them
%if ('latitude' in runInfo) and ('longitude' in runInfo) :
<br>
## display information about any data filtering on the lons/lats
% if ('data_filter_function_lat_in_a' in runInfo) and (not (runInfo['data_filter_function_lat_in_a'] is None)) :
Note: The latitude in file A was filtered.
Please see <a href="../${runInfo['config_file_name']}">this copy of
the original configuration file</a> to view the data filtering function.<br>
% endif
% if ('data_filter_function_lat_in_b' in runInfo) and (not (runInfo['data_filter_function_lat_in_b'] is None)) :
Note: The latitude in file B was filtered.
Please see <a href="../${runInfo['config_file_name']}">this copy of
the original configuration file</a> to view the data filtering function.<br>
% endif
% if ('data_filter_function_lon_in_a' in runInfo) and (not (runInfo['data_filter_function_lon_in_a'] is None)) :
Note: The longitude in file A was filtered.
Please see <a href="../${runInfo['config_file_name']}">this copy of
the original configuration file</a> to view the data filtering function.<br>
% endif
% if ('data_filter_function_lon_in_b' in runInfo) and (not (runInfo['data_filter_function_lon_in_b'] is None)) :
Note: The longitude in file B was filtered.
Please see <a href="../${runInfo['config_file_name']}">this copy of
the original configuration file</a> to view the data filtering function.<br>
% endif
</p>
## if there is a problem with the longitude/latitude correlation between the two files,
## make a nice big warning for the user
% if spatial.has_key('lon_lat_not_equal_points_count') and (spatial['lon_lat_not_equal_points_count'] > 0) :
## display the latitude and longitude variable names
<p>
WARNING: ${spatial['lon_lat_not_equal_points_count']} data points
(${report.make_formatted_display_string(spatial['lon_lat_not_equal_points_percent'])}% of all data)
show possible mismatch in values stored in file a
and file b longitude and latitude values. Depending on the degree of mismatch, some data value comparisons
in this report may be distorted or spatially nonsensical. Please consider re-running this report and including an
examination of your longitude and latitude variables with appropriate epsilons in order to analyze the significance
of the difference.<br>
## if we're showing images, link to graphs showing the problem
% if runInfo['shouldIncludeImages'] :
<a href="../LonLatMismatch.A.png">
View mismatching points in A's lon/lat system
</a><br>
<a href="../LonLatMismatch.B.png">
View mismatching points in B's lon/lat system
</a>
% if ('latitude_alt_name_in_b' in runInfo) :
latitude in A: ${runInfo['latitude']}<br>
latitude in B: ${runInfo['latitude_alt_name_in_b']}<br>
% else :
latitude: ${runInfo['latitude']} <br>
% endif
% if ('longitude_alt_name_in_b' in runInfo) :
longitude in A: ${runInfo['longitude']}<br>
longitude in B: ${runInfo['longitude_alt_name_in_b']}<br>
% else :
longitude: ${runInfo['longitude']}<br>
% endif
% if ('lon_lat_epsilon' in runInfo) and (runInfo['lon_lat_epsilon'] > 0.0) :
longitude/latitude comparison epsilon: ${runInfo['lon_lat_epsilon']}<br>
% endif
<br>
## display information about any data filtering on the lons/lats
% if ('data_filter_function_lat_in_a' in runInfo) and (not (runInfo['data_filter_function_lat_in_a'] is None)) :
Note: The latitude in file A was filtered.
Please see <a href="../${runInfo['config_file_name']}">this copy of
the original configuration file</a> to view the data filtering function.<br>
% endif
% if ('data_filter_function_lat_in_b' in runInfo) and (not (runInfo['data_filter_function_lat_in_b'] is None)) :
Note: The latitude in file B was filtered.
Please see <a href="../${runInfo['config_file_name']}">this copy of
the original configuration file</a> to view the data filtering function.<br>
% endif
% if ('data_filter_function_lon_in_a' in runInfo) and (not (runInfo['data_filter_function_lon_in_a'] is None)) :
Note: The longitude in file A was filtered.
Please see <a href="../${runInfo['config_file_name']}">this copy of
the original configuration file</a> to view the data filtering function.<br>
% endif
% if ('data_filter_function_lon_in_b' in runInfo) and (not (runInfo['data_filter_function_lon_in_b'] is None)) :
Note: The longitude in file B was filtered.
Please see <a href="../${runInfo['config_file_name']}">this copy of
the original configuration file</a> to view the data filtering function.<br>
% endif
</p>
## if there is a problem with the longitude/latitude correlation between the two files,
## make a nice big warning for the user
% if spatial.has_key('lon_lat_not_equal_points_count') and (spatial['lon_lat_not_equal_points_count'] > 0) :
<p>
WARNING: ${spatial['lon_lat_not_equal_points_count']} data points
(${report.make_formatted_display_string(spatial['lon_lat_not_equal_points_percent'])}% of all data)
show possible mismatch in values stored in file a
and file b longitude and latitude values. Depending on the degree of mismatch, some data value comparisons
in this report may be distorted or spatially nonsensical. Please consider re-running this report and including an
examination of your longitude and latitude variables with appropriate epsilons in order to analyze the significance
of the difference.<br>
## if we're showing images, link to graphs showing the problem
% if runInfo['shouldIncludeImages'] :
<a href="../LonLatMismatch.A.png">
View mismatching points in A's lon/lat system
</a><br>
<a href="../LonLatMismatch.B.png">
View mismatching points in B's lon/lat system
</a>
% endif
</p>
% endif
## end of the if to display lon/lat info
% endif
% if (len(imageNames['original']) > 0) :
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment