Refactored data structures involved in report generation to be more flexible,...

Refactored data structures involved in report generation to be more flexible, consistent, and easier to understand

git-svn-id: https://svn.ssec.wisc.edu/repos/glance/trunk@15 8a9318a1-56ba-4d59-b755-99d26321be01

pyglance/glance/compare.py

@@ -454,6 +454,8 @@ python -m glance.compare plotDiffs A.hdf B.hdf [optional output path]
                                                    "B", outputPath, True)
             
         # set some things up to hold info for our reports
+        # this is going to be in the form
+        # [var_name] = {"passEpsilonPercent": percent ok with epsilon, "epsilon": epsilon)
         variableComparisons = {}
         
         # go through each of the possible variables in our files
@@ -491,11 +493,14 @@ python -m glance.compare plotDiffs A.hdf B.hdf [optional output path]
                     #get info on the variable
                     variableStats = delta.summarize(aData, bData, epsilon, (missing, missing), spaciallyInvalidMask)
                     # hang on to our good % and our epsilon value to describe our comparison
-                    variableComparisons[name] = ((1.0 - variableStats['outside_epsilon_fraction']) * 100.0, epsilon)
+                    variableComparisons[name] = {'passEpsilonPercent': ((1.0 - variableStats['outside_epsilon_fraction']) * 100.0),
+                                                 'epsilon': epsilon
+                                                 }
                     print ('generating report for: ' + name)
                     report.generate_and_save_variable_report(aFileName, bFileName, fileAmd5sum, fileBmd5sum, name,
+                                                             latitudeVariableName, longitudeVariableName, 
                                                              outputPath, name + ".html",
-                                                             epsilon, missing, variableStats, shouldGenerateImages,
+                                                             epsilon, missing, {'general': variableStats}, shouldGenerateImages,
                                                              lastModifiedTimeA, lastModifiedTimeB, currentTime,
                                                              currentUser, currentMachine)
                     

pyglance/glance/mainreport.txt

+<%doc>
+This Mako template is intended to create a summary report page for glance reportGen.
+
+Created by Eva Schiffer Jun 2009.
+Copyright (c) 2009 University of Wisconsin SSEC. All rights reserved.
+</%doc>
+
 <%!
     import glance.report as report
 %>
@@ -9,58 +16,57 @@
     <h2>Comparison Summary</h2>
     
     <p>
-        comparison produced ${currentTime} by user ${currentUser} on ${currentMachine}
+        comparison produced ${runInfo['time']} by user ${runInfo['user']} on ${runInfo['machine']}
     </p>
     
+    ## show information on each of the files we're comparing (should only be two)
+    % for fileKey in sorted(list(files)) :
+        <% tempFileInfo = files[fileKey] %>
+        <p>
+            ${fileKey}: ${tempFileInfo['path']} <br>
+            md5sum for ${fileKey}: ${tempFileInfo['md5sum']} <br>
+            last modified: ${tempFileInfo['lastModifiedTime']}
+        </p>
+    % endfor
     <p>
-        file A: ${fileAName} <br>
-        md5sum for file A: ${aMD5SUM} <br>
-        last modified: ${lastModifiedTimeA}
-    </p>
-    <p>
-        file B: ${fileBName} <br>
-        md5sum for file B: ${bMD5SUM} <br>
-        last modified: ${lastModifiedTimeB}
-    </p>
-    <p>
-        longitude: ${longitudeName} <br>
-        latitude: ${latitudeName} <br>
+        latitude: ${runInfo['latitude']} <br>
+        longitude: ${runInfo['longitude']}
     </p>
     
     ## if we have some unique spacially invalid points, report them
     <% haveUniqueSpacialInvalidity = False %>
-    %if not (numSpacInvInA is None) :
-        %if numSpacInvInA > 0 :
-            <% haveUniqueSpacialInvalidity = True %>
-            <p>
-                File A has ${numSpacInvInA} spacially valid data points not present in File B.<br>
-                File A has a total of ${report.make_formatted_display_string(perSpacInvPtsInA)}% spacially invalid data. <br>
-                <a href="./SpatialMismatch.A.png"><img src="./SpatialMismatch.A.small.png"></a>
-            </p>
-        % endif
-    %elif not (perSpacInvPtsInA is None) :
+    ## we will assume the whole substructure will be present if there's a problem with file A
+    ## that we need to report on specifically
+    %if spatial.has_key('file A') and (spatial['file A']['numInvPts'] > 0) :
+        <%
+            fileAInfo = spatial['file A']
+            haveUniqueSpacialInvalidity = True
+        %>
         <p>
-            File A has a total of ${report.make_formatted_display_string(perSpacInvPtsInA)}% spacially invalid data. <br>
+            File A has ${fileAInfo['numInvPts']} spacially valid data points not present in File B.<br>
+            File A has a total of ${report.make_formatted_display_string(fileAInfo['perInvPts'])}% spacially invalid data. <br>
+            <a href="./SpatialMismatch.A.png"><img src="./SpatialMismatch.A.small.png"></a>
         </p>
     % endif
-    %if not (numSpacInvInB is None) :
-        %if numSpacInvInB > 0 :
-            <% haveUniqueSpacialInvalidity = True %>
-            <p>
-                File B has ${numSpacInvInB} spacially valid data points not present in File A.<br>
-                File B has a total of ${report.make_formatted_display_string(perSpacInvPtsInB)}% spacially invalid data.<br>
-                <a href="./SpatialMismatch.B.png"><img src="./SpatialMismatch.B.small.png"></a>
-            </p>
-        % endif
-    %elif not (perSpacInvPtsInB is None) :
+    ## we will assume the whole substructure will be present if there's a problem with file B
+    ## that we need to report on specifically
+    %if spatial.has_key('file B') and (spatial['file B']['numInvPts'] > 0) :
+        <%
+            fileBInfo = spatial['file B']
+            haveUniqueSpacialInvalidity = True
+        %>
         <p>
-            File B has a total of ${report.make_formatted_display_string(perSpacInvPtsInB)}% spacially invalid data. <br>
+            File B has ${fileBInfo['numInvPts']} spacially valid data points not present in File A.<br>
+            File B has a total of ${report.make_formatted_display_string(fileBInfo['perInvPts'])}% spacially invalid data. <br>
+            <a href="./SpatialMismatch.B.png"><img src="./SpatialMismatch.B.small.png"></a>
         </p>
     % endif
-    % if not (perSpacInvPtsInBoth is None) :
-        % if perSpacInvPtsInBoth > 0 :
+    ## report on shared spatial invalidity if there is any
+    % if spatial.has_key('perInvPtsInBoth') :
+        <% perInBoth = spatial['perInvPtsInBoth'] %>
+        % if perInBoth > 0 :
             <p>
-                data that is spacially invalid in either file: ${report.make_formatted_display_string(perSpacInvPtsInBoth)}%
+                data that is spacially invalid in either file: ${report.make_formatted_display_string(perInBoth)}%
             </p>
         % endif
     % endif
@@ -70,16 +76,19 @@
     ## report on all the variables that were compared and give a basic stat idea of how well they did
     <blockquote>
         <p>
-            % for variableName, info in sorted(list(comparedVariables.items())) :
+            % for variableName in sorted(list(variables)) :
+                <% tempVariableInfo = variables[variableName] %>
                 Variable: <a href="${variableName | u}.html">${variableName}</a> <br>
-                Epsilon used: ${info[1]} <br>
-                Finite values within one epsilon of difference: ${report.make_formatted_display_string(info[0])}% <br>
+                Epsilon used: ${tempVariableInfo['epsilon']} <br>
+                Finite values within one epsilon of difference:
+                ${report.make_formatted_display_string(tempVariableInfo['passEpsilonPercent'])}% <br>
                 <br>
             % endfor
         </p>
     </blockquote>
     
     ## report the names of variables shared between the two files
+    <% sharedVars = varNames['sharedVars'] %>
     % if len(sharedVars) > 0 :
         <h3>Shared Variables</h3>
         <p>
@@ -93,6 +102,10 @@
     % endif
     
     ## check to see if there are any unique variables we need to report
+    <%
+        uniqueToAVars = varNames['uniqueToAVars']
+        uniqueToBVars = varNames['uniqueToBVars']
+    %>
     % if (len(uniqueToAVars) > 0) or (len(uniqueToBVars) > 0) :
         <h3>Unique Variables</h3>
         

pyglance/glance/report.py

@@ -59,40 +59,66 @@ def generate_and_save_summary_report(fileAName, fileBName,
                                      lastModifiedTimeA, lastModifiedTimeB,
                                      currentTime, currentUser,
                                      currentMachine,
-                                     numSpaciallyInvalidOnlyInA=None,
-                                     numSpaciallyInvalidOnlyInB=None,
-                                     percentSpaciallyInvalidPtsInA=None,
-                                     percentSpaciallyInvalidPtsInB=None,
-                                     percentSpaciallyInvalidPtsInBoth=None,
+                                     numSpacInvOnlyInA=None,
+                                     numSpacInvOnlyInB=None,
+                                     perSpacInvPtsInA=None,
+                                     perSpacInvPtsInB=None,
+                                     perSpacInvPtsInBoth=None,
                                      uniqueToAVars=[], uniqueToBVars=[], sharedVars=[]) :
     """
     given two files, and information about them, save a summary of their comparison
     The summary report, in html format will be saved to the given outputPath/outputFile
-    Variables should be a dictionary containing the name of each compared variable and the
+    
+    Variables should be a dictionary keyed on the name of each compared variable and containing the
     % of data values that were judged to be "similar enough" between file A and B (according
-    to the epsilon originally inputed for the comparison)
+    to the epsilon originally inputed for the comparison) and the epsilon used for the comparison
+        variables[var_name] = {"passEpsilonPercent": percent "similar enough" according to epsilon, "epsilon": epsilon)
+    more keys can be added in the future if more detailed data reporting is desired on the main report page
     """
-    kwargs = {'fileAName': fileAName,
-              'fileBName': fileBName,
-              'aMD5SUM': aMD5SUM,
-              'bMD5SUM': bMD5SUM,
-              'longitudeName': longitudeName,
-              'latitudeName': latitudeName,
-              'numSpacInvInA': numSpaciallyInvalidOnlyInA,
-              'numSpacInvInB': numSpaciallyInvalidOnlyInB,
-              'perSpacInvPtsInA': percentSpaciallyInvalidPtsInA,
-              'perSpacInvPtsInB': percentSpaciallyInvalidPtsInB,
-              'perSpacInvPtsInBoth': percentSpaciallyInvalidPtsInBoth,
-              'uniqueToAVars': uniqueToAVars,
-              'uniqueToBVars': uniqueToBVars,
-              'sharedVars': sharedVars,
-              'comparedVariables': variables,
-              'lastModifiedTimeA': lastModifiedTimeA,
-              'lastModifiedTimeB': lastModifiedTimeB,
-              'currentTime': currentTime,
-              'currentUser': currentUser,
-              'currentMachine': currentMachine
-              }
+    
+    # pack up all the data needed to build the summary report
+    
+    # information about the run in general
+    runInfo = {'machine': currentMachine,
+               'user': currentUser,
+               'time': currentTime,
+               'latitude': latitudeName,
+               'longitude': longitudeName
+               }
+    # information about the files that were compared
+    files = {'file A': {'path': fileAName,
+                        #'displayName': fileADisplayName, # TODO
+                        'lastModifiedTime': lastModifiedTimeA,
+                        'md5sum': aMD5SUM
+                        },
+             'file B': {'path': fileBName,
+                        #'displayName': fileADisplayName, # TODO
+                        'lastModifiedTime': lastModifiedTimeB,
+                        'md5sum': bMD5SUM
+                        }
+                }
+    # information about the spatial validity of the data
+    spatial = {'file A': {'numInvPts': numSpacInvOnlyInA,
+                          'perInvPts': perSpacInvPtsInA
+                          },
+               'file B': {'numInvPts': numSpacInvOnlyInB,
+                          'perInvPts': perSpacInvPtsInB
+                          },
+               'perInvPtsInBoth': perSpacInvPtsInBoth
+        }
+    # information about the variables present in the files
+    varNames = {'uniqueToAVars': uniqueToAVars,
+                'uniqueToBVars': uniqueToBVars,
+                'sharedVars': sharedVars
+                }
+    # build the full kwargs with all the info
+    kwargs = { 'runInfo': runInfo,
+               'files': files,
+               'spatial': spatial,
+               'varNames': varNames,
+               'variables': variables 
+               }
+              
     _make_and_save_page((outputPath + "/" + reportFileName), './mainreport.txt', **kwargs)
     
     return
@@ -110,9 +136,10 @@ def generate_and_save_doc_page(definitions, outputPath) :
 def generate_and_save_variable_report(fileAName, fileBName,
                                       aMD5SUM, bMD5SUM,
                                       variableName,
+                                      latitudeName, longitudeName,
                                       outputPath, reportFileName,
                                       epsilon, missingDataValue,
-                                      statsData, shouldIncludeImages,
+                                      statGroups, shouldIncludeImages,
                                       lastModifiedTimeA, lastModifiedTimeB,
                                       currentTime, currentUser,
                                       currentMachine) :
@@ -120,7 +147,42 @@ def generate_and_save_variable_report(fileAName, fileBName,
     given two files and information about the comparison of one of their variables,
     generate an html report about that variable and store it the outputPath/reportFileName
     provided
+    statGroups is a dictionary in the form
+         statGroups['stat group display name'] = {a dictionary of stats/values to show}
+    ie. there may be many different groups of stats that should each be displayed
     """
+    
+    # pack up all the data for a report on a particular variable
+    
+    # information about the run in general
+    runInfo = {'machine': currentMachine,
+               'user': currentUser,
+               'time': currentTime,
+               'latitude': latitudeName,
+               'longitude': longitudeName,
+               'variableName': variableName,
+               'epsilon': epsilon,
+               'missingDataValue': missingDataValue,
+               'shouldIncludeImages': shouldIncludeImages
+               }
+    # information about the files that were compared
+    files = {'file A': {'path': fileAName,
+                        #'displayName': fileADisplayName, # TODO
+                        'lastModifiedTime': lastModifiedTimeA,
+                        'md5sum': aMD5SUM
+                        },
+             'file B': {'path': fileBName,
+                        #'displayName': fileADisplayName, # TODO
+                        'lastModifiedTime': lastModifiedTimeB,
+                        'md5sum': bMD5SUM
+                        }
+                }
+    # put all the info together in the kwargs
+    kwargs = { 'runInfo': runInfo,
+               'files' : files,
+               'statGroups': statGroups 
+               }
+    '''
     kwargs = {'fileAName': fileAName,
               'fileBName': fileBName,
               'aMD5SUM' : aMD5SUM,
@@ -136,6 +198,8 @@ def generate_and_save_variable_report(fileAName, fileBName,
               'currentUser': currentUser,
               'currentMachine': currentMachine
               }
+              '''
+              
     _make_and_save_page((outputPath + "/" + reportFileName), './variablereport.txt', **kwargs)
     
     return

pyglance/glance/variablereport.txt

+<%doc>
+This Mako template is intended to create a variable specific report page for glance reportGen.
+
+Created by Eva Schiffer Jun 2009.
+Copyright (c) 2009 University of Wisconsin SSEC. All rights reserved.
+</%doc>
+
 <%!
     import types as types
     import glance.report as report
 %>
 
+## we will use this a lot, so hang on to it
+<% variableName = runInfo['variableName'] %>
 
 <title>${variableName} Variable Comparison</title>
 </head>
@@ -11,20 +20,24 @@
     <h1>${variableName} Variable Comparison</h1>
     
     <p>
-        comparison produced ${currentTime} by user ${currentUser} on ${currentMachine}
+        comparison produced ${runInfo['time']} by user ${runInfo['user']} on ${runInfo['machine']}
     </p>
     
+    ## show information on each of the files we're comparing (should only be two)
+    % for fileKey in sorted(list(files)) :
+        <% tempFileInfo = files[fileKey] %>
+        <p>
+            ${fileKey}: ${tempFileInfo['path']} <br>
+            md5sum for ${fileKey}: ${tempFileInfo['md5sum']} <br>
+            last modified: ${tempFileInfo['lastModifiedTime']}
+        </p>
+    % endfor
     <p>
-        file A: ${fileAName} <br>
-        md5sum for file A: ${aMD5SUM} <br>
-        last modified: ${lastModifiedTimeA}
-    </p>
-    <p>
-        file B: ${fileBName} <br>
-        md5sum for file B: ${bMD5SUM} <br>
-        last modified: ${lastModifiedTimeB}
+        latitude: ${runInfo['latitude']} <br>
+        longitude: ${runInfo['longitude']}
     </p>
     
+    <% shouldIncludeImages = runInfo['shouldIncludeImages'] %>
     % if shouldIncludeImages :
         <h3>Original Data</h3>
         <p>
@@ -38,8 +51,9 @@
     ## comparison parameters
     <p>
         variable name: ${variableName} <br>
-        epsilon value: ${epsilon} <br>
+        epsilon value: ${runInfo['epsilon']} <br>
         "missing" data value:
+        <% missingDataValue = runInfo['missingDataValue'] %>
         % if missingDataValue is None :
             None
         % else :
@@ -48,13 +62,22 @@
     </p>
     
     <h3>Statistical Summary</h3>
-    <p>
-        <blockquote>
-            % for statName, statValue in sorted(list(statsData.items())) :
-                ${statName}<a href="./doc.html">*</a>: ${report.make_formatted_display_string(statValue)} <br>
-            % endfor
-        </blockquote>
-    </p>
+    
+    ## list out all of the statistics groups we have
+    <dl>
+    % for setName in sorted(list(statGroups)) :
+        <% dataSet = statGroups[setName] %>
+            <dt>
+                ${setName}
+            </dt>
+            <dd>
+                % for statName, statValue in sorted(list(dataSet.items())) :
+                    ${statName}<a href="./doc.html">*</a>: ${report.make_formatted_display_string(statValue)} <br>
+                % endfor
+                <br>
+            <dd>
+    % endfor
+    </dl>
     
     % if shouldIncludeImages :
     <p>