Fix indexing on invalidate

scene_checks.py

@@ -23,8 +23,11 @@ def safing_check(frame, parameters):
         return frame
 
     hatch_closing = ((frame.hatchOpen == 1)  & ((frame.hatchOpen == -3).diff(-1) == 1)).shift(1)
-    frame['safing_check'] = (hatch_closing & frame.sceneMirrorPosition.isin([ord('H'), ord('A')])) * 1
+    mirror_safing = (hatch_closing & frame.sceneMirrorPosition.isin([ord('H'), ord('A')]))
+    frame['safing_check'] = mirror_safing * 1
+    frame.ix[mirror_safing, 'qc_notes'] = 'mirror likely safed during view'
     frame = invalidate_records(frame, 'safing_check')
+    
     return frame
 
 

test.py

@@ -13,22 +13,22 @@ def test_it_works():
 
 def test_invalidate_record():
     invalidated = invalidate_record(three_scenes.copy(), 2, 'test', .5)
-    assert invalidated.iloc[2].test == .5
-    assert pd.np.isnan(invalidated.iloc[3].test)
-    assert pd.np.isnan(invalidated.iloc[1].test)
+    assert invalidated.loc[2].test == .5
+    assert pd.np.isnan(invalidated.loc[3].test)
+    assert pd.np.isnan(invalidated.loc[1].test)
 
     tricky_index = three_scenes.copy()
     tricky_index.index = tricky_index.index - 1
-    invalidated = invalidate_record(tricky_index, 2, 'test', .5)
-    assert invalidated.iloc[2].test == .5, invalidated.test
-    assert pd.np.isnan(invalidated.iloc[3].test)
-    assert pd.np.isnan(invalidated.iloc[1].test)
+    invalidated = invalidate_record(tricky_index, 1, 'test', .5)
+    assert invalidated.loc[1].test == .5, invalidated.test
+    assert pd.np.isnan(invalidated.loc[2].test), invalidated
+    assert pd.np.isnan(invalidated.loc[0].test), invalidated
 
     corrupt_cal = three_scenes.copy()
     invalidated = invalidate_record(corrupt_cal, 3, 'test', .5)
-    assert invalidated.iloc[2].test == .5, invalidated.test
-    assert invalidated.iloc[4].test != .5, invalidated.test
-    assert invalidated.iloc[5].test == .5, invalidated.test
+    assert invalidated.loc[2].test == .5, invalidated.test
+    assert invalidated.loc[4].test != .5, invalidated.test
+    assert invalidated.loc[5].test == .5, invalidated.test
 
 
 def test_invalidate_records():
@@ -37,30 +37,30 @@ def test_invalidate_records():
     corrupt_cal['test'][3] = 1
     
     invalidated = invalidate_records(corrupt_cal, 'test')
-    assert invalidated.iloc[2].test == 1, invalidated.test
-    assert invalidated.iloc[4].test == 0, invalidated.test
-    assert invalidated.iloc[5].test == 1, invalidated.test
+    assert invalidated.loc[2].test == 1, invalidated.test
+    assert invalidated.loc[4].test == 0, invalidated.test
+    assert invalidated.loc[5].test == 1, invalidated.test
 
 def test_invalidate_record_and_annotate():
     invalidated = invalidate_record(three_scenes.copy(), 2, 'test', .5, 'test annotation')
-    assert invalidated.iloc[2].test == .5
-    assert pd.np.isnan(invalidated.iloc[3].test)
-    assert pd.np.isnan(invalidated.iloc[1].test)
-    assert invalidated.iloc[2].qc_notes == 'test annotation'
+    assert invalidated.loc[2].test == .5
+    assert pd.np.isnan(invalidated.loc[3].test)
+    assert pd.np.isnan(invalidated.loc[1].test)
+    assert invalidated.loc[2].qc_notes == 'test annotation'
 
     tricky_index = three_scenes.copy()
     tricky_index.index = tricky_index.index - 1
-    invalidated = invalidate_record(tricky_index, 2, 'test', .5, 'test annotation')
-    assert invalidated.iloc[2].test == .5, invalidated.test
-    assert pd.np.isnan(invalidated.iloc[3].test)
-    assert pd.np.isnan(invalidated.iloc[1].test)
-    assert invalidated.iloc[2].qc_notes == 'test annotation'
+    invalidated = invalidate_record(tricky_index, 1, 'test', .5, 'test annotation')
+    assert invalidated.loc[1].test == .5, invalidated.test
+    assert pd.np.isnan(invalidated.loc[2].test)
+    assert pd.np.isnan(invalidated.loc[0].test)
+    assert invalidated.loc[1].qc_notes == 'test annotation'
 
     corrupt_cal = three_scenes.copy()
     invalidated = invalidate_record(corrupt_cal, 3, 'test', .5)
-    assert invalidated.iloc[2].test == .5, invalidated.test
-    assert invalidated.iloc[2].qc_notes == 'invalid calibration:3'
-    assert invalidated.iloc[5].test == .5, invalidated.test
-    assert invalidated.iloc[5].qc_notes == 'invalid calibration:3'
+    assert invalidated.loc[2].test == .5, invalidated.test
+    assert invalidated.loc[2].qc_notes == 'invalid calibration:3'
+    assert invalidated.loc[5].test == .5, invalidated.test
+    assert invalidated.loc[5].qc_notes == 'invalid calibration:3'
 
 three_scenes = pd.DataFrame({'sceneMirrorPosition': list(map(ord, 'HASAHSAHSAH')), 'qc_notes':''})

util.py

@@ -7,43 +7,47 @@ def invalidate_records(frame, check_name):
         invalidate_record(frame, index, check_name, percent)
     return frame
 
-def invalidate_record(frame, index, check_name, value, annotation=''):
-    frame.ix[frame.index[index], check_name] = value
+def invalidate_record(frame, loc, check_name, value, annotation=''):
+    frame.loc[loc, check_name] = value
     if annotation:
-        if frame.ix[frame.index[index], 'qc_notes']:
-            frame.ix[frame.index[index], 'qc_notes'] = ','.join([frame.ix[frame.index[index],'qc_notes'], annotation])
+        if 'qc_notes' not in frame:
+            frame['qc_notes'] = None
+        if type(frame.loc[loc, 'qc_notes']) == str and len(frame.loc[loc, 'qc_notes']) > 0:
+            frame.loc[loc, 'qc_notes'] = ','.join([frame.loc[loc,'qc_notes'], annotation])
         else:
-            frame.ix[frame.index[index], 'qc_notes'] = annotation
+            frame.loc[loc, 'qc_notes'] = annotation
 
-    corrupt_view = frame.iloc[index].sceneMirrorPosition
+    corrupt_view = frame.loc[loc,'sceneMirrorPosition']
     if corrupt_view in [ord('H'),ord('A')]:
     
         def invalidate_neighbor(neighbor):
-            if frame.sceneMirrorPosition.iloc[neighbor] == corrupt_view:
+            if frame.sceneMirrorPosition.loc[neighbor] == corrupt_view:
                 # Made one cycle, break
                 return True
-            elif frame.sceneMirrorPosition.iloc[neighbor] in [ord('H'), ord('A')]:
+            elif frame.sceneMirrorPosition.loc[neighbor] in [ord('H'), ord('A')]:
                 # Skip opposite calibration views
                 return
             else:
                 # Invalidate non-calibration views
-                frame.ix[frame.index[neighbor], check_name] = value
-                previous_notes = frame.ix[frame.index[neighbor], 'qc_notes']
-                if previous_notes:
-                    frame.ix[frame.index[neighbor], 'qc_notes'] = ','.join([previous_notes, 'invalid calibration:{:d}'.format(index)])
+                frame.loc[neighbor,check_name] = value
+                previous_notes = frame.loc[neighbor,'qc_notes']
+                if type(previous_notes) == str and len(previous_notes) > 0:
+                    frame.loc[neighbor,'qc_notes'] = ','.join([previous_notes, 'invalid calibration:{:d}'.format(loc)])
                 else:
-                    frame.ix[frame.index[neighbor], 'qc_notes'] = 'invalid calibration:{:d}'.format(index)
+                    frame.loc[neighbor,'qc_notes'] = 'invalid calibration:{:d}'.format(loc)
 
         # Corrupt calibration view, must also invalidate neighboring scenes
-        _idx = index + 1
+        # _idx is the iloc
+        _idx = frame.index.tolist().index(loc) + 1
         while _idx < len(frame):
-            if invalidate_neighbor(_idx):
+            if invalidate_neighbor(frame.index[_idx]):
                 break
             _idx += 1
 
-        _idx = index - 1
+        # _idx is the iloc
+        _idx = frame.index.tolist().index(loc) - 1
         while _idx >= 0:
-            if invalidate_neighbor(_idx):
+            if invalidate_neighbor(frame.index[_idx]):
                 break
             _idx -= 1
 
@@ -72,5 +76,7 @@ class BaseCheckList:
         # Filter bad records from previous level
         filtered_frame = frame.ix[frame.qc_percent < 1].copy()
         for check in self.checks:
+            original_shape = filtered_frame.shape
             filtered_frame = check(filtered_frame, self.parameters)
+            assert filtered_frame.shape[0] == original_shape[0]
         return self.update_qc_percent(frame.combine_first(filtered_frame))