Create netcdf by atomic move and fix frame merge

main.py

@@ -7,7 +7,8 @@ import pandas as pd
 import numpy as np
 from zipfile import ZipFile
 from io import BytesIO
-
+from tempfile import mkstemp
+from shutil import move
 
 from electronic_checks import electronic_checks
 from global_checks import global_checks
@@ -36,7 +37,8 @@ def save_quality(frame, qc_path):
     frame = frame.ix[pd.notnull(frame.sum_index)].set_index('sum_index').sort_index()
 
     # Define the netcdf
-    ncdf = netCDF4.Dataset(qc_path, 'w')
+    *_, temp = mkstemp()
+    ncdf = netCDF4.Dataset(temp, 'w')
     time = ncdf.createDimension('time', len(frame))
     base_time = ncdf.createVariable('base_time', 'i8', ())
     time_offset = ncdf.createVariable('time_offset', 'i8', ('time',))
@@ -59,6 +61,8 @@ def save_quality(frame, qc_path):
     qc_percent[:] = frame['qc_percent'].values
     qc_notes[:] = frame['qc_notes'].fillna('').values
     ncdf.close()
+    move(temp,qc_path)
+    
     
 
 def read_frame(cxs_file, sum_file):
@@ -145,7 +149,7 @@ def update_all(ftp_dir, sci_dir, parameters=None):
             parameters = {}
         # check for spikes in interferograms and add that quality column to housekeeping frame
         # merging by datetimes is not expected to work, will probably interleave with SUM records
-        frame_with_spikes = frame.merge(spike_check(igms, parameters), on='datetime', how='outer', sort=True)
+        frame_with_spikes = frame.merge(spike_check(igms, parameters), on='datetime', how='outer', suffixes=('','_igm'), sort=True)
         # Propogate spike data to surrounding records
         # Only propogate presence of spikes, not abscence
         frame_with_spikes.ix[frame_with_spikes.spike_check == False] = pd.np.nan