From 99ac579c540fb4630704f8881b39513ba67b8c7c Mon Sep 17 00:00:00 2001
From: Coda Phillips <cphillips@sean.ssec.wisc.edu>
Date: Fri, 1 Jul 2016 10:52:24 -0500
Subject: [PATCH] Write to netcdf files
---
main.py | 32 +++++++++++++++++++++++++++-----
1 file changed, 27 insertions(+), 5 deletions(-)
diff --git a/main.py b/main.py
index e6df9e0..6a35ca0 100644
--- a/main.py
+++ b/main.py
@@ -1,7 +1,8 @@
import os
from glob import glob
import re
-#from aeri_tools.io.dmv.housekeeping import get_all_housekeeping
+import netCDF4
+from aeri_tools.io.dmv.housekeeping import get_all_housekeeping
import electronic_checks
import global_checks
@@ -19,9 +20,28 @@ levels = [
thermal_checks.CheckList()
]
+def save_quality(frame, qc_path):
+ frame[['qc_percent','qc_notes','datetime']]
+ ncdf = netCDF4.Dataset(qc_path, 'w')
+ time = ncdf.createDimension('time', len(frame))
+ base_time = ncdf.createVariable('base_time', 'i8', ())
+ time_offset = ncdf.createVariable('time_offset', 'f8', ('time',))
+ qc_percent = ncdf.createVariable('qc_percent', 'f4', ('time',))
+ qc_notes = ncdf.createVariable('qc_notes', str, ('time',))
+ for check_mask in frame.filter(like='check'):
+ ncdf.createVariable(check_mask, 'f4', ('time',))[:] = frame[check_mask].values
+ base_time[:] = frame.datetime[0].timestamp()
+ time_offset[:] = (frame.datetime - frame.datetime[0]).values / 1e9
+ qc_percent[:] = frame['qc_percent'].values
+ qc_notes[:] = frame['qc_notes'].fillna('').values
+ ncdf.close()
+
+
def read_frame(cxs_file, sum_file):
- return get_all_housekeeping(cxs_file).combine_first(
+ hk = get_all_housekeeping(cxs_file).combine_first(
get_all_housekeeping(sum_file))
+ hk.index.name = 'datetime'
+ return hk.reset_index()
def check_frame(frame, parameters):
frame['qc_percent'] = 0
@@ -34,10 +54,12 @@ def check_frame(frame, parameters):
def update_all(ftp_dir, parameters=None):
cxs_files = glob(os.path.join(ftp_dir,'AE*','*B1.CXS'))
for qc_file, cxs_file, sum_file in files_to_update(cxs_files):
- hk = read_frame(cxs_file, sum_file)
+ print('Performing quality control for {}'.format(cxs_file))
+ frame = read_frame(cxs_file, sum_file)
if parameters is None:
parameters = {}
- check_frame(frame, parameters)
+ frame = check_frame(frame, parameters)
+ save_quality(frame, qc_file)
def files_to_update(cxs_files):
for cxs_file in cxs_files:
@@ -52,7 +74,7 @@ def files_to_update(cxs_files):
if max(os.path.getmtime(sum_file), os.path.getmtime(cxs_file)) > os.path.getmtime(qc_file):
yield (qc_file, cxs_file, sum_file)
else:
- yield (qc_file, cxs_file, sum_file)
+ yield (possible_qc, cxs_file, sum_file)
if __name__ == '__main__':
import argparse
--
GitLab