diff --git a/main.py b/main.py index 6c719b6dd3f1d26fef2eb20b304446e121865269..f2aec36f5b0a82c6ea2533974473f06fa5d2c455 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,7 @@ import os from glob import glob import re +from collections import defaultdict import netCDF4 from aeri_tools.io.dmv.housekeeping import get_all_housekeeping import pandas as pd @@ -39,7 +40,17 @@ def save_quality(frame, qc_path): ncdf.close() move(temp,qc_path) - + +def compute_calibration_graph(sceneMirrorPosition): + affected_records = defaultdict(set) + scene_regex = '(?<=(HA|AH))([^HA]+)(?=(HA|AH))' + for scene in re.finditer(scene_regex, ''.join(sceneMirrorPosition.fillna(ord('?')).apply(int).apply(chr))): + blackbody_indices = set(range(*scene.span(1))) + blackbody_indices.update(range(*scene.span(3))) + sky_view_indices = range(*scene.span(2)) + for blackbody_index in blackbody_indices: + affected_records[blackbody_index].update(sky_view_indices) + return affected_records def read_frame(cxs_file, sum_file): """ @@ -62,7 +73,9 @@ def read_frame(cxs_file, sum_file): # Combine extra data from SUM into CXS, many columns will have during calibration views hk = cxs.combine_first(sum_) hk.index.name = 'datetime' - return hk.reset_index() + hk = hk.reset_index() + hk.calibration_graph = compute_calibration_graph(hk.sceneMirrorPosition) + return hk def read_igms(spc_zip_path): """ @@ -128,6 +141,7 @@ def update_all(ftp_dir, sci_dir, checklist, parameters=None): frame_with_spikes['spike_check'] = frame_with_spikes.spike_check.ffill(limit=1).bfill(limit=1) # Reindex back to housekeeping frame (union of sum and cxs records), removing interleaved spike data frame_with_spikes = frame_with_spikes.ix[frame.index] + frame_with_spikes.calibration_graph = frame.calibration_graph # Perform qc on housekeeping frame frame_with_spikes = check_frame(frame_with_spikes, parameters, checklist)