Fix NetCDF generation not including valid 0 for QC variables

README.rst

@@ -133,14 +133,14 @@ Install averaging tasks to create average fields in the metobs_realtime bucket:
 
 .. code-block:: bash
 
-    python -m metobscommon.influxdb create_tasks --token <READ_WRITE_TOKEN>
+    python -m metobscommon.influxdb --influxdb-token <READ_WRITE_TOKEN> create_tasks
 
 Backfill InfluxDB Database
 --------------------------
 
 Insert data from an old tower file:
 
-    python -m aosstower.level_00.influxdb -vvv --bulk 5000 --influxdb-token <READ_WRITE_TOKEN> /data1/raw/aoss/tower/2018/05/08/aoss_tower.2018-05-08.ascii
+    python -m aosstower.level_00.influxdb --influxdb-token <READ_WRITE_TOKEN> -vvv --bulk 5000 /data1/raw/aoss/tower/2018/05/08/aoss_tower.2018-05-08.ascii
 
 The above command sends data in blocks of 5000 records. This is to improve
 performance of sending data to the InfluxDB instead of sending one record
@@ -148,7 +148,7 @@ at a time. A bulk value of 5000-10000 is preferred.
 
 Compute the averages for 5 second tower and data:
 
-    python -m metobscommon.influxdb -vvv run_manual_average --stations aoss.tower -s 2018-05-07T00:00:00 -e 2018-05-08T22:00:00 -d 1m 5m 1h --influxdb-token <READ_WRITE_TOKEN>
+    python -m metobscommon.influxdb --influxdb-token <READ_WRITE_TOKEN> -vvv run_manual_average --stations aoss.tower -s 2018-05-07T00:00:00 -e 2018-05-08T22:00:00 -d 1m 5m 1h
 
 Note the above computes the 1m, 5m, and 1h averages. The time range (-s/-e)
 must be at whole intervals for the average intervals specified otherwise

metobscommon/util/nc.py

@@ -9,6 +9,7 @@ This is used by instrument packages for generating Level B1 NetCDF4 files.
 import logging
 import numpy as np
 import pandas as pd
+from pandas.tseries.frequencies import to_offset
 from metobscommon.util import calc
 
 LOG = logging.getLogger(__name__)
@@ -188,13 +189,15 @@ def create_variables(nc_file, first_stamp, database, chunk_sizes=None, zlib=Fals
                                             dimensions=dims, zlib=zlib, chunksizes=these_chunks)
 
         qcVariable.long_name = 'data quality'
-        qcVariable.valid_range = np.byte(0b1), np.byte(0b1111)
-        qcVariable.flag_masks = np.byte(0b1), np.byte(0b10), np.byte(0b100), np.byte(0b1000)
+        qcVariable.valid_range = np.byte(0b0), np.byte(0b1111)
+        qcVariable.flag_masks = np.byte(0b0), np.byte(0b1), np.byte(0b10), np.byte(0b100), np.byte(0b1000)
 
-        flagMeaning = ['value is equal to missing_value',
-                       'value is less than the valid min',
-                       'value is greater than the valid max',
-                       'difference between current and previous values exceeds valid_delta.']
+        flagMeaning = [
+            'value is valid',
+            'value is equal to missing_value',
+            'value is less than the valid min',
+            'value is greater than the valid max',
+            'difference between current and previous values exceeds valid_delta.']
 
         qcVariable.flag_meaning = ', '.join(flagMeaning)
 
@@ -250,11 +253,13 @@ def minute_averages(frame):
     frame['wind_east'], frame['wind_north'], _ = calc.wind_vector_components(frame['wind_speed'], frame['wind_dir'])
     # round up each 1 minute group so data at time T is the average of data
     # from T - 1 (exclusive) to T (inclusive).
-    new_frame = frame.resample('1T', closed='right', loffset='1T').mean()
+    new_frame = frame.resample('1T', closed='right').mean()
+    new_frame.index = new_frame.index + to_offset("1T")
 
     # 2 minute rolling average of 5 second data (5 seconds * 24 = 120 seconds = 2 minutes)
     winds_frame_5s = frame[['wind_speed', 'wind_east', 'wind_north']]
-    winds_frame_5s = winds_frame_5s.resample('5S', closed='right', loffset='5S').mean()
+    winds_frame_5s = winds_frame_5s.resample('5S', closed='right').mean()
+    winds_frame_5s.index = winds_frame_5s.index + to_offset('5S')
     winds_frame_2m = winds_frame_5s.rolling(24, win_type='boxcar').mean()
 
     # rolling average is used for 1 minute output
@@ -265,7 +270,9 @@ def minute_averages(frame):
 
     gust = calculate_wind_gust(winds_frame_5s['wind_speed'], winds_frame_2m['wind_speed'])
     # "average" the gusts to minute resolution to match the rest of the data
-    new_frame['gust'] = gust.resample('1T', closed='right', loffset='1T').max()
+    new_gust = gust.resample('1T', closed='right').max()
+    new_gust.index = new_gust.index + to_offset('1T')
+    new_frame['gust'] = new_gust
 
     return new_frame.fillna(np.nan)