diff --git a/README.rst b/README.rst index f7c10fa124dbb77159ff58216492abd5369f9faf..38cad5701309135b45bb9e349aa3b266e91917ff 100644 --- a/README.rst +++ b/README.rst @@ -133,14 +133,14 @@ Install averaging tasks to create average fields in the metobs_realtime bucket: .. code-block:: bash - python -m metobscommon.influxdb create_tasks --token <READ_WRITE_TOKEN> + python -m metobscommon.influxdb --influxdb-token <READ_WRITE_TOKEN> create_tasks Backfill InfluxDB Database -------------------------- Insert data from an old tower file: - python -m aosstower.level_00.influxdb -vvv --bulk 5000 --influxdb-token <READ_WRITE_TOKEN> /data1/raw/aoss/tower/2018/05/08/aoss_tower.2018-05-08.ascii + python -m aosstower.level_00.influxdb --influxdb-token <READ_WRITE_TOKEN> -vvv --bulk 5000 /data1/raw/aoss/tower/2018/05/08/aoss_tower.2018-05-08.ascii The above command sends data in blocks of 5000 records. This is to improve performance of sending data to the InfluxDB instead of sending one record @@ -148,7 +148,7 @@ at a time. A bulk value of 5000-10000 is preferred. Compute the averages for 5 second tower and data: - python -m metobscommon.influxdb -vvv run_manual_average --stations aoss.tower -s 2018-05-07T00:00:00 -e 2018-05-08T22:00:00 -d 1m 5m 1h --influxdb-token <READ_WRITE_TOKEN> + python -m metobscommon.influxdb --influxdb-token <READ_WRITE_TOKEN> -vvv run_manual_average --stations aoss.tower -s 2018-05-07T00:00:00 -e 2018-05-08T22:00:00 -d 1m 5m 1h Note the above computes the 1m, 5m, and 1h averages. The time range (-s/-e) must be at whole intervals for the average intervals specified otherwise diff --git a/metobscommon/util/nc.py b/metobscommon/util/nc.py index 8e6bda6fcf1ad9fd47980f74391490c961222381..dbdf07f8f60b1d4de3b1104bb18698d72ec791ec 100644 --- a/metobscommon/util/nc.py +++ b/metobscommon/util/nc.py @@ -9,6 +9,7 @@ This is used by instrument packages for generating Level B1 NetCDF4 files. import logging import numpy as np import pandas as pd +from pandas.tseries.frequencies import to_offset from metobscommon.util import calc LOG = logging.getLogger(__name__) @@ -188,13 +189,15 @@ def create_variables(nc_file, first_stamp, database, chunk_sizes=None, zlib=Fals dimensions=dims, zlib=zlib, chunksizes=these_chunks) qcVariable.long_name = 'data quality' - qcVariable.valid_range = np.byte(0b1), np.byte(0b1111) - qcVariable.flag_masks = np.byte(0b1), np.byte(0b10), np.byte(0b100), np.byte(0b1000) + qcVariable.valid_range = np.byte(0b0), np.byte(0b1111) + qcVariable.flag_masks = np.byte(0b0), np.byte(0b1), np.byte(0b10), np.byte(0b100), np.byte(0b1000) - flagMeaning = ['value is equal to missing_value', - 'value is less than the valid min', - 'value is greater than the valid max', - 'difference between current and previous values exceeds valid_delta.'] + flagMeaning = [ + 'value is valid', + 'value is equal to missing_value', + 'value is less than the valid min', + 'value is greater than the valid max', + 'difference between current and previous values exceeds valid_delta.'] qcVariable.flag_meaning = ', '.join(flagMeaning) @@ -250,11 +253,13 @@ def minute_averages(frame): frame['wind_east'], frame['wind_north'], _ = calc.wind_vector_components(frame['wind_speed'], frame['wind_dir']) # round up each 1 minute group so data at time T is the average of data # from T - 1 (exclusive) to T (inclusive). - new_frame = frame.resample('1T', closed='right', loffset='1T').mean() + new_frame = frame.resample('1T', closed='right').mean() + new_frame.index = new_frame.index + to_offset("1T") # 2 minute rolling average of 5 second data (5 seconds * 24 = 120 seconds = 2 minutes) winds_frame_5s = frame[['wind_speed', 'wind_east', 'wind_north']] - winds_frame_5s = winds_frame_5s.resample('5S', closed='right', loffset='5S').mean() + winds_frame_5s = winds_frame_5s.resample('5S', closed='right').mean() + winds_frame_5s.index = winds_frame_5s.index + to_offset('5S') winds_frame_2m = winds_frame_5s.rolling(24, win_type='boxcar').mean() # rolling average is used for 1 minute output @@ -265,7 +270,9 @@ def minute_averages(frame): gust = calculate_wind_gust(winds_frame_5s['wind_speed'], winds_frame_2m['wind_speed']) # "average" the gusts to minute resolution to match the rest of the data - new_frame['gust'] = gust.resample('1T', closed='right', loffset='1T').max() + new_gust = gust.resample('1T', closed='right').max() + new_gust.index = new_gust.index + to_offset('1T') + new_frame['gust'] = new_gust return new_frame.fillna(np.nan)