From c1f5d51eee1b18a509cd3c4ffa81072b86d10c1e Mon Sep 17 00:00:00 2001 From: Matthew Westphall <westphall@wisc.edu> Date: Wed, 8 Feb 2017 22:06:51 +0000 Subject: [PATCH] updated netCDF4 datatypes to match ARM standard --- aosstower/level_b1/avg_database.py | 838 ----------------------------- aosstower/level_b1/nc.py | 23 +- 2 files changed, 17 insertions(+), 844 deletions(-) delete mode 100644 aosstower/level_b1/avg_database.py diff --git a/aosstower/level_b1/avg_database.py b/aosstower/level_b1/avg_database.py deleted file mode 100644 index 43b78fc..0000000 --- a/aosstower/level_b1/avg_database.py +++ /dev/null @@ -1,838 +0,0 @@ -# CF Metadata 1.6 compliant structure for NetCDF file. This is essentially -# a pythonified version of the CDL -from collections import OrderedDict -import aosstower.schema as schema -BASIC_STATION = { - - 'globals': { - 'source': 'surface observation', - 'institution': 'UW SSEC', - 'featureType': 'timeSeries', - 'Conventions': 'ARM-1.2 CF-1.6', - 'data_level': 'b1', - 'datastream' : 'aoss.tower.nc-1mo-1d.lb1.v00', - 'software_version': '00' - }, - - 'dimensions': { - 'max_len_station_name': 32, - 'time': None # Unlimited - }, - - 'variables': OrderedDict({ - 'base_time': { - 'string': '', # Filled in by creator - 'standard_name': 'time', - 'long_name': 'base time as unix timestamp', - 'units': 'seconds since 1970-01-01 00:00:00 0:00', - '_type': 'd', - '_shape': tuple(), - }, - 'time_offset': { - 'long_name': 'time offset from base_time', - 'standard_name': 'time', - 'units': '', # Filled in by creator - '_type': 'd', - '_shape': ('time',), - }, - 'time': { - 'long_name': 'time offset from epoch', - 'standard_name': 'time', - 'units': 'seconds since 1970-01-01 00:00:00 0:00', - '_type': 'd', - '_shape': ('time',), - }, - 'station_name': { - 'cf_role': 'timeseries_id', - 'long_name': 'station name', - '_type': 'S1', - '_shape': ('max_len_station_name',), - }, - 'lat': { - 'standard_name': 'latitude', - 'units': 'degrees_north', - 'valid_min': -90, - 'valid_max': 90, - '_type': 'f', - '_shape': tuple(), - }, - 'lon': { - 'standard_name': 'longitude', - 'units': 'degrees_east', - 'valid_min': -180, - 'valid_max': 180, - '_type': 'f', - '_shape': tuple(), - }, - 'alt': { - 'long_name': 'vertical distance', - 'standard_name': 'height', - 'units': 'm', - 'positive': 'up', - 'axis': 'Z', - '_type': 'f', - '_shape': tuple(), - } - }) -} - -AOSS_VARS = OrderedDict({ - 'box_temp_high': { - 'standard_name': 'air_temperature', - 'units': 'degC', - 'description': 'Auxillary Temperature', - '_type': 'f', - '_shape': ('time',), - }, - - 'box_temp_low': { - 'standard_name': 'air_temperature', - 'units': 'degC', - 'description': 'Auxillary Temperature', - '_type': 'f', - '_shape': ('time',), - }, - - 'box_temp_mean': { - 'standard_name': 'air_temperature', - 'units': 'degC', - 'description': 'Auxillary Temperature', - '_type': 'f', - '_shape': ('time',), - }, - - 'box_presure_high': { - 'standard_name': 'air_pressure', - 'description': 'Pressure inside the data logger enclosure', - 'units': 'hpa', - 'valid_min': 850, - 'valid_max': '1100', - '_type': 'f', - '_shape': ('time',), - }, - - 'box_presure_low': { - 'standard_name': 'air_pressure', - 'description': 'Pressure inside the data logger enclosure', - 'units': 'hpa', - 'valid_min': 850, - 'valid_max': '1100', - '_type': 'f', - '_shape': ('time',), - }, - - 'box_presure_mean': { - 'standard_name': 'air_pressure', - 'description': 'Pressure inside the data logger enclosure', - 'units': 'hpa', - 'valid_min': 850, - 'valid_max': '1100', - '_type': 'f', - '_shape': ('time',), - }, - - 'paro_air_temp_period_high': { - 'standard_name': '', - 'description': '', - 'units': '1', - '_type': 'f', - '_shape': ('time',), - }, - - 'paro_air_temp_period_low': { - 'standard_name': '', - 'description': '', - 'units': '1', - '_type': 'f', - '_shape': ('time',), - }, - - 'paro_air_temp_period_mean': { - 'standard_name': '', - 'description': '', - 'units': '1', - '_type': 'f', - '_shape': ('time',), - }, - - 'paro_pressure_period_high': { - 'standard_name': '', - 'description': '', - 'units': '1', - '_type': 'f', - '_shape': ('time',), - }, - - 'paro_pressure_period_low': { - 'standard_name': '', - 'description': '', - 'units': '1', - '_type': 'f', - '_shape': ('time',), - }, - - 'paro_pressure_period_mean': { - 'standard_name': '', - 'description': '', - 'units': '1', - '_type': 'f', - '_shape': ('time',), - }, - - 'paro_air_temp_high': { - 'standard_name': 'air_temperature', - 'description': '', - 'units': 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'paro_air_temp_low': { - 'standard_name': 'air_temperature', - 'description': '', - 'units': 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'paro_air_temp_mean': { - 'standard_name': 'air_temperature', - 'description': '', - 'units': 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'pressure_high': { - 'standard_name': 'air_pressure', - 'description': 'Air pressure as measured from the PAROSCI pressure sensor', - 'units': 'hpa', - 'valid_min': 850, - 'valid_max': 1100, - '_type': 'f', - '_shape': ('time',), - }, - - 'pressure_low': { - 'standard_name': 'air_pressure', - 'description': 'Air pressure as measured from the PAROSCI pressure sensor', - 'units': 'hpa', - 'valid_min': 850, - 'valid_max': 1100, - '_type': 'f', - '_shape': ('time',), - }, - - 'pressure_mean': { - 'standard_name': 'air_pressure', - 'description': 'Air pressure as measured from the PAROSCI pressure sensor', - 'units': 'hpa', - 'valid_min': 850, - 'valid_max': 1100, - '_type': 'f', - '_shape': ('time',), - }, - - 'paro_cal_sig_high': { - 'standard_name': '', - 'description': '', - 'units': '', - '_type': 'f', - '_shape': ('time',), - }, - - 'paro_cal_sig_low': { - 'standard_name': '', - 'description': '', - 'units': '', - '_type': 'f', - '_shape': ('time',), - }, - - 'paro_cal_sig_mean': { - 'standard_name': '', - 'description': '', - 'units': '', - '_type': 'f', - '_shape': ('time',), - }, - - 'box_rh_high': { - 'standard_name': 'relative humidity', - 'description': 'Relative humidity inside the data logger enclosure', - 'units': '%', - 'valid_min': 0, - 'valid_max': 100, - '_type': 'f', - '_shape': ('time',), - }, - - 'box_rh_low': { - 'standard_name': 'relative humidity', - 'description': 'Relative humidity inside the data logger enclosure', - 'units': '%', - 'valid_min': 0, - 'valid_max': 100, - '_type': 'f', - '_shape': ('time',), - }, - - 'box_rh_mean': { - 'standard_name': 'relative humidity', - 'description': 'Relative humidity inside the data logger enclosure', - 'units': '%', - 'valid_min': 0, - 'valid_max': 100, - '_type': 'f', - '_shape': ('time',), - }, - - 'box_air_temp_high': { - 'standard_name': 'air_temperature', - 'description': 'Air temperature inside the data logger enclosure', - 'units': 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'box_air_temp_low': { - 'standard_name': 'air_temperature', - 'description': 'Air temperature inside the data logger enclosure', - 'units': 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'box_air_temp_mean': { - 'standard_name': 'air_temperature', - 'description': 'Air temperature inside the data logger enclosure', - 'units': 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'air_temp_2_high': { - 'standard_name': 'air_temperature', - 'description': 'Auxillary air temperature', - 'units': 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'air_temp_2_low': { - 'standard_name': 'air_temperature', - 'description': 'Auxillary air temperature', - 'units': 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'air_temp_2_mean': { - 'standard_name': 'air_temperature', - 'description': 'Auxillary air temperature', - 'units': 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'air_temp_3_high': { - 'standard_name': 'air_temperature', - 'description': 'Auxillary air temperature', - 'units' : 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'air_temp_3_low': { - 'standard_name': 'air_temperature', - 'description': 'Auxillary air temperature', - 'units' : 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'air_temp_3_mean': { - 'standard_name': 'air_temperature', - 'description': 'Auxillary air temperature', - 'units' : 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'air_temp_4_high': { - 'standard_name': 'air_temperature', - 'description' : 'Auxillary air temperature', - 'units' : 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'air_temp_4_low': { - 'standard_name': 'air_temperature', - 'description' : 'Auxillary air temperature', - 'units' : 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'air_temp_4_mean': { - 'standard_name': 'air_temperature', - 'description' : 'Auxillary air temperature', - 'units' : 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'air_temp_5_high': { - 'standard_name': 'air_temperature', - 'descripiton': 'Auxillary air temperature', - 'units': 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'air_temp_5_low': { - 'standard_name': 'air_temperature', - 'descripiton': 'Auxillary air temperature', - 'units': 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), -}, - - 'air_temp_5_mean': { - 'standard_name': 'air_temperature', - 'descripiton': 'Auxillary air temperature', - 'units': 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'wind_speed_high': { - 'standard_name': 'wind_speed', - 'description': 'wind_speed', - 'units': 'm*s^-1', - 'valid_min': 0, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'wind_speed_low': { - 'standard_name': 'wind_speed', - 'description': 'wind_speed', - 'units': 'm*s^-1', - 'valid_min': 0, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'wind_speed_mean': { - 'standard_name': 'wind_speed', - 'description': 'wind_speed', - 'units': 'm*s^-1', - 'valid_min': 0, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'wind_dir_high': { - 'standard_name': 'wind_direction', - 'description': 'wind_direction', - 'units': 'degrees', - 'valid_min': 0, - 'valid_max': 360, - '_type': 'f', - '_shape': ('time',), - }, - - 'wind_dir_low': { - 'standard_name': 'wind_direction', - 'description': 'wind_direction', - 'units': 'degrees', - 'valid_min': 0, - 'valid_max': 360, - '_type': 'f', - '_shape': ('time',), - }, - - 'wind_dir_mean': { - 'standard_name': 'wind_direction', - 'description': 'wind_direction', - 'units': 'degrees', - 'valid_min': 0, - 'valid_max': 360, - '_type': 'f', - '_shape': ('time',), - }, - - 'rh_shield_freq_high': { - 'standard_name': '', - 'descrption' : '', - 'units': 'hz', - '_type': 'f', - '_shape': ('time',), - }, - - 'rh_shield_freq_low': { - 'standard_name': '', - 'descrption' : '', - 'units': 'hz', - '_type': 'f', - '_shape': ('time',), - }, - - 'rh_shield_freq_mean': { - 'standard_name': '', - 'descrption' : '', - 'units': 'hz', - '_type': 'f', - '_shape': ('time',), - }, - - 'rh_high': { - 'standard_name': 'relative_humidity', - 'descripiton': 'Relative humidity', - 'units': '%', - 'valid_min': 0, - 'valid_max': 100, - '_type': 'f', - '_shape': ('time',), - }, - - 'rh_low': { - 'standard_name': 'relative_humidity', - 'descripiton': 'Relative humidity', - 'units': '%', - 'valid_min': 0, - 'valid_max': 100, - '_type': 'f', - '_shape': ('time',), - }, - - 'rh_mean': { - 'standard_name': 'relative_humidity', - 'descripiton': 'Relative humidity', - 'units': '%', - 'valid_min': 0, - 'valid_max': 100, - '_type': 'f', - '_shape': ('time',), - }, - - 'air_temp_6_3m_high': { - 'standard_name': 'air_temperature', - 'description': 'Air temperature 6.3m from tower base', - 'units': 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'air_temp_6_3m_low': { - 'standard_name': 'air_temperature', - 'description': 'Air temperature 6.3m from tower base', - 'units': 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'air_temp_6_3m_mean': { - 'standard_name': 'air_temperature', - 'description': 'Air temperature 6.3m from tower base', - 'units': 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'dewpoint_high': { - 'standard_name': 'dewpoint_temperature', - 'description': 'Calculated dewpoint temperature', - 'units': 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'dewpoint_low': { - 'standard_name': 'dewpoint_temperature', - 'description': 'Calculated dewpoint temperature', - 'units': 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'dewpoint_mean': { - 'standard_name': 'dewpoint_temperature', - 'description': 'Calculated dewpoint temperature', - 'units': 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'rtd_shield_freq_high': { - 'standard_name': '', - 'description': '', - 'units': '', - '_type': 'f', - '_shape': ('time',), - }, - - 'rtd_shield_freq_low': { - 'standard_name': '', - 'description': '', - 'units': '', - '_type': 'f', - '_shape': ('time',), - }, - - 'rtd_shield_freq_mean': { - 'standard_name': '', - 'description': '', - 'units': '', - '_type': 'f', - '_shape': ('time',), - }, - - 'air_temp_high': { - 'standard_name': 'air_temperature', - 'description': 'Air temperature', - 'units': 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'air_temp_low': { - 'standard_name': 'air_temperature', - 'description': 'Air temperature', - 'units': 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'air_temp_mean': { - 'standard_name': 'air_temperature', - 'description': 'Air temperature', - 'units': 'degC', - 'valid_min': -50, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'solar_flux_high': { - 'standard_name': 'solar_flux', - 'description': 'Solar flux', - 'units': 'w*m^-2', - 'valid_min': 0, - 'valid_max': 3000, - '_type': 'f', - '_shape': ('time',), - }, - - 'solar_flux_low': { - 'standard_name': 'solar_flux', - 'description': 'Solar flux', - 'units': 'w*m^-2', - 'valid_min': 0, - 'valid_max': 3000, - '_type': 'f', - '_shape': ('time',), - }, - - 'solar_flux_mean': { - 'standard_name': 'solar_flux', - 'description': 'Solar flux', - 'units': 'w*m^-2', - 'valid_min': 0, - 'valid_max': 3000, - '_type': 'f', - '_shape': ('time',), - }, - - 'precip_high': { - 'standard_name': '', - 'description': 'Precipitation', - 'units': 'mm', - 'valid_min': 0, - 'valid_max': 254, - '_type': 'f', - '_shape': ('time',), - }, - - 'precip_low': { - 'standard_name': '', - 'description': 'Precipitation', - 'units': 'mm', - 'valid_min': 0, - 'valid_max': 254, - '_type': 'f', - '_shape': ('time',), - }, - - 'precip_mean': { - 'standard_name': '', - 'description': 'Precipitation', - 'units': 'mm', - 'valid_min': 0, - 'valid_max': 254, - '_type': 'f', - '_shape': ('time',), - }, - - 'accum_precip_high': { - 'standard_name': 'axxumulated_precipitation', - 'description': 'Precipitation accumulated since 0Z', - 'units': 'mm', - 'valid_min': 0, - 'valid_max': 254, - '_type': 'f', - '_shape': ('time',), - }, - - 'accum_precip_low': { - 'standard_name': 'axxumulated_precipitation', - 'description': 'Precipitation accumulated since 0Z', - 'units': 'mm', - 'valid_min': 0, - 'valid_max': 254, - '_type': 'f', - '_shape': ('time',), - }, - - 'accum_precip_mean': { - 'standard_name': 'axxumulated_precipitation', - 'description': 'Precipitation accumulated since 0Z', - 'units': 'mm', - 'valid_min': 0, - 'valid_max': 254, - '_type': 'f', - '_shape': ('time',), - }, - - 'altimeter_high': { - 'standard_name': '', - 'description': '', - 'units': 'inHg', - '_type': 'f', - '_shape': ('time',), - }, - - 'altimeter_low': { - 'standard_name': '', - 'description': '', - 'units': 'inHg', - '_type': 'f', - '_shape': ('time',), - }, - - 'altimeter_mean': { - 'standard_name': '', - 'description': '', - 'units': 'inHg', - '_type': 'f', - '_shape': ('time',), - }, - - 'gust_high': - { - 'standard_name': 'wind_speed_of_gust', - 'descripiton': 'Wind gust over the previous 2 minutes', - 'units': 'm/s', - 'valid_min': 0, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'gust_low': - { - 'standard_name': 'wind_speed_of_gust', - 'descripiton': 'Wind gust over the previous 2 minutes', - 'units': 'm/s', - 'valid_min': 0, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - }, - - 'peak_wind': - { - 'standard_name': 'wind_speed', - 'description': 'fastest 5-second wind_speed', - 'units': 'm/s', - 'valid_min': 0, - 'valid_max': 50, - '_type': 'f', - '_shape': ('time',), - } -}) - -def make_Var_db(aoss_vars): - var_dict = {} - for key in aoss_vars: - #there's a typo in some var names - entry = aoss_vars[key] - key = key.replace('presure','pressure') - var_dict[key] = schema.Var( - entry.get('_type','f'), - entry.get('standard_name',''), - key, - entry.get('description',''), - entry.get('units',''), - entry.get('valid_min',''), - entry.get('valid_max',''), - ) - return var_dict -AOSS_VARS = make_Var_db(AOSS_VARS) diff --git a/aosstower/level_b1/nc.py b/aosstower/level_b1/nc.py index 81a8146..7115e8c 100644 --- a/aosstower/level_b1/nc.py +++ b/aosstower/level_b1/nc.py @@ -4,7 +4,6 @@ import logging import pandas as pd from datetime import datetime as dt from aosstower.l00 import parser -import avg_database from netCDF4 import Dataset import numpy as np import platform @@ -14,6 +13,18 @@ import calc LOG = logging.getLogger(__name__) + +#create the '_mean','_low','_high' file structure +def make_mean_dict(source_dict): + dest_dict = {} + for key in source_dict: + dest_dict[key+'_high'] = source_dict[key] + dest_dict[key+'_mean'] = source_dict[key] + dest_dict[key+'_low'] = source_dict[key] + return dest_dict + +mean_database = make_mean_dict(parser.database) + def filterArray(array, valid_min, valid_max): qcControl = [] @@ -67,8 +78,8 @@ def createVariables(ncFile, firstStamp, chunksizes, zlib, database=parser.databa 'lon': [np.float32, None, float(-999), '-180L', 'longitude', None, 'degrees_east', '180L', None], 'lat': [np.float32, None, float(-999), '-90L', 'latitude', None, 'degrees_north', '90L', None], 'alt': [np.float32, None, float(-999), None, 'height', 'vertical distance', 'm', None, None], - 'base_time': [np.float32, None, float(-999), None, 'time', btln, btu, None, None], - 'time_offset': [np.float32, 'time', float(-999), None, 'time', tln, tu, None, None], + 'base_time': [np.int32, None, float(-999), None, 'time', btln, btu, None, None], + 'time_offset': [np.float64, 'time', float(-999), None, 'time', tln, tu, None, None], 'station_name': ['c', 'max_len_station_name', '-', None, None, 'station name', None, None, 'timeseries_id'], 'time': [np.float32, 'time', float(-999), None, None, "Time offset from epoch", "seconds since 1970-01-01 00:00:00Z", None, None, None] } @@ -312,7 +323,7 @@ def writeVars(ncFile, frame, database=parser.database): baseTimeValue = baseTimeValue.total_seconds() #create time numpy - timeNumpy = np.empty(len(stamps), dtype='float32') + timeNumpy = np.empty(len(stamps), dtype='float64') counter = 0 @@ -328,7 +339,7 @@ def writeVars(ncFile, frame, database=parser.database): fileVar = ncFile.variables fileVar['base_time'].assignValue(baseTimeValue) fileVar['time_offset'][:] = timeNumpy - fileVar['time'][:] = timeNumpy + baseTimeValue + fileVar['time'][:] = timeNumpy #write coordinate var values to file #alt might not be right, need to verify @@ -497,7 +508,7 @@ def main(): logging.basicConfig(level=level) - database = avg_database.AOSS_VARS if args.interval else parser.database + database = mean_database if args.interval else parser.database if(args.start_time and args.end_time): result = createGiantNetCDF(args.start_time, args.end_time, args.input_files, args.output[0], args.zlib, args.chunk_size, args.interval, database) -- GitLab