Skip to content
Snippets Groups Projects
Commit c1f5d51e authored by Matthew Westphall's avatar Matthew Westphall
Browse files

updated netCDF4 datatypes to match ARM standard

parent 0cae1fd7
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
aosstower/level_b1/avg_database.py deleted 100644 → 0
+ 0
− 838
View file @ 0cae1fd7
# CF Metadata 1.6 compliant structure for NetCDF file. This is essentially
# a pythonified version of the CDL
from collections import OrderedDict
import aosstower.schema as schema
BASIC_STATION = {
'globals': {
'source': 'surface observation',
'institution': 'UW SSEC',
'featureType': 'timeSeries',
'Conventions': 'ARM-1.2 CF-1.6',
'data_level': 'b1',
'datastream' : 'aoss.tower.nc-1mo-1d.lb1.v00',
'software_version': '00'
},
'dimensions': {
'max_len_station_name': 32,
'time': None # Unlimited
},
'variables': OrderedDict({
'base_time': {
'string': '', # Filled in by creator
'standard_name': 'time',
'long_name': 'base time as unix timestamp',
'units': 'seconds since 1970-01-01 00:00:00 0:00',
'_type': 'd',
'_shape': tuple(),
},
'time_offset': {
'long_name': 'time offset from base_time',
'standard_name': 'time',
'units': '', # Filled in by creator
'_type': 'd',
'_shape': ('time',),
},
'time': {
'long_name': 'time offset from epoch',
'standard_name': 'time',
'units': 'seconds since 1970-01-01 00:00:00 0:00',
'_type': 'd',
'_shape': ('time',),
},
'station_name': {
'cf_role': 'timeseries_id',
'long_name': 'station name',
'_type': 'S1',
'_shape': ('max_len_station_name',),
},
'lat': {
'standard_name': 'latitude',
'units': 'degrees_north',
'valid_min': -90,
'valid_max': 90,
'_type': 'f',
'_shape': tuple(),
},
'lon': {
'standard_name': 'longitude',
'units': 'degrees_east',
'valid_min': -180,
'valid_max': 180,
'_type': 'f',
'_shape': tuple(),
},
'alt': {
'long_name': 'vertical distance',
'standard_name': 'height',
'units': 'm',
'positive': 'up',
'axis': 'Z',
'_type': 'f',
'_shape': tuple(),
}
})
}
AOSS_VARS = OrderedDict({
'box_temp_high': {
'standard_name': 'air_temperature',
'units': 'degC',
'description': 'Auxillary Temperature',
'_type': 'f',
'_shape': ('time',),
},
'box_temp_low': {
'standard_name': 'air_temperature',
'units': 'degC',
'description': 'Auxillary Temperature',
'_type': 'f',
'_shape': ('time',),
},
'box_temp_mean': {
'standard_name': 'air_temperature',
'units': 'degC',
'description': 'Auxillary Temperature',
'_type': 'f',
'_shape': ('time',),
},
'box_presure_high': {
'standard_name': 'air_pressure',
'description': 'Pressure inside the data logger enclosure',
'units': 'hpa',
'valid_min': 850,
'valid_max': '1100',
'_type': 'f',
'_shape': ('time',),
},
'box_presure_low': {
'standard_name': 'air_pressure',
'description': 'Pressure inside the data logger enclosure',
'units': 'hpa',
'valid_min': 850,
'valid_max': '1100',
'_type': 'f',
'_shape': ('time',),
},
'box_presure_mean': {
'standard_name': 'air_pressure',
'description': 'Pressure inside the data logger enclosure',
'units': 'hpa',
'valid_min': 850,
'valid_max': '1100',
'_type': 'f',
'_shape': ('time',),
},
'paro_air_temp_period_high': {
'standard_name': '',
'description': '',
'units': '1',
'_type': 'f',
'_shape': ('time',),
},
'paro_air_temp_period_low': {
'standard_name': '',
'description': '',
'units': '1',
'_type': 'f',
'_shape': ('time',),
},
'paro_air_temp_period_mean': {
'standard_name': '',
'description': '',
'units': '1',
'_type': 'f',
'_shape': ('time',),
},
'paro_pressure_period_high': {
'standard_name': '',
'description': '',
'units': '1',
'_type': 'f',
'_shape': ('time',),
},
'paro_pressure_period_low': {
'standard_name': '',
'description': '',
'units': '1',
'_type': 'f',
'_shape': ('time',),
},
'paro_pressure_period_mean': {
'standard_name': '',
'description': '',
'units': '1',
'_type': 'f',
'_shape': ('time',),
},
'paro_air_temp_high': {
'standard_name': 'air_temperature',
'description': '',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'paro_air_temp_low': {
'standard_name': 'air_temperature',
'description': '',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'paro_air_temp_mean': {
'standard_name': 'air_temperature',
'description': '',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'pressure_high': {
'standard_name': 'air_pressure',
'description': 'Air pressure as measured from the PAROSCI pressure sensor',
'units': 'hpa',
'valid_min': 850,
'valid_max': 1100,
'_type': 'f',
'_shape': ('time',),
},
'pressure_low': {
'standard_name': 'air_pressure',
'description': 'Air pressure as measured from the PAROSCI pressure sensor',
'units': 'hpa',
'valid_min': 850,
'valid_max': 1100,
'_type': 'f',
'_shape': ('time',),
},
'pressure_mean': {
'standard_name': 'air_pressure',
'description': 'Air pressure as measured from the PAROSCI pressure sensor',
'units': 'hpa',
'valid_min': 850,
'valid_max': 1100,
'_type': 'f',
'_shape': ('time',),
},
'paro_cal_sig_high': {
'standard_name': '',
'description': '',
'units': '',
'_type': 'f',
'_shape': ('time',),
},
'paro_cal_sig_low': {
'standard_name': '',
'description': '',
'units': '',
'_type': 'f',
'_shape': ('time',),
},
'paro_cal_sig_mean': {
'standard_name': '',
'description': '',
'units': '',
'_type': 'f',
'_shape': ('time',),
},
'box_rh_high': {
'standard_name': 'relative humidity',
'description': 'Relative humidity inside the data logger enclosure',
'units': '%',
'valid_min': 0,
'valid_max': 100,
'_type': 'f',
'_shape': ('time',),
},
'box_rh_low': {
'standard_name': 'relative humidity',
'description': 'Relative humidity inside the data logger enclosure',
'units': '%',
'valid_min': 0,
'valid_max': 100,
'_type': 'f',
'_shape': ('time',),
},
'box_rh_mean': {
'standard_name': 'relative humidity',
'description': 'Relative humidity inside the data logger enclosure',
'units': '%',
'valid_min': 0,
'valid_max': 100,
'_type': 'f',
'_shape': ('time',),
},
'box_air_temp_high': {
'standard_name': 'air_temperature',
'description': 'Air temperature inside the data logger enclosure',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'box_air_temp_low': {
'standard_name': 'air_temperature',
'description': 'Air temperature inside the data logger enclosure',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'box_air_temp_mean': {
'standard_name': 'air_temperature',
'description': 'Air temperature inside the data logger enclosure',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_2_high': {
'standard_name': 'air_temperature',
'description': 'Auxillary air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_2_low': {
'standard_name': 'air_temperature',
'description': 'Auxillary air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_2_mean': {
'standard_name': 'air_temperature',
'description': 'Auxillary air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_3_high': {
'standard_name': 'air_temperature',
'description': 'Auxillary air temperature',
'units' : 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_3_low': {
'standard_name': 'air_temperature',
'description': 'Auxillary air temperature',
'units' : 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_3_mean': {
'standard_name': 'air_temperature',
'description': 'Auxillary air temperature',
'units' : 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_4_high': {
'standard_name': 'air_temperature',
'description' : 'Auxillary air temperature',
'units' : 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_4_low': {
'standard_name': 'air_temperature',
'description' : 'Auxillary air temperature',
'units' : 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_4_mean': {
'standard_name': 'air_temperature',
'description' : 'Auxillary air temperature',
'units' : 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_5_high': {
'standard_name': 'air_temperature',
'descripiton': 'Auxillary air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_5_low': {
'standard_name': 'air_temperature',
'descripiton': 'Auxillary air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_5_mean': {
'standard_name': 'air_temperature',
'descripiton': 'Auxillary air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'wind_speed_high': {
'standard_name': 'wind_speed',
'description': 'wind_speed',
'units': 'm*s^-1',
'valid_min': 0,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'wind_speed_low': {
'standard_name': 'wind_speed',
'description': 'wind_speed',
'units': 'm*s^-1',
'valid_min': 0,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'wind_speed_mean': {
'standard_name': 'wind_speed',
'description': 'wind_speed',
'units': 'm*s^-1',
'valid_min': 0,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'wind_dir_high': {
'standard_name': 'wind_direction',
'description': 'wind_direction',
'units': 'degrees',
'valid_min': 0,
'valid_max': 360,
'_type': 'f',
'_shape': ('time',),
},
'wind_dir_low': {
'standard_name': 'wind_direction',
'description': 'wind_direction',
'units': 'degrees',
'valid_min': 0,
'valid_max': 360,
'_type': 'f',
'_shape': ('time',),
},
'wind_dir_mean': {
'standard_name': 'wind_direction',
'description': 'wind_direction',
'units': 'degrees',
'valid_min': 0,
'valid_max': 360,
'_type': 'f',
'_shape': ('time',),
},
'rh_shield_freq_high': {
'standard_name': '',
'descrption' : '',
'units': 'hz',
'_type': 'f',
'_shape': ('time',),
},
'rh_shield_freq_low': {
'standard_name': '',
'descrption' : '',
'units': 'hz',
'_type': 'f',
'_shape': ('time',),
},
'rh_shield_freq_mean': {
'standard_name': '',
'descrption' : '',
'units': 'hz',
'_type': 'f',
'_shape': ('time',),
},
'rh_high': {
'standard_name': 'relative_humidity',
'descripiton': 'Relative humidity',
'units': '%',
'valid_min': 0,
'valid_max': 100,
'_type': 'f',
'_shape': ('time',),
},
'rh_low': {
'standard_name': 'relative_humidity',
'descripiton': 'Relative humidity',
'units': '%',
'valid_min': 0,
'valid_max': 100,
'_type': 'f',
'_shape': ('time',),
},
'rh_mean': {
'standard_name': 'relative_humidity',
'descripiton': 'Relative humidity',
'units': '%',
'valid_min': 0,
'valid_max': 100,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_6_3m_high': {
'standard_name': 'air_temperature',
'description': 'Air temperature 6.3m from tower base',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_6_3m_low': {
'standard_name': 'air_temperature',
'description': 'Air temperature 6.3m from tower base',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_6_3m_mean': {
'standard_name': 'air_temperature',
'description': 'Air temperature 6.3m from tower base',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'dewpoint_high': {
'standard_name': 'dewpoint_temperature',
'description': 'Calculated dewpoint temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'dewpoint_low': {
'standard_name': 'dewpoint_temperature',
'description': 'Calculated dewpoint temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'dewpoint_mean': {
'standard_name': 'dewpoint_temperature',
'description': 'Calculated dewpoint temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'rtd_shield_freq_high': {
'standard_name': '',
'description': '',
'units': '',
'_type': 'f',
'_shape': ('time',),
},
'rtd_shield_freq_low': {
'standard_name': '',
'description': '',
'units': '',
'_type': 'f',
'_shape': ('time',),
},
'rtd_shield_freq_mean': {
'standard_name': '',
'description': '',
'units': '',
'_type': 'f',
'_shape': ('time',),
},
'air_temp_high': {
'standard_name': 'air_temperature',
'description': 'Air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_low': {
'standard_name': 'air_temperature',
'description': 'Air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_mean': {
'standard_name': 'air_temperature',
'description': 'Air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'solar_flux_high': {
'standard_name': 'solar_flux',
'description': 'Solar flux',
'units': 'w*m^-2',
'valid_min': 0,
'valid_max': 3000,
'_type': 'f',
'_shape': ('time',),
},
'solar_flux_low': {
'standard_name': 'solar_flux',
'description': 'Solar flux',
'units': 'w*m^-2',
'valid_min': 0,
'valid_max': 3000,
'_type': 'f',
'_shape': ('time',),
},
'solar_flux_mean': {
'standard_name': 'solar_flux',
'description': 'Solar flux',
'units': 'w*m^-2',
'valid_min': 0,
'valid_max': 3000,
'_type': 'f',
'_shape': ('time',),
},
'precip_high': {
'standard_name': '',
'description': 'Precipitation',
'units': 'mm',
'valid_min': 0,
'valid_max': 254,
'_type': 'f',
'_shape': ('time',),
},
'precip_low': {
'standard_name': '',
'description': 'Precipitation',
'units': 'mm',
'valid_min': 0,
'valid_max': 254,
'_type': 'f',
'_shape': ('time',),
},
'precip_mean': {
'standard_name': '',
'description': 'Precipitation',
'units': 'mm',
'valid_min': 0,
'valid_max': 254,
'_type': 'f',
'_shape': ('time',),
},
'accum_precip_high': {
'standard_name': 'axxumulated_precipitation',
'description': 'Precipitation accumulated since 0Z',
'units': 'mm',
'valid_min': 0,
'valid_max': 254,
'_type': 'f',
'_shape': ('time',),
},
'accum_precip_low': {
'standard_name': 'axxumulated_precipitation',
'description': 'Precipitation accumulated since 0Z',
'units': 'mm',
'valid_min': 0,
'valid_max': 254,
'_type': 'f',
'_shape': ('time',),
},
'accum_precip_mean': {
'standard_name': 'axxumulated_precipitation',
'description': 'Precipitation accumulated since 0Z',
'units': 'mm',
'valid_min': 0,
'valid_max': 254,
'_type': 'f',
'_shape': ('time',),
},
'altimeter_high': {
'standard_name': '',
'description': '',
'units': 'inHg',
'_type': 'f',
'_shape': ('time',),
},
'altimeter_low': {
'standard_name': '',
'description': '',
'units': 'inHg',
'_type': 'f',
'_shape': ('time',),
},
'altimeter_mean': {
'standard_name': '',
'description': '',
'units': 'inHg',
'_type': 'f',
'_shape': ('time',),
},
'gust_high':
{
'standard_name': 'wind_speed_of_gust',
'descripiton': 'Wind gust over the previous 2 minutes',
'units': 'm/s',
'valid_min': 0,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'gust_low':
{
'standard_name': 'wind_speed_of_gust',
'descripiton': 'Wind gust over the previous 2 minutes',
'units': 'm/s',
'valid_min': 0,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'peak_wind':
{
'standard_name': 'wind_speed',
'description': 'fastest 5-second wind_speed',
'units': 'm/s',
'valid_min': 0,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
}
})
def make_Var_db(aoss_vars):
var_dict = {}
for key in aoss_vars:
#there's a typo in some var names
entry = aoss_vars[key]
key = key.replace('presure','pressure')
var_dict[key] = schema.Var(
entry.get('_type','f'),
entry.get('standard_name',''),
key,
entry.get('description',''),
entry.get('units',''),
entry.get('valid_min',''),
entry.get('valid_max',''),
)
return var_dict
AOSS_VARS = make_Var_db(AOSS_VARS)
aosstower/level_b1/nc.py
+ 17
− 6
View file @ c1f5d51e
...@@ -4,7 +4,6 @@ import logging ...@@ -4,7 +4,6 @@ import logging
import pandas as pd import pandas as pd
from datetime import datetime as dt from datetime import datetime as dt
from aosstower.l00 import parser from aosstower.l00 import parser
import avg_database
from netCDF4 import Dataset from netCDF4 import Dataset
import numpy as np import numpy as np
import platform import platform
...@@ -14,6 +13,18 @@ import calc ...@@ -14,6 +13,18 @@ import calc
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
#create the '_mean','_low','_high' file structure
def make_mean_dict(source_dict):
dest_dict = {}
for key in source_dict:
dest_dict[key+'_high'] = source_dict[key]
dest_dict[key+'_mean'] = source_dict[key]
dest_dict[key+'_low'] = source_dict[key]
return dest_dict
mean_database = make_mean_dict(parser.database)
def filterArray(array, valid_min, valid_max): def filterArray(array, valid_min, valid_max):
qcControl = [] qcControl = []
...@@ -67,8 +78,8 @@ def createVariables(ncFile, firstStamp, chunksizes, zlib, database=parser.databa ...@@ -67,8 +78,8 @@ def createVariables(ncFile, firstStamp, chunksizes, zlib, database=parser.databa
'lon': [np.float32, None, float(-999), '-180L', 'longitude', None, 'degrees_east', '180L', None], 'lon': [np.float32, None, float(-999), '-180L', 'longitude', None, 'degrees_east', '180L', None],
'lat': [np.float32, None, float(-999), '-90L', 'latitude', None, 'degrees_north', '90L', None], 'lat': [np.float32, None, float(-999), '-90L', 'latitude', None, 'degrees_north', '90L', None],
'alt': [np.float32, None, float(-999), None, 'height', 'vertical distance', 'm', None, None], 'alt': [np.float32, None, float(-999), None, 'height', 'vertical distance', 'm', None, None],
'base_time': [np.float32, None, float(-999), None, 'time', btln, btu, None, None], 'base_time': [np.int32, None, float(-999), None, 'time', btln, btu, None, None],
'time_offset': [np.float32, 'time', float(-999), None, 'time', tln, tu, None, None], 'time_offset': [np.float64, 'time', float(-999), None, 'time', tln, tu, None, None],
'station_name': ['c', 'max_len_station_name', '-', None, None, 'station name', None, None, 'timeseries_id'], 'station_name': ['c', 'max_len_station_name', '-', None, None, 'station name', None, None, 'timeseries_id'],
'time': [np.float32, 'time', float(-999), None, None, "Time offset from epoch", "seconds since 1970-01-01 00:00:00Z", None, None, None] 'time': [np.float32, 'time', float(-999), None, None, "Time offset from epoch", "seconds since 1970-01-01 00:00:00Z", None, None, None]
} }
...@@ -312,7 +323,7 @@ def writeVars(ncFile, frame, database=parser.database): ...@@ -312,7 +323,7 @@ def writeVars(ncFile, frame, database=parser.database):
baseTimeValue = baseTimeValue.total_seconds() baseTimeValue = baseTimeValue.total_seconds()
#create time numpy #create time numpy
timeNumpy = np.empty(len(stamps), dtype='float32') timeNumpy = np.empty(len(stamps), dtype='float64')
counter = 0 counter = 0
...@@ -328,7 +339,7 @@ def writeVars(ncFile, frame, database=parser.database): ...@@ -328,7 +339,7 @@ def writeVars(ncFile, frame, database=parser.database):
fileVar = ncFile.variables fileVar = ncFile.variables
fileVar['base_time'].assignValue(baseTimeValue) fileVar['base_time'].assignValue(baseTimeValue)
fileVar['time_offset'][:] = timeNumpy fileVar['time_offset'][:] = timeNumpy
fileVar['time'][:] = timeNumpy + baseTimeValue fileVar['time'][:] = timeNumpy
#write coordinate var values to file #write coordinate var values to file
#alt might not be right, need to verify #alt might not be right, need to verify
...@@ -497,7 +508,7 @@ def main(): ...@@ -497,7 +508,7 @@ def main():
logging.basicConfig(level=level) logging.basicConfig(level=level)
database = avg_database.AOSS_VARS if args.interval else parser.database database = mean_database if args.interval else parser.database
if(args.start_time and args.end_time): if(args.start_time and args.end_time):
result = createGiantNetCDF(args.start_time, args.end_time, args.input_files, args.output[0], args.zlib, args.chunk_size, result = createGiantNetCDF(args.start_time, args.end_time, args.input_files, args.output[0], args.zlib, args.chunk_size,
args.interval, database) args.interval, database)
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment