Skip to content
Snippets Groups Projects
Commit c1f5d51e authored by Matthew Westphall's avatar Matthew Westphall
Browse files

updated netCDF4 datatypes to match ARM standard

parent 0cae1fd7
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
aosstower/level_b1/avg_database.py deleted 100644 → 0
+ 0
− 838
View file @ 0cae1fd7
# CF Metadata 1.6 compliant structure for NetCDF file. This is essentially
# a pythonified version of the CDL
from collections import OrderedDict
import aosstower.schema as schema
BASIC_STATION = {
'globals': {
'source': 'surface observation',
'institution': 'UW SSEC',
'featureType': 'timeSeries',
'Conventions': 'ARM-1.2 CF-1.6',
'data_level': 'b1',
'datastream' : 'aoss.tower.nc-1mo-1d.lb1.v00',
'software_version': '00'
},
'dimensions': {
'max_len_station_name': 32,
'time': None # Unlimited
},
'variables': OrderedDict({
'base_time': {
'string': '', # Filled in by creator
'standard_name': 'time',
'long_name': 'base time as unix timestamp',
'units': 'seconds since 1970-01-01 00:00:00 0:00',
'_type': 'd',
'_shape': tuple(),
},
'time_offset': {
'long_name': 'time offset from base_time',
'standard_name': 'time',
'units': '', # Filled in by creator
'_type': 'd',
'_shape': ('time',),
},
'time': {
'long_name': 'time offset from epoch',
'standard_name': 'time',
'units': 'seconds since 1970-01-01 00:00:00 0:00',
'_type': 'd',
'_shape': ('time',),
},
'station_name': {
'cf_role': 'timeseries_id',
'long_name': 'station name',
'_type': 'S1',
'_shape': ('max_len_station_name',),
},
'lat': {
'standard_name': 'latitude',
'units': 'degrees_north',
'valid_min': -90,
'valid_max': 90,
'_type': 'f',
'_shape': tuple(),
},
'lon': {
'standard_name': 'longitude',
'units': 'degrees_east',
'valid_min': -180,
'valid_max': 180,
'_type': 'f',
'_shape': tuple(),
},
'alt': {
'long_name': 'vertical distance',
'standard_name': 'height',
'units': 'm',
'positive': 'up',
'axis': 'Z',
'_type': 'f',
'_shape': tuple(),
}
})
}
AOSS_VARS = OrderedDict({
'box_temp_high': {
'standard_name': 'air_temperature',
'units': 'degC',
'description': 'Auxillary Temperature',
'_type': 'f',
'_shape': ('time',),
},
'box_temp_low': {
'standard_name': 'air_temperature',
'units': 'degC',
'description': 'Auxillary Temperature',
'_type': 'f',
'_shape': ('time',),
},
'box_temp_mean': {
'standard_name': 'air_temperature',
'units': 'degC',
'description': 'Auxillary Temperature',
'_type': 'f',
'_shape': ('time',),
},
'box_presure_high': {
'standard_name': 'air_pressure',
'description': 'Pressure inside the data logger enclosure',
'units': 'hpa',
'valid_min': 850,
'valid_max': '1100',
'_type': 'f',
'_shape': ('time',),
},
'box_presure_low': {
'standard_name': 'air_pressure',
'description': 'Pressure inside the data logger enclosure',
'units': 'hpa',
'valid_min': 850,
'valid_max': '1100',
'_type': 'f',
'_shape': ('time',),
},
'box_presure_mean': {
'standard_name': 'air_pressure',
'description': 'Pressure inside the data logger enclosure',
'units': 'hpa',
'valid_min': 850,
'valid_max': '1100',
'_type': 'f',
'_shape': ('time',),
},
'paro_air_temp_period_high': {
'standard_name': '',
'description': '',
'units': '1',
'_type': 'f',
'_shape': ('time',),
},
'paro_air_temp_period_low': {
'standard_name': '',
'description': '',
'units': '1',
'_type': 'f',
'_shape': ('time',),
},
'paro_air_temp_period_mean': {
'standard_name': '',
'description': '',
'units': '1',
'_type': 'f',
'_shape': ('time',),
},
'paro_pressure_period_high': {
'standard_name': '',
'description': '',
'units': '1',
'_type': 'f',
'_shape': ('time',),
},
'paro_pressure_period_low': {
'standard_name': '',
'description': '',
'units': '1',
'_type': 'f',
'_shape': ('time',),
},
'paro_pressure_period_mean': {
'standard_name': '',
'description': '',
'units': '1',
'_type': 'f',
'_shape': ('time',),
},
'paro_air_temp_high': {
'standard_name': 'air_temperature',
'description': '',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'paro_air_temp_low': {
'standard_name': 'air_temperature',
'description': '',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'paro_air_temp_mean': {
'standard_name': 'air_temperature',
'description': '',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'pressure_high': {
'standard_name': 'air_pressure',
'description': 'Air pressure as measured from the PAROSCI pressure sensor',
'units': 'hpa',
'valid_min': 850,
'valid_max': 1100,
'_type': 'f',
'_shape': ('time',),
},
'pressure_low': {
'standard_name': 'air_pressure',
'description': 'Air pressure as measured from the PAROSCI pressure sensor',
'units': 'hpa',
'valid_min': 850,
'valid_max': 1100,
'_type': 'f',
'_shape': ('time',),
},
'pressure_mean': {
'standard_name': 'air_pressure',
'description': 'Air pressure as measured from the PAROSCI pressure sensor',
'units': 'hpa',
'valid_min': 850,
'valid_max': 1100,
'_type': 'f',
'_shape': ('time',),
},
'paro_cal_sig_high': {
'standard_name': '',
'description': '',
'units': '',
'_type': 'f',
'_shape': ('time',),
},
'paro_cal_sig_low': {
'standard_name': '',
'description': '',
'units': '',
'_type': 'f',
'_shape': ('time',),
},
'paro_cal_sig_mean': {
'standard_name': '',
'description': '',
'units': '',
'_type': 'f',
'_shape': ('time',),
},
'box_rh_high': {
'standard_name': 'relative humidity',
'description': 'Relative humidity inside the data logger enclosure',
'units': '%',
'valid_min': 0,
'valid_max': 100,
'_type': 'f',
'_shape': ('time',),
},
'box_rh_low': {
'standard_name': 'relative humidity',
'description': 'Relative humidity inside the data logger enclosure',
'units': '%',
'valid_min': 0,
'valid_max': 100,
'_type': 'f',
'_shape': ('time',),
},
'box_rh_mean': {
'standard_name': 'relative humidity',
'description': 'Relative humidity inside the data logger enclosure',
'units': '%',
'valid_min': 0,
'valid_max': 100,
'_type': 'f',
'_shape': ('time',),
},
'box_air_temp_high': {
'standard_name': 'air_temperature',
'description': 'Air temperature inside the data logger enclosure',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'box_air_temp_low': {
'standard_name': 'air_temperature',
'description': 'Air temperature inside the data logger enclosure',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'box_air_temp_mean': {
'standard_name': 'air_temperature',
'description': 'Air temperature inside the data logger enclosure',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_2_high': {
'standard_name': 'air_temperature',
'description': 'Auxillary air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_2_low': {
'standard_name': 'air_temperature',
'description': 'Auxillary air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_2_mean': {
'standard_name': 'air_temperature',
'description': 'Auxillary air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_3_high': {
'standard_name': 'air_temperature',
'description': 'Auxillary air temperature',
'units' : 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_3_low': {
'standard_name': 'air_temperature',
'description': 'Auxillary air temperature',
'units' : 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_3_mean': {
'standard_name': 'air_temperature',
'description': 'Auxillary air temperature',
'units' : 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_4_high': {
'standard_name': 'air_temperature',
'description' : 'Auxillary air temperature',
'units' : 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_4_low': {
'standard_name': 'air_temperature',
'description' : 'Auxillary air temperature',
'units' : 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_4_mean': {
'standard_name': 'air_temperature',
'description' : 'Auxillary air temperature',
'units' : 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_5_high': {
'standard_name': 'air_temperature',
'descripiton': 'Auxillary air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_5_low': {
'standard_name': 'air_temperature',
'descripiton': 'Auxillary air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_5_mean': {
'standard_name': 'air_temperature',
'descripiton': 'Auxillary air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'wind_speed_high': {
'standard_name': 'wind_speed',
'description': 'wind_speed',
'units': 'm*s^-1',
'valid_min': 0,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'wind_speed_low': {
'standard_name': 'wind_speed',
'description': 'wind_speed',
'units': 'm*s^-1',
'valid_min': 0,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'wind_speed_mean': {
'standard_name': 'wind_speed',
'description': 'wind_speed',
'units': 'm*s^-1',
'valid_min': 0,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'wind_dir_high': {
'standard_name': 'wind_direction',
'description': 'wind_direction',
'units': 'degrees',
'valid_min': 0,
'valid_max': 360,
'_type': 'f',
'_shape': ('time',),
},
'wind_dir_low': {
'standard_name': 'wind_direction',
'description': 'wind_direction',
'units': 'degrees',
'valid_min': 0,
'valid_max': 360,
'_type': 'f',
'_shape': ('time',),
},
'wind_dir_mean': {
'standard_name': 'wind_direction',
'description': 'wind_direction',
'units': 'degrees',
'valid_min': 0,
'valid_max': 360,
'_type': 'f',
'_shape': ('time',),
},
'rh_shield_freq_high': {
'standard_name': '',
'descrption' : '',
'units': 'hz',
'_type': 'f',
'_shape': ('time',),
},
'rh_shield_freq_low': {
'standard_name': '',
'descrption' : '',
'units': 'hz',
'_type': 'f',
'_shape': ('time',),
},
'rh_shield_freq_mean': {
'standard_name': '',
'descrption' : '',
'units': 'hz',
'_type': 'f',
'_shape': ('time',),
},
'rh_high': {
'standard_name': 'relative_humidity',
'descripiton': 'Relative humidity',
'units': '%',
'valid_min': 0,
'valid_max': 100,
'_type': 'f',
'_shape': ('time',),
},
'rh_low': {
'standard_name': 'relative_humidity',
'descripiton': 'Relative humidity',
'units': '%',
'valid_min': 0,
'valid_max': 100,
'_type': 'f',
'_shape': ('time',),
},
'rh_mean': {
'standard_name': 'relative_humidity',
'descripiton': 'Relative humidity',
'units': '%',
'valid_min': 0,
'valid_max': 100,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_6_3m_high': {
'standard_name': 'air_temperature',
'description': 'Air temperature 6.3m from tower base',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_6_3m_low': {
'standard_name': 'air_temperature',
'description': 'Air temperature 6.3m from tower base',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_6_3m_mean': {
'standard_name': 'air_temperature',
'description': 'Air temperature 6.3m from tower base',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'dewpoint_high': {
'standard_name': 'dewpoint_temperature',
'description': 'Calculated dewpoint temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'dewpoint_low': {
'standard_name': 'dewpoint_temperature',
'description': 'Calculated dewpoint temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'dewpoint_mean': {
'standard_name': 'dewpoint_temperature',
'description': 'Calculated dewpoint temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'rtd_shield_freq_high': {
'standard_name': '',
'description': '',
'units': '',
'_type': 'f',
'_shape': ('time',),
},
'rtd_shield_freq_low': {
'standard_name': '',
'description': '',
'units': '',
'_type': 'f',
'_shape': ('time',),
},
'rtd_shield_freq_mean': {
'standard_name': '',
'description': '',
'units': '',
'_type': 'f',
'_shape': ('time',),
},
'air_temp_high': {
'standard_name': 'air_temperature',
'description': 'Air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_low': {
'standard_name': 'air_temperature',
'description': 'Air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_mean': {
'standard_name': 'air_temperature',
'description': 'Air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'solar_flux_high': {
'standard_name': 'solar_flux',
'description': 'Solar flux',
'units': 'w*m^-2',
'valid_min': 0,
'valid_max': 3000,
'_type': 'f',
'_shape': ('time',),
},
'solar_flux_low': {
'standard_name': 'solar_flux',
'description': 'Solar flux',
'units': 'w*m^-2',
'valid_min': 0,
'valid_max': 3000,
'_type': 'f',
'_shape': ('time',),
},
'solar_flux_mean': {
'standard_name': 'solar_flux',
'description': 'Solar flux',
'units': 'w*m^-2',
'valid_min': 0,
'valid_max': 3000,
'_type': 'f',
'_shape': ('time',),
},
'precip_high': {
'standard_name': '',
'description': 'Precipitation',
'units': 'mm',
'valid_min': 0,
'valid_max': 254,
'_type': 'f',
'_shape': ('time',),
},
'precip_low': {
'standard_name': '',
'description': 'Precipitation',
'units': 'mm',
'valid_min': 0,
'valid_max': 254,
'_type': 'f',
'_shape': ('time',),
},
'precip_mean': {
'standard_name': '',
'description': 'Precipitation',
'units': 'mm',
'valid_min': 0,
'valid_max': 254,
'_type': 'f',
'_shape': ('time',),
},
'accum_precip_high': {
'standard_name': 'axxumulated_precipitation',
'description': 'Precipitation accumulated since 0Z',
'units': 'mm',
'valid_min': 0,
'valid_max': 254,
'_type': 'f',
'_shape': ('time',),
},
'accum_precip_low': {
'standard_name': 'axxumulated_precipitation',
'description': 'Precipitation accumulated since 0Z',
'units': 'mm',
'valid_min': 0,
'valid_max': 254,
'_type': 'f',
'_shape': ('time',),
},
'accum_precip_mean': {
'standard_name': 'axxumulated_precipitation',
'description': 'Precipitation accumulated since 0Z',
'units': 'mm',
'valid_min': 0,
'valid_max': 254,
'_type': 'f',
'_shape': ('time',),
},
'altimeter_high': {
'standard_name': '',
'description': '',
'units': 'inHg',
'_type': 'f',
'_shape': ('time',),
},
'altimeter_low': {
'standard_name': '',
'description': '',
'units': 'inHg',
'_type': 'f',
'_shape': ('time',),
},
'altimeter_mean': {
'standard_name': '',
'description': '',
'units': 'inHg',
'_type': 'f',
'_shape': ('time',),
},
'gust_high':
{
'standard_name': 'wind_speed_of_gust',
'descripiton': 'Wind gust over the previous 2 minutes',
'units': 'm/s',
'valid_min': 0,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'gust_low':
{
'standard_name': 'wind_speed_of_gust',
'descripiton': 'Wind gust over the previous 2 minutes',
'units': 'm/s',
'valid_min': 0,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'peak_wind':
{
'standard_name': 'wind_speed',
'description': 'fastest 5-second wind_speed',
'units': 'm/s',
'valid_min': 0,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
}
})
def make_Var_db(aoss_vars):
var_dict = {}
for key in aoss_vars:
#there's a typo in some var names
entry = aoss_vars[key]
key = key.replace('presure','pressure')
var_dict[key] = schema.Var(
entry.get('_type','f'),
entry.get('standard_name',''),
key,
entry.get('description',''),
entry.get('units',''),
entry.get('valid_min',''),
entry.get('valid_max',''),
)
return var_dict
AOSS_VARS = make_Var_db(AOSS_VARS)
aosstower/level_b1/nc.py
+ 17
− 6
View file @ c1f5d51e
......@@ -4,7 +4,6 @@ import logging
import pandas as pd
from datetime import datetime as dt
from aosstower.l00 import parser
import avg_database
from netCDF4 import Dataset
import numpy as np
import platform
......@@ -14,6 +13,18 @@ import calc
LOG = logging.getLogger(__name__)
#create the '_mean','_low','_high' file structure
def make_mean_dict(source_dict):
dest_dict = {}
for key in source_dict:
dest_dict[key+'_high'] = source_dict[key]
dest_dict[key+'_mean'] = source_dict[key]
dest_dict[key+'_low'] = source_dict[key]
return dest_dict
mean_database = make_mean_dict(parser.database)
def filterArray(array, valid_min, valid_max):
qcControl = []
......@@ -67,8 +78,8 @@ def createVariables(ncFile, firstStamp, chunksizes, zlib, database=parser.databa
'lon': [np.float32, None, float(-999), '-180L', 'longitude', None, 'degrees_east', '180L', None],
'lat': [np.float32, None, float(-999), '-90L', 'latitude', None, 'degrees_north', '90L', None],
'alt': [np.float32, None, float(-999), None, 'height', 'vertical distance', 'm', None, None],
'base_time': [np.float32, None, float(-999), None, 'time', btln, btu, None, None],
'time_offset': [np.float32, 'time', float(-999), None, 'time', tln, tu, None, None],
'base_time': [np.int32, None, float(-999), None, 'time', btln, btu, None, None],
'time_offset': [np.float64, 'time', float(-999), None, 'time', tln, tu, None, None],
'station_name': ['c', 'max_len_station_name', '-', None, None, 'station name', None, None, 'timeseries_id'],
'time': [np.float32, 'time', float(-999), None, None, "Time offset from epoch", "seconds since 1970-01-01 00:00:00Z", None, None, None]
}
......@@ -312,7 +323,7 @@ def writeVars(ncFile, frame, database=parser.database):
baseTimeValue = baseTimeValue.total_seconds()
#create time numpy
timeNumpy = np.empty(len(stamps), dtype='float32')
timeNumpy = np.empty(len(stamps), dtype='float64')
counter = 0
......@@ -328,7 +339,7 @@ def writeVars(ncFile, frame, database=parser.database):
fileVar = ncFile.variables
fileVar['base_time'].assignValue(baseTimeValue)
fileVar['time_offset'][:] = timeNumpy
fileVar['time'][:] = timeNumpy + baseTimeValue
fileVar['time'][:] = timeNumpy
#write coordinate var values to file
#alt might not be right, need to verify
......@@ -497,7 +508,7 @@ def main():
logging.basicConfig(level=level)
database = avg_database.AOSS_VARS if args.interval else parser.database
database = mean_database if args.interval else parser.database
if(args.start_time and args.end_time):
result = createGiantNetCDF(args.start_time, args.end_time, args.input_files, args.output[0], args.zlib, args.chunk_size,
args.interval, database)
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment