Skip to content
Snippets Groups Projects
Commit 42481b39 authored by kgao's avatar kgao
Browse files

Finished monthly netCDF header draft 

need to reorganize headers in the future
parent 721fed3c
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
aosstower/level_b1/monthly/nc_monthly.py 0 → 100644
+ 945
− 0
View file @ 42481b39
import platform
from netCDF4 import Dataset
from datetime import datetime as dt
from collections import OrderedDict
import os
# CF Metadata 1.6 compliant structure for NetCDF file. This is essentially
# a pythonified version of the CDL
BASIC_STATION = {
'globals': {
'source': 'surface observation',
'institution': 'UW SSEC',
'featureType': 'timeSeries',
'Conventions': 'ARM-1.2 CF-1.6',
'data_level': 'b1',
'datastream' : 'aoss.tower.y_nc.lb1.v00',
'software_version': '00'
},
'dimensions': {
'max_len_station_name': 32,
'time': None # Unlimited
},
'variables': OrderedDict({
'base_time': {
'string': '', # Filled in by creator
'standard_name': 'time',
'long_name': 'base time as unix timestamp',
'units': 'seconds since 1970-01-01 00:00:00 0:00',
'_type': 'd',
'_shape': tuple(),
},
'time_offset': {
'long_name': 'time offset from base_time',
'standard_name': 'time',
'units': '', # Filled in by creator
'_type': 'd',
'_shape': ('time',),
},
'time': {
'long_name': 'time offset from epoch',
'standard_name': 'time',
'units': 'seconds since 1970-01-01 00:00:00 0:00',
'_type': 'd',
'_shape': ('time',),
},
'station_name': {
'cf_role': 'timeseries_id',
'long_name': 'station name',
'_type': 'S1',
'_shape': ('max_len_station_name',),
},
'lat': {
'standard_name': 'latitude',
'units': 'degrees_north',
'valid_min': -90,
'valid_max': 90,
'_type': 'f',
'_shape': tuple(),
},
'lon': {
'standard_name': 'longitude',
'units': 'degrees_east',
'valid_min': -180,
'valid_max': 180,
'_type': 'f',
'_shape': tuple(),
},
'alt': {
'long_name': 'vertical distance',
'standard_name': 'height',
'units': 'm',
'positive': 'up',
'axis': 'Z',
'_type': 'f',
'_shape': tuple(),
}
})
}
AOSS_VARS = OrderedDict({
'box_temp_high': {
'standard_name': 'air_temperature',
'units': 'degC',
'description': 'Auxillary Temperature',
'_type': 'f',
'_shape': ('time',),
},
'box_temp_low': {
'standard_name': 'air_temperature',
'units': 'degC',
'description': 'Auxillary Temperature',
'_type': 'f',
'_shape': ('time',),
},
'box_temp_mean': {
'standard_name': 'air_temperature',
'units': 'degC',
'description': 'Auxillary Temperature',
'_type': 'f',
'_shape': ('time',),
},
'box_presure_high': {
'standard_name': 'air_pressure',
'description': 'Pressure inside the data logger enclosure',
'units': 'hpa',
'valid_min': 850,
'valid_max': '1100',
'_type': 'f',
'_shape': ('time',),
},
'box_presure_low': {
'standard_name': 'air_pressure',
'description': 'Pressure inside the data logger enclosure',
'units': 'hpa',
'valid_min': 850,
'valid_max': '1100',
'_type': 'f',
'_shape': ('time',),
},
'box_presure_mean': {
'standard_name': 'air_pressure',
'description': 'Pressure inside the data logger enclosure',
'units': 'hpa',
'valid_min': 850,
'valid_max': '1100',
'_type': 'f',
'_shape': ('time',),
},
'paro_air_temp_period_high': {
'standard_name': '',
'description': '',
'units': '1',
'_type': 'f',
'_shape': ('time',),
},
'paro_air_temp_period_low': {
'standard_name': '',
'description': '',
'units': '1',
'_type': 'f',
'_shape': ('time',),
},
'paro_air_temp_period_mean': {
'standard_name': '',
'description': '',
'units': '1',
'_type': 'f',
'_shape': ('time',),
},
'paro_pressure_period_high': {
'standard_name': '',
'description': '',
'units': '1',
'_type': 'f',
'_shape': ('time',),
},
'paro_pressure_period_low': {
'standard_name': '',
'description': '',
'units': '1',
'_type': 'f',
'_shape': ('time',),
},
'paro_pressure_period_mean': {
'standard_name': '',
'description': '',
'units': '1',
'_type': 'f',
'_shape': ('time',),
},
'paro_air_temp_high': {
'standard_name': 'air_temperature',
'description': '',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'paro_air_temp_low': {
'standard_name': 'air_temperature',
'description': '',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'paro_air_temp_mean': {
'standard_name': 'air_temperature',
'description': '',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'pressure_high': {
'standard_name': 'air_pressure',
'description': 'Air pressure as measured from the PAROSCI pressure sensor',
'units': 'hpa',
'valid_min': 850,
'valid_max': 1100,
'_type': 'f',
'_shape': ('time',),
},
'pressure_low': {
'standard_name': 'air_pressure',
'description': 'Air pressure as measured from the PAROSCI pressure sensor',
'units': 'hpa',
'valid_min': 850,
'valid_max': 1100,
'_type': 'f',
'_shape': ('time',),
},
'pressure_mean': {
'standard_name': 'air_pressure',
'description': 'Air pressure as measured from the PAROSCI pressure sensor',
'units': 'hpa',
'valid_min': 850,
'valid_max': 1100,
'_type': 'f',
'_shape': ('time',),
},
'paro_cal_sig_high': {
'standard_name': '',
'description': '',
'units': '',
'_type': 'f',
'_shape': ('time',),
},
'paro_cal_sig_low': {
'standard_name': '',
'description': '',
'units': '',
'_type': 'f',
'_shape': ('time',),
},
'paro_cal_sig_mean': {
'standard_name': '',
'description': '',
'units': '',
'_type': 'f',
'_shape': ('time',),
},
'box_rh_high': {
'standard_name': 'relative humidity',
'description': 'Relative humidity inside the data logger enclosure',
'units': '%',
'valid_min': 0,
'valid_max': 100,
'_type': 'f',
'_shape': ('time',),
},
'box_rh_low': {
'standard_name': 'relative humidity',
'description': 'Relative humidity inside the data logger enclosure',
'units': '%',
'valid_min': 0,
'valid_max': 100,
'_type': 'f',
'_shape': ('time',),
},
'box_rh_mean': {
'standard_name': 'relative humidity',
'description': 'Relative humidity inside the data logger enclosure',
'units': '%',
'valid_min': 0,
'valid_max': 100,
'_type': 'f',
'_shape': ('time',),
},
'box_air_temp_high': {
'standard_name': 'air_temperature',
'description': 'Air temperature inside the data logger enclosure',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'box_air_temp_low': {
'standard_name': 'air_temperature',
'description': 'Air temperature inside the data logger enclosure',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'box_air_temp_mean': {
'standard_name': 'air_temperature',
'description': 'Air temperature inside the data logger enclosure',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_2_high': {
'standard_name': 'air_temperature',
'description': 'Auxillary air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_2_low': {
'standard_name': 'air_temperature',
'description': 'Auxillary air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_2_mean': {
'standard_name': 'air_temperature',
'description': 'Auxillary air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_3_high': {
'standard_name': 'air_temperature',
'description': 'Auxillary air temperature',
'units' : 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_3_low': {
'standard_name': 'air_temperature',
'description': 'Auxillary air temperature',
'units' : 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_3_mean': {
'standard_name': 'air_temperature',
'description': 'Auxillary air temperature',
'units' : 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_4_high': {
'standard_name': 'air_temperature',
'description' : 'Auxillary air temperature',
'units' : 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_4_low': {
'standard_name': 'air_temperature',
'description' : 'Auxillary air temperature',
'units' : 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_4_mean': {
'standard_name': 'air_temperature',
'description' : 'Auxillary air temperature',
'units' : 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_5_high': {
'standard_name': 'air_temperature',
'descripiton': 'Auxillary air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_5_low': {
'standard_name': 'air_temperature',
'descripiton': 'Auxillary air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_5_mean': {
'standard_name': 'air_temperature',
'descripiton': 'Auxillary air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'wind_speed_high': {
'standard_name': 'wind_speed',
'description': 'wind_speed',
'units': 'm*s^-1',
'valid_min': 0,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'wind_speed_low': {
'standard_name': 'wind_speed',
'description': 'wind_speed',
'units': 'm*s^-1',
'valid_min': 0,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'wind_speed_mean': {
'standard_name': 'wind_speed',
'description': 'wind_speed',
'units': 'm*s^-1',
'valid_min': 0,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'wind_dir_high': {
'standard_name': 'wind_direction',
'description': 'wind_direction',
'units': 'degrees',
'valid_min': 0,
'valid_max': 360,
'_type': 'f',
'_shape': ('time',),
},
'wind_dir_low': {
'standard_name': 'wind_direction',
'description': 'wind_direction',
'units': 'degrees',
'valid_min': 0,
'valid_max': 360,
'_type': 'f',
'_shape': ('time',),
},
'wind_dir_mean': {
'standard_name': 'wind_direction',
'description': 'wind_direction',
'units': 'degrees',
'valid_min': 0,
'valid_max': 360,
'_type': 'f',
'_shape': ('time',),
},
'rh_shield_freq_high': {
'standard_name': '',
'descrption' : '',
'units': 'hz',
'_type': 'f',
'_shape': ('time',),
},
'rh_shield_freq_low': {
'standard_name': '',
'descrption' : '',
'units': 'hz',
'_type': 'f',
'_shape': ('time',),
},
'rh_shield_freq_mean': {
'standard_name': '',
'descrption' : '',
'units': 'hz',
'_type': 'f',
'_shape': ('time',),
},
'rh_high': {
'standard_name': 'relative_humidity',
'descripiton': 'Relative humidity',
'units': '%',
'valid_min': 0,
'valid_max': 100,
'_type': 'f',
'_shape': ('time',),
},
'rh_low': {
'standard_name': 'relative_humidity',
'descripiton': 'Relative humidity',
'units': '%',
'valid_min': 0,
'valid_max': 100,
'_type': 'f',
'_shape': ('time',),
},
'rh_mean': {
'standard_name': 'relative_humidity',
'descripiton': 'Relative humidity',
'units': '%',
'valid_min': 0,
'valid_max': 100,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_6_3m_high': {
'standard_name': 'air_temperature',
'description': 'Air temperature 6.3m from tower base',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_6_3m_low': {
'standard_name': 'air_temperature',
'description': 'Air temperature 6.3m from tower base',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_6_3m_mean': {
'standard_name': 'air_temperature',
'description': 'Air temperature 6.3m from tower base',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'dewpoint_high': {
'standard_name': 'dewpoint_temperature',
'description': 'Calculated dewpoint temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'dewpoint_low': {
'standard_name': 'dewpoint_temperature',
'description': 'Calculated dewpoint temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'dewpoint_mean': {
'standard_name': 'dewpoint_temperature',
'description': 'Calculated dewpoint temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'rtd_shield_freq_high': {
'standard_name': '',
'description': '',
'units': '',
'_type': 'f',
'_shape': ('time',),
},
'rtd_shield_freq_low': {
'standard_name': '',
'description': '',
'units': '',
'_type': 'f',
'_shape': ('time',),
},
'rtd_shield_freq_mean': {
'standard_name': '',
'description': '',
'units': '',
'_type': 'f',
'_shape': ('time',),
},
'air_temp_high': {
'standard_name': 'air_temperature',
'description': 'Air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_low': {
'standard_name': 'air_temperature',
'description': 'Air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'air_temp_mean': {
'standard_name': 'air_temperature',
'description': 'Air temperature',
'units': 'degC',
'valid_min': -50,
'valid_max': 50,
'_type': 'f',
'_shape': ('time',),
},
'solar_flux_high': {
'standard_name': 'solar_flux',
'description': 'Solar flux',
'units': 'w*m^-2',
'valid_min': 0,
'valid_max': 3000,
'_type': 'f',
'_shape': ('time',),
},
'solar_flux_low': {
'standard_name': 'solar_flux',
'description': 'Solar flux',
'units': 'w*m^-2',
'valid_min': 0,
'valid_max': 3000,
'_type': 'f',
'_shape': ('time',),
},
'solar_flux_mean': {
'standard_name': 'solar_flux',
'description': 'Solar flux',
'units': 'w*m^-2',
'valid_min': 0,
'valid_max': 3000,
'_type': 'f',
'_shape': ('time',),
},
'precip_high': {
'standard_name': '',
'description': 'Precipitation',
'units': 'mm',
'valid_min': 0,
'valid_max': 254,
'_type': 'f',
'_shape': ('time',),
},
'precip_low': {
'standard_name': '',
'description': 'Precipitation',
'units': 'mm',
'valid_min': 0,
'valid_max': 254,
'_type': 'f',
'_shape': ('time',),
},
'precip_mean': {
'standard_name': '',
'description': 'Precipitation',
'units': 'mm',
'valid_min': 0,
'valid_max': 254,
'_type': 'f',
'_shape': ('time',),
},
'accum_precip_high': {
'standard_name': 'axxumulated_precipitation',
'description': 'Precipitation accumulated since 0Z',
'units': 'mm',
'valid_min': 0,
'valid_max': 254,
'_type': 'f',
'_shape': ('time',),
},
'accum_precip_low': {
'standard_name': 'axxumulated_precipitation',
'description': 'Precipitation accumulated since 0Z',
'units': 'mm',
'valid_min': 0,
'valid_max': 254,
'_type': 'f',
'_shape': ('time',),
},
'accum_precip_mean': {
'standard_name': 'axxumulated_precipitation',
'description': 'Precipitation accumulated since 0Z',
'units': 'mm',
'valid_min': 0,
'valid_max': 254,
'_type': 'f',
'_shape': ('time',),
},
'altimeter_high': {
'standard_name': '',
'description': '',
'units': 'inHg',
'_type': 'f',
'_shape': ('time',),
},
'altimeter_low': {
'standard_name': '',
'description': '',
'units': 'inHg',
'_type': 'f',
'_shape': ('time',),
},
'altimeter_mean': {
'standard_name': '',
'description': '',
'units': 'inHg',
'_type': 'f',
'_shape': ('time',),
},
'gust_high':
{
'standard_name': 'wind_speed_of_gust',
'descripiton': 'Wind gust over the previous 2 minutes',
'units': 'm/s',
'valid_min': 0,
'valid_max': '50',
'_type': 'f',
'_shape': ('time',),
},
'gust_low':
{
'standard_name': 'wind_speed_of_gust',
'descripiton': 'Wind gust over the previous 2 minutes',
'units': 'm/s',
'valid_min': 0,
'valid_max': '50',
'_type': 'f',
'_shape': ('time',),
},
'gust_mean':
{
'standard_name': 'wind_speed_of_gust',
'descripiton': 'Wind gust over the previous 2 minutes',
'units': 'm/s',
'valid_min': 0,
'valid_max': '50',
'_type': 'f',
'_shape': ('time',),
}
})
def _dt_convert(date):
return None
def createGiantNetCDF(start, end, directories, output, zlib, chunk_size):
return False
def writeDimensions(ncFile):
for name, size in BASIC_STATION['dimensions'].items():
ncFile.createDimension(name, size)
return ncFile
def createVariables(ncFile, firstStamp, chunksizes, zlib):
bts = firstStamp.strftime('%Y-%m-%d 00:00:00Z')
tu = 'seconds since ' + firstStamp.strftime('%Y-%m-%d 00:00:00Z')
for name, value in BASIC_STATION['globals'].items():
setattr(ncFile, name, value)
#generate history
ncFile.history = ' '.join(platform.uname()) + " " + os.path.basename(__file__)
for name, attrs in BASIC_STATION['variables'].items():
shape = attrs['_shape']
type_ = attrs['_type']
if name == 'max_len_station_name':
if (chunksizes) and chunksizes[0] > 32:
variable = ncFile.createVariable(name, type_, shape, fill_value=-999, zlib=zlib, chunksizes=[32])
else:
variable = ncFile.createVariable(name, type_, shape, fill_value=-999, zlib=zlib, chunksizes=chunksizes)
else:
variable = ncFile.createVariable(name, type_, shape, fill_value=-999, zlib=zlib, chunksizes=chunksizes)
if name == 'base_time':
variable.string = bts
if name == 'time':
variable.units = tu
for aname in attrs:
if aname.startswith('_'):
continue
setattr(variable, aname, attrs[aname])
for name, attrs in AOSS_VARS.items():
shape = attrs['_shape']
type_ = attrs['_type']
variable = ncFile.createVariable(name, type_, shape, fill_value=-999, zlib=zlib, chunksizes=chunksizes)
for aname in attrs:
if aname.startswith('_'):
continue
setattr(variable, aname, attrs[aname])
return ncFile
def main():
import argparse
#argparse description
parser = argparse.ArgumentParser(description="Convert level_00 aoss tower data to level_b1 monthly files")
#argparse verbosity info
parser.add_argument('-v', '--verbose', action="count", default=int(os.environ.get("VERBOSITY", 2)),
dest='verbosity',
help='each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG (default INFO)')
#argparse start and end times
parser.add_argument('-s', '--start-time', type=_dt_convert,
help="Start time of massive netcdf file, if only -s is given, a netcdf file for only that month is given" +
". Formats allowed: \'YYYY-MM-DD\'")
parser.add_argument('-e', '--end-time', type=_dt_convert, help='End time of massive netcdf file. Formats allowed:' +
"\'YYYY-MM-DD\'")
parser.add_argument('-cs', '--chunk-size', type=int, help='chunk Size for the netCDF file')
parser.add_argument('-z', '--zlib', action='store_true', help='compress netCDF file with zlib')
parser.add_argument("input_directories", nargs="+",
help="aoss_tower level_00 paths")
parser.add_argument('-o', '--output', required=True, nargs="+", help="filename pattern or filename. " +
"Should be along the lines of <filepath>/aoss_tower.YYYY-MM-DD.nc")
args = parser.parse_args()
levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
level=levels[min(3, args.verbosity)]
logging.basicConfig(level=level)
if(args.start_time and args.end_time):
result = createGiantNetCDF(args.start_time, args.end_time, args.input_files, args.output[0], args.zlib, args.chunk_size)
if(result == False):
raise IOError('An empty ASCII file was found')
elif(args.start_time):
end_time = args.start_time.replace(hour=23, minute=59, second=59)
result = createGiantNetCDF(args.start_time, end_time, args.input_directories, args.output[0], args.zlib, args.chunk_size)
if(result == False):
raise IOError('An empty ASCII file was found')
elif(args.end_time):
print('USAGE: start time must be specified when end time is specified')
else:
createMultiple(args.input_files, args.output, args.zlib, args.chunk_size)
if __name__ == "__main__":
# main()
ncFile = Dataset('aoss_tower.2013-06.SUMMARY.nc', 'w', format='NETCDF4_CLASSIC')
ncFile = writeDimensions(ncFile)
ncFile = createVariables(ncFile, dt(2003, 6, 1), [30], True)
ncFile.close()
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment