diff --git a/aosstower/level_b1/monthly/nc_monthly.py b/aosstower/level_b1/monthly/nc_monthly.py new file mode 100644 index 0000000000000000000000000000000000000000..4a9a0682b11f0a7de280dbd608711ab734de970e --- /dev/null +++ b/aosstower/level_b1/monthly/nc_monthly.py @@ -0,0 +1,945 @@ +import platform +from netCDF4 import Dataset +from datetime import datetime as dt +from collections import OrderedDict +import os + +# CF Metadata 1.6 compliant structure for NetCDF file. This is essentially +# a pythonified version of the CDL +BASIC_STATION = { + + 'globals': { + 'source': 'surface observation', + 'institution': 'UW SSEC', + 'featureType': 'timeSeries', + 'Conventions': 'ARM-1.2 CF-1.6', + 'data_level': 'b1', + 'datastream' : 'aoss.tower.y_nc.lb1.v00', + 'software_version': '00' + }, + + 'dimensions': { + 'max_len_station_name': 32, + 'time': None # Unlimited + }, + + 'variables': OrderedDict({ + 'base_time': { + 'string': '', # Filled in by creator + 'standard_name': 'time', + 'long_name': 'base time as unix timestamp', + 'units': 'seconds since 1970-01-01 00:00:00 0:00', + '_type': 'd', + '_shape': tuple(), + }, + 'time_offset': { + 'long_name': 'time offset from base_time', + 'standard_name': 'time', + 'units': '', # Filled in by creator + '_type': 'd', + '_shape': ('time',), + }, + 'time': { + 'long_name': 'time offset from epoch', + 'standard_name': 'time', + 'units': 'seconds since 1970-01-01 00:00:00 0:00', + '_type': 'd', + '_shape': ('time',), + }, + 'station_name': { + 'cf_role': 'timeseries_id', + 'long_name': 'station name', + '_type': 'S1', + '_shape': ('max_len_station_name',), + }, + 'lat': { + 'standard_name': 'latitude', + 'units': 'degrees_north', + 'valid_min': -90, + 'valid_max': 90, + '_type': 'f', + '_shape': tuple(), + }, + 'lon': { + 'standard_name': 'longitude', + 'units': 'degrees_east', + 'valid_min': -180, + 'valid_max': 180, + '_type': 'f', + '_shape': tuple(), + }, + 'alt': { + 'long_name': 'vertical distance', + 'standard_name': 'height', + 'units': 'm', + 'positive': 'up', + 'axis': 'Z', + '_type': 'f', + '_shape': tuple(), + } + }) +} + +AOSS_VARS = OrderedDict({ + 'box_temp_high': { + 'standard_name': 'air_temperature', + 'units': 'degC', + 'description': 'Auxillary Temperature', + '_type': 'f', + '_shape': ('time',), + }, + + 'box_temp_low': { + 'standard_name': 'air_temperature', + 'units': 'degC', + 'description': 'Auxillary Temperature', + '_type': 'f', + '_shape': ('time',), + }, + + 'box_temp_mean': { + 'standard_name': 'air_temperature', + 'units': 'degC', + 'description': 'Auxillary Temperature', + '_type': 'f', + '_shape': ('time',), + }, + + 'box_presure_high': { + 'standard_name': 'air_pressure', + 'description': 'Pressure inside the data logger enclosure', + 'units': 'hpa', + 'valid_min': 850, + 'valid_max': '1100', + '_type': 'f', + '_shape': ('time',), + }, + + 'box_presure_low': { + 'standard_name': 'air_pressure', + 'description': 'Pressure inside the data logger enclosure', + 'units': 'hpa', + 'valid_min': 850, + 'valid_max': '1100', + '_type': 'f', + '_shape': ('time',), + }, + + 'box_presure_mean': { + 'standard_name': 'air_pressure', + 'description': 'Pressure inside the data logger enclosure', + 'units': 'hpa', + 'valid_min': 850, + 'valid_max': '1100', + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_air_temp_period_high': { + 'standard_name': '', + 'description': '', + 'units': '1', + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_air_temp_period_low': { + 'standard_name': '', + 'description': '', + 'units': '1', + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_air_temp_period_mean': { + 'standard_name': '', + 'description': '', + 'units': '1', + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_pressure_period_high': { + 'standard_name': '', + 'description': '', + 'units': '1', + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_pressure_period_low': { + 'standard_name': '', + 'description': '', + 'units': '1', + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_pressure_period_mean': { + 'standard_name': '', + 'description': '', + 'units': '1', + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_air_temp_high': { + 'standard_name': 'air_temperature', + 'description': '', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_air_temp_low': { + 'standard_name': 'air_temperature', + 'description': '', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_air_temp_mean': { + 'standard_name': 'air_temperature', + 'description': '', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'pressure_high': { + 'standard_name': 'air_pressure', + 'description': 'Air pressure as measured from the PAROSCI pressure sensor', + 'units': 'hpa', + 'valid_min': 850, + 'valid_max': 1100, + '_type': 'f', + '_shape': ('time',), + }, + + 'pressure_low': { + 'standard_name': 'air_pressure', + 'description': 'Air pressure as measured from the PAROSCI pressure sensor', + 'units': 'hpa', + 'valid_min': 850, + 'valid_max': 1100, + '_type': 'f', + '_shape': ('time',), + }, + + 'pressure_mean': { + 'standard_name': 'air_pressure', + 'description': 'Air pressure as measured from the PAROSCI pressure sensor', + 'units': 'hpa', + 'valid_min': 850, + 'valid_max': 1100, + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_cal_sig_high': { + 'standard_name': '', + 'description': '', + 'units': '', + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_cal_sig_low': { + 'standard_name': '', + 'description': '', + 'units': '', + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_cal_sig_mean': { + 'standard_name': '', + 'description': '', + 'units': '', + '_type': 'f', + '_shape': ('time',), + }, + + 'box_rh_high': { + 'standard_name': 'relative humidity', + 'description': 'Relative humidity inside the data logger enclosure', + 'units': '%', + 'valid_min': 0, + 'valid_max': 100, + '_type': 'f', + '_shape': ('time',), + }, + + 'box_rh_low': { + 'standard_name': 'relative humidity', + 'description': 'Relative humidity inside the data logger enclosure', + 'units': '%', + 'valid_min': 0, + 'valid_max': 100, + '_type': 'f', + '_shape': ('time',), + }, + + 'box_rh_mean': { + 'standard_name': 'relative humidity', + 'description': 'Relative humidity inside the data logger enclosure', + 'units': '%', + 'valid_min': 0, + 'valid_max': 100, + '_type': 'f', + '_shape': ('time',), + }, + + 'box_air_temp_high': { + 'standard_name': 'air_temperature', + 'description': 'Air temperature inside the data logger enclosure', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'box_air_temp_low': { + 'standard_name': 'air_temperature', + 'description': 'Air temperature inside the data logger enclosure', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'box_air_temp_mean': { + 'standard_name': 'air_temperature', + 'description': 'Air temperature inside the data logger enclosure', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_2_high': { + 'standard_name': 'air_temperature', + 'description': 'Auxillary air temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_2_low': { + 'standard_name': 'air_temperature', + 'description': 'Auxillary air temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_2_mean': { + 'standard_name': 'air_temperature', + 'description': 'Auxillary air temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_3_high': { + 'standard_name': 'air_temperature', + 'description': 'Auxillary air temperature', + 'units' : 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_3_low': { + 'standard_name': 'air_temperature', + 'description': 'Auxillary air temperature', + 'units' : 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_3_mean': { + 'standard_name': 'air_temperature', + 'description': 'Auxillary air temperature', + 'units' : 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_4_high': { + 'standard_name': 'air_temperature', + 'description' : 'Auxillary air temperature', + 'units' : 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_4_low': { + 'standard_name': 'air_temperature', + 'description' : 'Auxillary air temperature', + 'units' : 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_4_mean': { + 'standard_name': 'air_temperature', + 'description' : 'Auxillary air temperature', + 'units' : 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_5_high': { + 'standard_name': 'air_temperature', + 'descripiton': 'Auxillary air temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_5_low': { + 'standard_name': 'air_temperature', + 'descripiton': 'Auxillary air temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), +}, + + 'air_temp_5_mean': { + 'standard_name': 'air_temperature', + 'descripiton': 'Auxillary air temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'wind_speed_high': { + 'standard_name': 'wind_speed', + 'description': 'wind_speed', + 'units': 'm*s^-1', + 'valid_min': 0, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'wind_speed_low': { + 'standard_name': 'wind_speed', + 'description': 'wind_speed', + 'units': 'm*s^-1', + 'valid_min': 0, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'wind_speed_mean': { + 'standard_name': 'wind_speed', + 'description': 'wind_speed', + 'units': 'm*s^-1', + 'valid_min': 0, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'wind_dir_high': { + 'standard_name': 'wind_direction', + 'description': 'wind_direction', + 'units': 'degrees', + 'valid_min': 0, + 'valid_max': 360, + '_type': 'f', + '_shape': ('time',), + }, + + 'wind_dir_low': { + 'standard_name': 'wind_direction', + 'description': 'wind_direction', + 'units': 'degrees', + 'valid_min': 0, + 'valid_max': 360, + '_type': 'f', + '_shape': ('time',), + }, + + 'wind_dir_mean': { + 'standard_name': 'wind_direction', + 'description': 'wind_direction', + 'units': 'degrees', + 'valid_min': 0, + 'valid_max': 360, + '_type': 'f', + '_shape': ('time',), + }, + + 'rh_shield_freq_high': { + 'standard_name': '', + 'descrption' : '', + 'units': 'hz', + '_type': 'f', + '_shape': ('time',), + }, + + 'rh_shield_freq_low': { + 'standard_name': '', + 'descrption' : '', + 'units': 'hz', + '_type': 'f', + '_shape': ('time',), + }, + + 'rh_shield_freq_mean': { + 'standard_name': '', + 'descrption' : '', + 'units': 'hz', + '_type': 'f', + '_shape': ('time',), + }, + + 'rh_high': { + 'standard_name': 'relative_humidity', + 'descripiton': 'Relative humidity', + 'units': '%', + 'valid_min': 0, + 'valid_max': 100, + '_type': 'f', + '_shape': ('time',), + }, + + 'rh_low': { + 'standard_name': 'relative_humidity', + 'descripiton': 'Relative humidity', + 'units': '%', + 'valid_min': 0, + 'valid_max': 100, + '_type': 'f', + '_shape': ('time',), + }, + + 'rh_mean': { + 'standard_name': 'relative_humidity', + 'descripiton': 'Relative humidity', + 'units': '%', + 'valid_min': 0, + 'valid_max': 100, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_6_3m_high': { + 'standard_name': 'air_temperature', + 'description': 'Air temperature 6.3m from tower base', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_6_3m_low': { + 'standard_name': 'air_temperature', + 'description': 'Air temperature 6.3m from tower base', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_6_3m_mean': { + 'standard_name': 'air_temperature', + 'description': 'Air temperature 6.3m from tower base', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'dewpoint_high': { + 'standard_name': 'dewpoint_temperature', + 'description': 'Calculated dewpoint temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'dewpoint_low': { + 'standard_name': 'dewpoint_temperature', + 'description': 'Calculated dewpoint temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'dewpoint_mean': { + 'standard_name': 'dewpoint_temperature', + 'description': 'Calculated dewpoint temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'rtd_shield_freq_high': { + 'standard_name': '', + 'description': '', + 'units': '', + '_type': 'f', + '_shape': ('time',), + }, + + 'rtd_shield_freq_low': { + 'standard_name': '', + 'description': '', + 'units': '', + '_type': 'f', + '_shape': ('time',), + }, + + 'rtd_shield_freq_mean': { + 'standard_name': '', + 'description': '', + 'units': '', + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_high': { + 'standard_name': 'air_temperature', + 'description': 'Air temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_low': { + 'standard_name': 'air_temperature', + 'description': 'Air temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_mean': { + 'standard_name': 'air_temperature', + 'description': 'Air temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'solar_flux_high': { + 'standard_name': 'solar_flux', + 'description': 'Solar flux', + 'units': 'w*m^-2', + 'valid_min': 0, + 'valid_max': 3000, + '_type': 'f', + '_shape': ('time',), + }, + + 'solar_flux_low': { + 'standard_name': 'solar_flux', + 'description': 'Solar flux', + 'units': 'w*m^-2', + 'valid_min': 0, + 'valid_max': 3000, + '_type': 'f', + '_shape': ('time',), + }, + + 'solar_flux_mean': { + 'standard_name': 'solar_flux', + 'description': 'Solar flux', + 'units': 'w*m^-2', + 'valid_min': 0, + 'valid_max': 3000, + '_type': 'f', + '_shape': ('time',), + }, + + 'precip_high': { + 'standard_name': '', + 'description': 'Precipitation', + 'units': 'mm', + 'valid_min': 0, + 'valid_max': 254, + '_type': 'f', + '_shape': ('time',), + }, + + 'precip_low': { + 'standard_name': '', + 'description': 'Precipitation', + 'units': 'mm', + 'valid_min': 0, + 'valid_max': 254, + '_type': 'f', + '_shape': ('time',), + }, + + 'precip_mean': { + 'standard_name': '', + 'description': 'Precipitation', + 'units': 'mm', + 'valid_min': 0, + 'valid_max': 254, + '_type': 'f', + '_shape': ('time',), + }, + + 'accum_precip_high': { + 'standard_name': 'axxumulated_precipitation', + 'description': 'Precipitation accumulated since 0Z', + 'units': 'mm', + 'valid_min': 0, + 'valid_max': 254, + '_type': 'f', + '_shape': ('time',), + }, + + 'accum_precip_low': { + 'standard_name': 'axxumulated_precipitation', + 'description': 'Precipitation accumulated since 0Z', + 'units': 'mm', + 'valid_min': 0, + 'valid_max': 254, + '_type': 'f', + '_shape': ('time',), + }, + + 'accum_precip_mean': { + 'standard_name': 'axxumulated_precipitation', + 'description': 'Precipitation accumulated since 0Z', + 'units': 'mm', + 'valid_min': 0, + 'valid_max': 254, + '_type': 'f', + '_shape': ('time',), + }, + + 'altimeter_high': { + 'standard_name': '', + 'description': '', + 'units': 'inHg', + '_type': 'f', + '_shape': ('time',), + }, + + 'altimeter_low': { + 'standard_name': '', + 'description': '', + 'units': 'inHg', + '_type': 'f', + '_shape': ('time',), + }, + + 'altimeter_mean': { + 'standard_name': '', + 'description': '', + 'units': 'inHg', + '_type': 'f', + '_shape': ('time',), + }, + + 'gust_high': + { + 'standard_name': 'wind_speed_of_gust', + 'descripiton': 'Wind gust over the previous 2 minutes', + 'units': 'm/s', + 'valid_min': 0, + 'valid_max': '50', + '_type': 'f', + '_shape': ('time',), + }, + + 'gust_low': + { + 'standard_name': 'wind_speed_of_gust', + 'descripiton': 'Wind gust over the previous 2 minutes', + 'units': 'm/s', + 'valid_min': 0, + 'valid_max': '50', + '_type': 'f', + '_shape': ('time',), + }, + + 'gust_mean': + { + 'standard_name': 'wind_speed_of_gust', + 'descripiton': 'Wind gust over the previous 2 minutes', + 'units': 'm/s', + 'valid_min': 0, + 'valid_max': '50', + '_type': 'f', + '_shape': ('time',), + } +}) + +def _dt_convert(date): + return None + +def createGiantNetCDF(start, end, directories, output, zlib, chunk_size): + return False + +def writeDimensions(ncFile): + for name, size in BASIC_STATION['dimensions'].items(): + ncFile.createDimension(name, size) + + return ncFile + +def createVariables(ncFile, firstStamp, chunksizes, zlib): + bts = firstStamp.strftime('%Y-%m-%d 00:00:00Z') + + tu = 'seconds since ' + firstStamp.strftime('%Y-%m-%d 00:00:00Z') + + for name, value in BASIC_STATION['globals'].items(): + setattr(ncFile, name, value) + + #generate history + ncFile.history = ' '.join(platform.uname()) + " " + os.path.basename(__file__) + + for name, attrs in BASIC_STATION['variables'].items(): + shape = attrs['_shape'] + type_ = attrs['_type'] + + if name == 'max_len_station_name': + if (chunksizes) and chunksizes[0] > 32: + variable = ncFile.createVariable(name, type_, shape, fill_value=-999, zlib=zlib, chunksizes=[32]) + + else: + variable = ncFile.createVariable(name, type_, shape, fill_value=-999, zlib=zlib, chunksizes=chunksizes) + + else: + variable = ncFile.createVariable(name, type_, shape, fill_value=-999, zlib=zlib, chunksizes=chunksizes) + + if name == 'base_time': + variable.string = bts + + if name == 'time': + variable.units = tu + + for aname in attrs: + if aname.startswith('_'): + continue + + setattr(variable, aname, attrs[aname]) + + for name, attrs in AOSS_VARS.items(): + shape = attrs['_shape'] + type_ = attrs['_type'] + + variable = ncFile.createVariable(name, type_, shape, fill_value=-999, zlib=zlib, chunksizes=chunksizes) + + for aname in attrs: + if aname.startswith('_'): + continue + + setattr(variable, aname, attrs[aname]) + + return ncFile + + +def main(): + import argparse + + #argparse description + parser = argparse.ArgumentParser(description="Convert level_00 aoss tower data to level_b1 monthly files") + + #argparse verbosity info + parser.add_argument('-v', '--verbose', action="count", default=int(os.environ.get("VERBOSITY", 2)), + dest='verbosity', + help='each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG (default INFO)') + + #argparse start and end times + parser.add_argument('-s', '--start-time', type=_dt_convert, + help="Start time of massive netcdf file, if only -s is given, a netcdf file for only that month is given" + + ". Formats allowed: \'YYYY-MM-DD\'") + parser.add_argument('-e', '--end-time', type=_dt_convert, help='End time of massive netcdf file. Formats allowed:' + + "\'YYYY-MM-DD\'") + parser.add_argument('-cs', '--chunk-size', type=int, help='chunk Size for the netCDF file') + parser.add_argument('-z', '--zlib', action='store_true', help='compress netCDF file with zlib') + + parser.add_argument("input_directories", nargs="+", + help="aoss_tower level_00 paths") + + parser.add_argument('-o', '--output', required=True, nargs="+", help="filename pattern or filename. " + + "Should be along the lines of <filepath>/aoss_tower.YYYY-MM-DD.nc") + args = parser.parse_args() + + levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG] + level=levels[min(3, args.verbosity)] + logging.basicConfig(level=level) + + + if(args.start_time and args.end_time): + result = createGiantNetCDF(args.start_time, args.end_time, args.input_files, args.output[0], args.zlib, args.chunk_size) + if(result == False): + raise IOError('An empty ASCII file was found') + + elif(args.start_time): + end_time = args.start_time.replace(hour=23, minute=59, second=59) + result = createGiantNetCDF(args.start_time, end_time, args.input_directories, args.output[0], args.zlib, args.chunk_size) + if(result == False): + raise IOError('An empty ASCII file was found') + + elif(args.end_time): + print('USAGE: start time must be specified when end time is specified') + + else: + createMultiple(args.input_files, args.output, args.zlib, args.chunk_size) + +if __name__ == "__main__": + # main() + ncFile = Dataset('aoss_tower.2013-06.SUMMARY.nc', 'w', format='NETCDF4_CLASSIC') + ncFile = writeDimensions(ncFile) + ncFile = createVariables(ncFile, dt(2003, 6, 1), [30], True) + + ncFile.close()