From 3dd305e205e0817e49561fc0615f1edb11a139e3 Mon Sep 17 00:00:00 2001 From: mwestphall <westphall@wisc.edu> Date: Thu, 12 Jan 2017 15:29:47 -0600 Subject: [PATCH] added --interval command line arg which adds function of monthly to daily --- aosstower/level_b1/__init__.py | 0 aosstower/level_b1/daily/autoRun.py | 3 - aosstower/level_b1/daily/avg_database.py | 838 +++++++++++++++++++++++ aosstower/level_b1/daily/bundle.py | 139 ---- aosstower/level_b1/daily/bundle.pyc | Bin 3992 -> 0 bytes aosstower/level_b1/daily/foo.txt | 163 +++++ aosstower/level_b1/daily/nc.py | 78 ++- aosstower/level_b1/daily/nc.pyc | Bin 8817 -> 0 bytes aosstower/level_b1/monthly/nc_monthly.py | 66 +- 9 files changed, 1112 insertions(+), 175 deletions(-) create mode 100644 aosstower/level_b1/__init__.py delete mode 100755 aosstower/level_b1/daily/autoRun.py create mode 100644 aosstower/level_b1/daily/avg_database.py delete mode 100644 aosstower/level_b1/daily/bundle.py delete mode 100644 aosstower/level_b1/daily/bundle.pyc create mode 100644 aosstower/level_b1/daily/foo.txt delete mode 100644 aosstower/level_b1/daily/nc.pyc diff --git a/aosstower/level_b1/__init__.py b/aosstower/level_b1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aosstower/level_b1/daily/autoRun.py b/aosstower/level_b1/daily/autoRun.py deleted file mode 100755 index 0d15a20..0000000 --- a/aosstower/level_b1/daily/autoRun.py +++ /dev/null @@ -1,3 +0,0 @@ -from bundle import createYesterdayFile - -createYesterdayFile() diff --git a/aosstower/level_b1/daily/avg_database.py b/aosstower/level_b1/daily/avg_database.py new file mode 100644 index 0000000..43b78fc --- /dev/null +++ b/aosstower/level_b1/daily/avg_database.py @@ -0,0 +1,838 @@ +# CF Metadata 1.6 compliant structure for NetCDF file. This is essentially +# a pythonified version of the CDL +from collections import OrderedDict +import aosstower.schema as schema +BASIC_STATION = { + + 'globals': { + 'source': 'surface observation', + 'institution': 'UW SSEC', + 'featureType': 'timeSeries', + 'Conventions': 'ARM-1.2 CF-1.6', + 'data_level': 'b1', + 'datastream' : 'aoss.tower.nc-1mo-1d.lb1.v00', + 'software_version': '00' + }, + + 'dimensions': { + 'max_len_station_name': 32, + 'time': None # Unlimited + }, + + 'variables': OrderedDict({ + 'base_time': { + 'string': '', # Filled in by creator + 'standard_name': 'time', + 'long_name': 'base time as unix timestamp', + 'units': 'seconds since 1970-01-01 00:00:00 0:00', + '_type': 'd', + '_shape': tuple(), + }, + 'time_offset': { + 'long_name': 'time offset from base_time', + 'standard_name': 'time', + 'units': '', # Filled in by creator + '_type': 'd', + '_shape': ('time',), + }, + 'time': { + 'long_name': 'time offset from epoch', + 'standard_name': 'time', + 'units': 'seconds since 1970-01-01 00:00:00 0:00', + '_type': 'd', + '_shape': ('time',), + }, + 'station_name': { + 'cf_role': 'timeseries_id', + 'long_name': 'station name', + '_type': 'S1', + '_shape': ('max_len_station_name',), + }, + 'lat': { + 'standard_name': 'latitude', + 'units': 'degrees_north', + 'valid_min': -90, + 'valid_max': 90, + '_type': 'f', + '_shape': tuple(), + }, + 'lon': { + 'standard_name': 'longitude', + 'units': 'degrees_east', + 'valid_min': -180, + 'valid_max': 180, + '_type': 'f', + '_shape': tuple(), + }, + 'alt': { + 'long_name': 'vertical distance', + 'standard_name': 'height', + 'units': 'm', + 'positive': 'up', + 'axis': 'Z', + '_type': 'f', + '_shape': tuple(), + } + }) +} + +AOSS_VARS = OrderedDict({ + 'box_temp_high': { + 'standard_name': 'air_temperature', + 'units': 'degC', + 'description': 'Auxillary Temperature', + '_type': 'f', + '_shape': ('time',), + }, + + 'box_temp_low': { + 'standard_name': 'air_temperature', + 'units': 'degC', + 'description': 'Auxillary Temperature', + '_type': 'f', + '_shape': ('time',), + }, + + 'box_temp_mean': { + 'standard_name': 'air_temperature', + 'units': 'degC', + 'description': 'Auxillary Temperature', + '_type': 'f', + '_shape': ('time',), + }, + + 'box_presure_high': { + 'standard_name': 'air_pressure', + 'description': 'Pressure inside the data logger enclosure', + 'units': 'hpa', + 'valid_min': 850, + 'valid_max': '1100', + '_type': 'f', + '_shape': ('time',), + }, + + 'box_presure_low': { + 'standard_name': 'air_pressure', + 'description': 'Pressure inside the data logger enclosure', + 'units': 'hpa', + 'valid_min': 850, + 'valid_max': '1100', + '_type': 'f', + '_shape': ('time',), + }, + + 'box_presure_mean': { + 'standard_name': 'air_pressure', + 'description': 'Pressure inside the data logger enclosure', + 'units': 'hpa', + 'valid_min': 850, + 'valid_max': '1100', + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_air_temp_period_high': { + 'standard_name': '', + 'description': '', + 'units': '1', + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_air_temp_period_low': { + 'standard_name': '', + 'description': '', + 'units': '1', + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_air_temp_period_mean': { + 'standard_name': '', + 'description': '', + 'units': '1', + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_pressure_period_high': { + 'standard_name': '', + 'description': '', + 'units': '1', + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_pressure_period_low': { + 'standard_name': '', + 'description': '', + 'units': '1', + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_pressure_period_mean': { + 'standard_name': '', + 'description': '', + 'units': '1', + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_air_temp_high': { + 'standard_name': 'air_temperature', + 'description': '', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_air_temp_low': { + 'standard_name': 'air_temperature', + 'description': '', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_air_temp_mean': { + 'standard_name': 'air_temperature', + 'description': '', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'pressure_high': { + 'standard_name': 'air_pressure', + 'description': 'Air pressure as measured from the PAROSCI pressure sensor', + 'units': 'hpa', + 'valid_min': 850, + 'valid_max': 1100, + '_type': 'f', + '_shape': ('time',), + }, + + 'pressure_low': { + 'standard_name': 'air_pressure', + 'description': 'Air pressure as measured from the PAROSCI pressure sensor', + 'units': 'hpa', + 'valid_min': 850, + 'valid_max': 1100, + '_type': 'f', + '_shape': ('time',), + }, + + 'pressure_mean': { + 'standard_name': 'air_pressure', + 'description': 'Air pressure as measured from the PAROSCI pressure sensor', + 'units': 'hpa', + 'valid_min': 850, + 'valid_max': 1100, + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_cal_sig_high': { + 'standard_name': '', + 'description': '', + 'units': '', + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_cal_sig_low': { + 'standard_name': '', + 'description': '', + 'units': '', + '_type': 'f', + '_shape': ('time',), + }, + + 'paro_cal_sig_mean': { + 'standard_name': '', + 'description': '', + 'units': '', + '_type': 'f', + '_shape': ('time',), + }, + + 'box_rh_high': { + 'standard_name': 'relative humidity', + 'description': 'Relative humidity inside the data logger enclosure', + 'units': '%', + 'valid_min': 0, + 'valid_max': 100, + '_type': 'f', + '_shape': ('time',), + }, + + 'box_rh_low': { + 'standard_name': 'relative humidity', + 'description': 'Relative humidity inside the data logger enclosure', + 'units': '%', + 'valid_min': 0, + 'valid_max': 100, + '_type': 'f', + '_shape': ('time',), + }, + + 'box_rh_mean': { + 'standard_name': 'relative humidity', + 'description': 'Relative humidity inside the data logger enclosure', + 'units': '%', + 'valid_min': 0, + 'valid_max': 100, + '_type': 'f', + '_shape': ('time',), + }, + + 'box_air_temp_high': { + 'standard_name': 'air_temperature', + 'description': 'Air temperature inside the data logger enclosure', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'box_air_temp_low': { + 'standard_name': 'air_temperature', + 'description': 'Air temperature inside the data logger enclosure', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'box_air_temp_mean': { + 'standard_name': 'air_temperature', + 'description': 'Air temperature inside the data logger enclosure', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_2_high': { + 'standard_name': 'air_temperature', + 'description': 'Auxillary air temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_2_low': { + 'standard_name': 'air_temperature', + 'description': 'Auxillary air temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_2_mean': { + 'standard_name': 'air_temperature', + 'description': 'Auxillary air temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_3_high': { + 'standard_name': 'air_temperature', + 'description': 'Auxillary air temperature', + 'units' : 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_3_low': { + 'standard_name': 'air_temperature', + 'description': 'Auxillary air temperature', + 'units' : 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_3_mean': { + 'standard_name': 'air_temperature', + 'description': 'Auxillary air temperature', + 'units' : 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_4_high': { + 'standard_name': 'air_temperature', + 'description' : 'Auxillary air temperature', + 'units' : 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_4_low': { + 'standard_name': 'air_temperature', + 'description' : 'Auxillary air temperature', + 'units' : 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_4_mean': { + 'standard_name': 'air_temperature', + 'description' : 'Auxillary air temperature', + 'units' : 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_5_high': { + 'standard_name': 'air_temperature', + 'descripiton': 'Auxillary air temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_5_low': { + 'standard_name': 'air_temperature', + 'descripiton': 'Auxillary air temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), +}, + + 'air_temp_5_mean': { + 'standard_name': 'air_temperature', + 'descripiton': 'Auxillary air temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'wind_speed_high': { + 'standard_name': 'wind_speed', + 'description': 'wind_speed', + 'units': 'm*s^-1', + 'valid_min': 0, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'wind_speed_low': { + 'standard_name': 'wind_speed', + 'description': 'wind_speed', + 'units': 'm*s^-1', + 'valid_min': 0, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'wind_speed_mean': { + 'standard_name': 'wind_speed', + 'description': 'wind_speed', + 'units': 'm*s^-1', + 'valid_min': 0, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'wind_dir_high': { + 'standard_name': 'wind_direction', + 'description': 'wind_direction', + 'units': 'degrees', + 'valid_min': 0, + 'valid_max': 360, + '_type': 'f', + '_shape': ('time',), + }, + + 'wind_dir_low': { + 'standard_name': 'wind_direction', + 'description': 'wind_direction', + 'units': 'degrees', + 'valid_min': 0, + 'valid_max': 360, + '_type': 'f', + '_shape': ('time',), + }, + + 'wind_dir_mean': { + 'standard_name': 'wind_direction', + 'description': 'wind_direction', + 'units': 'degrees', + 'valid_min': 0, + 'valid_max': 360, + '_type': 'f', + '_shape': ('time',), + }, + + 'rh_shield_freq_high': { + 'standard_name': '', + 'descrption' : '', + 'units': 'hz', + '_type': 'f', + '_shape': ('time',), + }, + + 'rh_shield_freq_low': { + 'standard_name': '', + 'descrption' : '', + 'units': 'hz', + '_type': 'f', + '_shape': ('time',), + }, + + 'rh_shield_freq_mean': { + 'standard_name': '', + 'descrption' : '', + 'units': 'hz', + '_type': 'f', + '_shape': ('time',), + }, + + 'rh_high': { + 'standard_name': 'relative_humidity', + 'descripiton': 'Relative humidity', + 'units': '%', + 'valid_min': 0, + 'valid_max': 100, + '_type': 'f', + '_shape': ('time',), + }, + + 'rh_low': { + 'standard_name': 'relative_humidity', + 'descripiton': 'Relative humidity', + 'units': '%', + 'valid_min': 0, + 'valid_max': 100, + '_type': 'f', + '_shape': ('time',), + }, + + 'rh_mean': { + 'standard_name': 'relative_humidity', + 'descripiton': 'Relative humidity', + 'units': '%', + 'valid_min': 0, + 'valid_max': 100, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_6_3m_high': { + 'standard_name': 'air_temperature', + 'description': 'Air temperature 6.3m from tower base', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_6_3m_low': { + 'standard_name': 'air_temperature', + 'description': 'Air temperature 6.3m from tower base', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_6_3m_mean': { + 'standard_name': 'air_temperature', + 'description': 'Air temperature 6.3m from tower base', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'dewpoint_high': { + 'standard_name': 'dewpoint_temperature', + 'description': 'Calculated dewpoint temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'dewpoint_low': { + 'standard_name': 'dewpoint_temperature', + 'description': 'Calculated dewpoint temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'dewpoint_mean': { + 'standard_name': 'dewpoint_temperature', + 'description': 'Calculated dewpoint temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'rtd_shield_freq_high': { + 'standard_name': '', + 'description': '', + 'units': '', + '_type': 'f', + '_shape': ('time',), + }, + + 'rtd_shield_freq_low': { + 'standard_name': '', + 'description': '', + 'units': '', + '_type': 'f', + '_shape': ('time',), + }, + + 'rtd_shield_freq_mean': { + 'standard_name': '', + 'description': '', + 'units': '', + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_high': { + 'standard_name': 'air_temperature', + 'description': 'Air temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_low': { + 'standard_name': 'air_temperature', + 'description': 'Air temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'air_temp_mean': { + 'standard_name': 'air_temperature', + 'description': 'Air temperature', + 'units': 'degC', + 'valid_min': -50, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'solar_flux_high': { + 'standard_name': 'solar_flux', + 'description': 'Solar flux', + 'units': 'w*m^-2', + 'valid_min': 0, + 'valid_max': 3000, + '_type': 'f', + '_shape': ('time',), + }, + + 'solar_flux_low': { + 'standard_name': 'solar_flux', + 'description': 'Solar flux', + 'units': 'w*m^-2', + 'valid_min': 0, + 'valid_max': 3000, + '_type': 'f', + '_shape': ('time',), + }, + + 'solar_flux_mean': { + 'standard_name': 'solar_flux', + 'description': 'Solar flux', + 'units': 'w*m^-2', + 'valid_min': 0, + 'valid_max': 3000, + '_type': 'f', + '_shape': ('time',), + }, + + 'precip_high': { + 'standard_name': '', + 'description': 'Precipitation', + 'units': 'mm', + 'valid_min': 0, + 'valid_max': 254, + '_type': 'f', + '_shape': ('time',), + }, + + 'precip_low': { + 'standard_name': '', + 'description': 'Precipitation', + 'units': 'mm', + 'valid_min': 0, + 'valid_max': 254, + '_type': 'f', + '_shape': ('time',), + }, + + 'precip_mean': { + 'standard_name': '', + 'description': 'Precipitation', + 'units': 'mm', + 'valid_min': 0, + 'valid_max': 254, + '_type': 'f', + '_shape': ('time',), + }, + + 'accum_precip_high': { + 'standard_name': 'axxumulated_precipitation', + 'description': 'Precipitation accumulated since 0Z', + 'units': 'mm', + 'valid_min': 0, + 'valid_max': 254, + '_type': 'f', + '_shape': ('time',), + }, + + 'accum_precip_low': { + 'standard_name': 'axxumulated_precipitation', + 'description': 'Precipitation accumulated since 0Z', + 'units': 'mm', + 'valid_min': 0, + 'valid_max': 254, + '_type': 'f', + '_shape': ('time',), + }, + + 'accum_precip_mean': { + 'standard_name': 'axxumulated_precipitation', + 'description': 'Precipitation accumulated since 0Z', + 'units': 'mm', + 'valid_min': 0, + 'valid_max': 254, + '_type': 'f', + '_shape': ('time',), + }, + + 'altimeter_high': { + 'standard_name': '', + 'description': '', + 'units': 'inHg', + '_type': 'f', + '_shape': ('time',), + }, + + 'altimeter_low': { + 'standard_name': '', + 'description': '', + 'units': 'inHg', + '_type': 'f', + '_shape': ('time',), + }, + + 'altimeter_mean': { + 'standard_name': '', + 'description': '', + 'units': 'inHg', + '_type': 'f', + '_shape': ('time',), + }, + + 'gust_high': + { + 'standard_name': 'wind_speed_of_gust', + 'descripiton': 'Wind gust over the previous 2 minutes', + 'units': 'm/s', + 'valid_min': 0, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'gust_low': + { + 'standard_name': 'wind_speed_of_gust', + 'descripiton': 'Wind gust over the previous 2 minutes', + 'units': 'm/s', + 'valid_min': 0, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + }, + + 'peak_wind': + { + 'standard_name': 'wind_speed', + 'description': 'fastest 5-second wind_speed', + 'units': 'm/s', + 'valid_min': 0, + 'valid_max': 50, + '_type': 'f', + '_shape': ('time',), + } +}) + +def make_Var_db(aoss_vars): + var_dict = {} + for key in aoss_vars: + #there's a typo in some var names + entry = aoss_vars[key] + key = key.replace('presure','pressure') + var_dict[key] = schema.Var( + entry.get('_type','f'), + entry.get('standard_name',''), + key, + entry.get('description',''), + entry.get('units',''), + entry.get('valid_min',''), + entry.get('valid_max',''), + ) + return var_dict +AOSS_VARS = make_Var_db(AOSS_VARS) diff --git a/aosstower/level_b1/daily/bundle.py b/aosstower/level_b1/daily/bundle.py deleted file mode 100644 index 66729cd..0000000 --- a/aosstower/level_b1/daily/bundle.py +++ /dev/null @@ -1,139 +0,0 @@ -from nc import getData, createGiantNetCDF, writeDimensions, createVariables, writeVars, minuteAverages -from datetime import datetime as dt -from datetime import timedelta as delta -import pandas as pd -import time -from netCDF4 import Dataset -import os - -def writeBack(frame, filename, cur_dt): - #get start - startString = str(frame.index[0]) - startObj = dt.strptime(startString.split(" ")[0], '%Y-%m-%d') - createDirectory(startObj) - - #get end - endObj = startObj.replace(hour=23, minute=59, second=59, microsecond=59) - - #input file - inputFiles = [] - inputFiles.append(filename) - inputFiles.append(createFilename(cur_dt - delta(days=1))) - - #output filename - outputFilename = 'aoss_tower.' + startString.split(" ")[0] + '.day.nc' - - #create file - createGiantNetCDF(startObj, endObj, inputFiles, outputFilename, True, None) - -def createDirectory(stamp): - if os.path.exists("/mnt/inst-data/regen_netcdf/aoss/tower/" + stamp.strftime('%Y/%m')): - os.chdir("/mnt/inst-data/regen_netcdf/aoss/tower/" + stamp.strftime('%Y/%m')) - - else: - os.makedirs("/mnt/inst-data/regen_netcdf/aoss/tower/" + stamp.strftime('%Y/%m')) - os.chdir("/mnt/inst-data/regen_netcdf/aoss/tower/" + stamp.strftime('%Y/%m')) - - -def createFilename(date): - filename = '/mnt/inst-data/aoss-tower/' - filename += date.strftime('%Y/%m/') - filename += 'rig_tower.' + date.strftime('%Y-%m-%d') + '.ascii' - - return filename - -def createNCFile(frame, filename): - firstStamp = dt.strptime(str(list(frame.index)[0]), '%Y-%m-%d %H:%M:%S') - - createDirectory(firstStamp) - - outputName = 'aoss_tower.' + firstStamp.strftime('%Y-%m-%d') + '.day.nc' - - ncFile = Dataset(outputName, 'w', format='NETCDF4_CLASSIC') - - ncFile = writeDimensions(ncFile) - - ncFile = createVariables(ncFile, firstStamp, [len(list(frame.index))], True) - - ncFile.inputFiles = ', '.join([filename]) - - ncFile = writeVars(ncFile, frame) - - ncFile.close() - -def create_files(start_date, end_date): - # for each file's name based on date - does not rely on what dates are - # in each file so should be similar code for buoy - # get file, discover which full dates excluding the start are in it - # from there we create the netcdf files - # for the start, we always recreate the netcdf using the previous file - # ignore the last unfulfilled date - - cur_dt = start_date - - for day in range((end_date - start_date).days + 1): - filename = createFilename(cur_dt) - print('grabbing ' + filename) - - df = getData([filename]) - - if(df.empty): - cur_dt += delta(days = 1) - continue - - DFList = [] - - for group in df.groupby(df.index.day): - DFList.append(group[1]) - - - if(len(DFList) <= 1): - frame = DFList[0] - - frame = minuteAverages(frame) - - createNCFile(frame, filename) - - elif(len(DFList) == 2): - if(len(list(DFList[0].index)) > len(list(DFList[1].index))): - frame = DFList[0] - - frame = minuteAverages(frame) - - createNCFile(frame, filename) - - #get second frame - frame = DFList[1] - - writeBack(frame, filename, cur_dt) - - else: - frame = DFList[1] - - frame = minuteAverages(frame) - - createNCFile(frame, filename) - - #get second frame - frame = DFList[0] - - writeBack(frame, filename, cur_dt) - - else: - print(len(DFList)) - - for frame in DFList: - print(len(list(frame.index))) - print(list(frame.index)[0]) - exit(0) - - - - #increment day - cur_dt += delta(days = 1) - -def createYesterdayFile(): - create_files(dt.today() - delta(days=1), dt.today() - delta(days=1)) - -#createYesterdayFile() -create_files(dt(2009,5,2), dt(2009,5,2))#dt.today() - delta(days=1)) diff --git a/aosstower/level_b1/daily/bundle.pyc b/aosstower/level_b1/daily/bundle.pyc deleted file mode 100644 index 99b5245907dca7496a03515d81c8f008c7504070..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3992 zcmcgvO>Y~?5v`sfDUlW}tz}!@_3lPNR#_fa;vz8Oz%i2bO0gWk;Q}Kgh6xzLXr@IE zMb1z&tylsiB!F_wCBGr3{EFlr<dU3n%q2e{m)w%~s!2)fb&g(1Y<BfmRad=wT?+r* zZvCPC$LEPme;WAy0-x%iXj1$c8HtRI^o@)I=?5|nr60<;CjFX>>(Z}_;krOZ4H-A2 zKj-C8<$38hy<C$~OZwOJO!OC|zo7lPjM_53F8%8l>lUTIC}~4}E7D(*bWS_(Njk5c z8<IA)vn**#J2xf0rkz`2Z;G9h6i2ruy)Ne=#o=u^7qLxAaqJTM=FWKlZ8h2HyoYOM z|HV(Ij-H!CGq7$ecCq7BQrg(r?JUk+&$`X6$8Hg$uglEYt!!-bDl76z$Nb7)#$^`o zjcnzbvvnA%ItG%PN0Z_>hn;?eizuab<l>$mqY2bsS(k+9a~Icb;#2(<jT1Q)85+js z{c|H{p}0VXp`3;?tT7T8<XjpG?s--dSC`WoM-9G)4Z+RRIz0^E!mDiIoH=-=c2<{x z$XP>98#0`eQ{%U(fD4=kf}6qb@VNTX&%@zwDqleaR?g<cwW#|P7T`>XiU8M^bB~jq z&UIX=m!T0yFRfO*@Y;*@wek8|>S_>P6-Q;Z!UaU<ILnWm&3-(SE1MK~igP)Yl*Lr$ z#xy>uG9FrSHZH1a*A=gAxlz%v8+oG29fiu3@Di|)r}h}`#Ac@f!^DIk(%EV<%G{N3 zZ)K%TTv47Va7sHF#feob<H^LrDqhYb<8d~!c|1Kw;~gm|Xy>!?$m&gdMeY?njz^X4 z&=ab6zvNDe`?F){isY!=g_*ou<>J!qxH8KJQ?ns|wKr5_U<tb|tjqGrk)th@+EyI7 zsZ85CSRmR|U!m81Rg7)-U=SDG%_6m3s4I6TWiiB^t8Sib+uWAGbN3NE`;3n5(#PuJ z?#TYij&|eEyL(4@I<gy+lgpI&GENQ%>FNd*2FvC<rWJk?HcZ>BhIhk+j^T&CQP>0W zggr1n3<z!FKb#_c1-LFB>NH^e%{i0b(@6mSEt5R;?b}iTJrEkb>Ql7caqhYql6@U1 z9Cu4Qu=y^cm8AQ6>#o0cMG&pM=&p@BjF2L!s5H2f*uC=MIL<0p&Cs=ffv)6Lnw2^{ zjt?yQQ{Lc_i0^pvZsQXzL(fOlMW6|S@=^GK(O@tiE`sqR2yo>Yyy+bZQ90+n9|G+# zJ{VQbSpn6oOs96#DZT0g9Px%jiJbMR59vAeVYeb?%WN>qTUs{bhKkEJ;ws59WzL9U z>d;hmH<>p2kVE*qJ5)!var*c1nR=~Zmd)KDp&5)DZBz*V2^|k%U;>mw7pZ^XO0qaz z2(rc;s8Zc&=tvN|@-<edBT4q4Wq^Oi-sgcK#e<vmIQE-Lxj3rzHBC5UV93DdHNFbq z{2_Q-2Qh{1yKMn0kgjF=a-ct41k>x=ywhPcJXcqI9cPhPU3>Ci?H3Q$cD}*0UyB!$ z38bqu|5`I)zbMDCC)D20p5b}Dx4ZfD(az4#H!C{uv(?TLz9Ma?Fe0C7bVo>`h;Ol? zRzn~0<&3y5q9&zvaBg&m0?|7xvOM}O=P4<%CL_ET`t(P4IYGQAPwpf8z^6z$6+KV( z$ij?ho{&P{EwJ1I6#2KKw@G#DkUe9XwV)NO1h;~P;8V1V9&TjR3%LCY{D~+6Psjf- zz@1OkJHBclWp2Ivv+<RKM;ieVF!n3kPYBPNVg--_;fC@)>H@ss!@87*S_Jqa0<XWA z`%rTDvE-<nv~oD!(4{~rHq6Vg$r57g<%*yhU{!&F00oCfy*d1yJO?5%jzt(e0Ga=t zqAHyic;VDT8x>prAEI}<wk`*Ol)o_;K{+wq0ev_Jf0v_BbTN8Rlk%zgKc{mTi_f4G zNuWyUno!K)Uv$rO@~TBwj(THH{`At;?I*|gXyMk&&xFT&i{YA?wQxDEFdN|2x1SvT z-aJRC0Rh)Bums3s;da&M&-IaF(0jhlAqECzytjusU{#B%sEwhFS)sY4s4wGuV0A*- zE5f8i<Yb+-!2C(1^%gT~P!>m%y%S9~%_&q>(Jk)yoXro}blH3ljpms>&RmC<XzFNX zva5A3*#NbO3K3tL=m8r|I$D_SYj$ltev0f?TU5EG=C_t?yDYJ)FJMmf5RJ??f{)FD zX$H50Cb+#3G=dM!t?-6v1xvvV^8?c|?XVRrhqLc%LEGFkw@s^_a6f-8((*n8WFsgL zh0HlRAac+B_^0IUqewvhgEg&1d^lMgbx2~-BR1qmb(bp`9ER@FUD|sm^&IO3Pf#yV zPTCT>$y2|^u+q1f>~BC&rgb%ge8qT$7h|3$9)7&iC3!1DN8ISipAPhpWN#AZX<TV- zHSzx<XbnZQq9Wrn;@^T#xQnp#j|5+8k5~2DCdDt&d>g)+hn|PyB0U<}pYW<$0(Hz1 Wei!hIHfY{#Hk)(J<>pVDwf_M&n<xzc diff --git a/aosstower/level_b1/daily/foo.txt b/aosstower/level_b1/daily/foo.txt new file mode 100644 index 0000000..c2bf21f --- /dev/null +++ b/aosstower/level_b1/daily/foo.txt @@ -0,0 +1,163 @@ +time +lon +station_name +lat +base_time +time_offset +alt +rh_shield_freq_mean +qc_rh_shield_freq_mean +paro_cal_sig_low +qc_paro_cal_sig_low +air_temp_4_low +qc_air_temp_4_low +paro_air_temp_period_low +qc_paro_air_temp_period_low +air_temp_3_high +qc_air_temp_3_high +air_temp_2_low +qc_air_temp_2_low +pressure_mean +qc_pressure_mean +rh_mean +qc_rh_mean +rtd_shield_freq_mean +qc_rtd_shield_freq_mean +box_rh_high +qc_box_rh_high +box_air_temp_high +qc_box_air_temp_high +paro_air_temp_high +qc_paro_air_temp_high +air_temp_5_mean +qc_air_temp_5_mean +box_air_temp_low +qc_box_air_temp_low +precip_mean +qc_precip_mean +rh_shield_freq_high +qc_rh_shield_freq_high +rh_low +qc_rh_low +box_presure_high +qc_box_presure_high +paro_cal_sig_mean +qc_paro_cal_sig_mean +dewpoint_mean +qc_dewpoint_mean +paro_cal_sig_high +qc_paro_cal_sig_high +dewpoint_high +qc_dewpoint_high +paro_air_temp_period_mean +qc_paro_air_temp_period_mean +box_temp_low +qc_box_temp_low +altimeter_low +qc_altimeter_low +precip_low +qc_precip_low +rh_shield_freq_low +qc_rh_shield_freq_low +paro_air_temp_period_high +qc_paro_air_temp_period_high +wind_speed_low +qc_wind_speed_low +paro_pressure_period_low +qc_paro_pressure_period_low +paro_air_temp_low +qc_paro_air_temp_low +box_rh_low +qc_box_rh_low +air_temp_4_mean +qc_air_temp_4_mean +air_temp_6_3m_high +qc_air_temp_6_3m_high +gust_high +qc_gust_high +air_temp_5_low +qc_air_temp_5_low +wind_dir_high +qc_wind_dir_high +box_presure_low +qc_box_presure_low +pressure_low +qc_pressure_low +rh_high +qc_rh_high +rtd_shield_freq_low +qc_rtd_shield_freq_low +wind_speed_mean +qc_wind_speed_mean +air_temp_high +qc_air_temp_high +precip_high +qc_precip_high +accum_precip_high +qc_accum_precip_high +solar_flux_high +qc_solar_flux_high +paro_pressure_period_mean +qc_paro_pressure_period_mean +air_temp_4_high +qc_air_temp_4_high +peak_wind +qc_peak_wind +gust_low +qc_gust_low +rtd_shield_freq_high +qc_rtd_shield_freq_high +wind_dir_mean +qc_wind_dir_mean +box_temp_high +qc_box_temp_high +accum_precip_low +qc_accum_precip_low +pressure_high +qc_pressure_high +air_temp_6_3m_low +qc_air_temp_6_3m_low +wind_dir_low +qc_wind_dir_low +air_temp_3_low +qc_air_temp_3_low +air_temp_mean +qc_air_temp_mean +altimeter_mean +qc_altimeter_mean +air_temp_2_high +qc_air_temp_2_high +altimeter_high +qc_altimeter_high +box_presure_mean +qc_box_presure_mean +air_temp_low +qc_air_temp_low +box_air_temp_mean +qc_box_air_temp_mean +box_temp_mean +qc_box_temp_mean +paro_air_temp_mean +qc_paro_air_temp_mean +solar_flux_low +qc_solar_flux_low +wind_speed_high +qc_wind_speed_high +air_temp_2_mean +qc_air_temp_2_mean +accum_precip_mean +qc_accum_precip_mean +dewpoint_low +qc_dewpoint_low +box_rh_mean +qc_box_rh_mean +air_temp_3_mean +qc_air_temp_3_mean +solar_flux_mean +qc_solar_flux_mean +paro_pressure_period_high +qc_paro_pressure_period_high +air_temp_6_3m_mean +qc_air_temp_6_3m_mean +air_temp_5_high +qc_air_temp_5_high diff --git a/aosstower/level_b1/daily/nc.py b/aosstower/level_b1/daily/nc.py index 7000340..f4a1d41 100644 --- a/aosstower/level_b1/daily/nc.py +++ b/aosstower/level_b1/daily/nc.py @@ -4,6 +4,7 @@ import logging import pandas as pd from datetime import datetime as dt from aosstower.l00 import parser +import avg_database from netCDF4 import Dataset import numpy as np import platform @@ -43,7 +44,7 @@ def writeDimensions(ncFile): return ncFile -def createVariables(ncFile, firstStamp, chunksizes, zlib): +def createVariables(ncFile, firstStamp, chunksizes, zlib, database=parser.database): #base_time long name btln = 'base time as unix timestamp' @@ -115,11 +116,11 @@ def createVariables(ncFile, firstStamp, chunksizes, zlib): if 'time' in key: variable.calendar = 'gregorian' - for entry in parser.database: + for entry in database: if(entry == 'stamp'): continue - varTup = parser.database[entry] + varTup = database[entry] variable = ncFile.createVariable(entry, np.float32, dimensions=('time'), fill_value=float(-99999), zlib=zlib, chunksizes=chunksizes) @@ -265,6 +266,24 @@ def minuteAverages(frame): return newFrame.fillna(-99999) +def averageOverInterval(frame,interval_width): + """takes a frame and an interval to average it over, and returns a minimum, + maximum, and average dataframe for that interval""" + ts = frame.index + #round each timestamp to the nearest n minutes + frame['interval'] = (ts.astype(int)-ts.astype(int)%(interval_width*60e9)).astype('datetime64[ns]') + outFrames = {} + outFrames['low'] = frame.groupby('interval').min() + outFrames['high'] = frame.groupby('interval').max() + outFrames['mean'] = frame.groupby('interval').mean() + del frame['interval'] + for key in outFrames: + #append the appropriate suffix to each column + columns = outFrames[key].columns + outFrames[key].columns = ['_'.join([col,key]) for col in columns] + outFrames = pd.concat(outFrames.values(),axis=1) + return outFrames + def getData(inputFiles): dictData = {} @@ -282,9 +301,8 @@ def getData(inputFiles): return pd.DataFrame(dictData).transpose().replace(-99999, np.nan) -def writeVars(ncFile, frame): +def writeVars(ncFile, frame, database=parser.database): stamps = list(frame.index) - baseDTObj = dt.strptime(str(stamps[0]).split(' ')[0], '%Y-%m-%d') #find out how much time elapsed @@ -308,7 +326,6 @@ def writeVars(ncFile, frame): counter += 1 fileVar = ncFile.variables - fileVar['base_time'].assignValue(baseTimeValue) fileVar['time_offset'][:] = timeNumpy fileVar['time'][:] = timeNumpy + baseTimeValue @@ -331,13 +348,16 @@ def writeVars(ncFile, frame): #writes data into file for varName in frame: + if varName not in fileVar: + logging.warn('Extraneous key: %s in frame'%varName) + continue dataList = frame[varName].tolist() dataArray = np.asarray(dataList) fileVar[varName][:] = dataArray - valid_min = parser.database[varName][5] - valid_max = parser.database[varName][6] + valid_min = database[varName][5] + valid_max = database[varName][6] fileVar['qc_' + varName][:] = filterArray(dataArray, valid_min, valid_max) @@ -366,7 +386,8 @@ def writeVars(ncFile, frame): # @param input filenames - list of filenames # @param output filename - filename of the netcdf file -def createGiantNetCDF(start, end, inputFiles, outputName, zlib, chunkSize): +def createGiantNetCDF(start, end, inputFiles, outputName, zlib, chunkSize, + interval_width = None, database=parser.database): default = False if(chunkSize): @@ -383,7 +404,9 @@ def createGiantNetCDF(start, end, inputFiles, outputName, zlib, chunkSize): else: frame = minuteAverages(frame) - + if interval_width: + frame = averageOverInterval(frame,interval_width) + if(start and end): frame = frame[start.strftime('%Y-%m-%d %H:%M:%S'): end.strftime('%Y-%m-%d %H:%M:%S')] @@ -396,11 +419,11 @@ def createGiantNetCDF(start, end, inputFiles, outputName, zlib, chunkSize): ncFile = writeDimensions(ncFile) - ncFile = createVariables(ncFile, firstStamp, chunksizes, zlib) + ncFile = createVariables(ncFile, firstStamp, chunksizes, zlib,database) ncFile.inputFiles = ', '.join(inputFiles) - ncFile = writeVars(ncFile, frame) + ncFile = writeVars(ncFile, frame,database) ncFile.close() @@ -437,47 +460,53 @@ def _dt_convert(datetime_str): return dt.strptime(datetime_str, '%Y-%m-%dT%H:%M:%S') except: return dt.strptime(datetime_str, '%Y-%m-%d') - + def main(): import argparse #argparse description - parser = argparse.ArgumentParser(description="Convert level_00 aoss tower data to level_a0") + argparser = argparse.ArgumentParser(description="Convert level_00 aoss tower data to level_a0") #argparse verbosity info - parser.add_argument('-v', '--verbose', action="count", default=int(os.environ.get("VERBOSITY", 2)), + argparser.add_argument('-v', '--verbose', action="count", default=int(os.environ.get("VERBOSITY", 2)), dest='verbosity', help='each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG (default INFO)') #argparse start and end times - parser.add_argument('-s', '--start-time', type=_dt_convert, + argparser.add_argument('-s', '--start-time', type=_dt_convert, help="Start time of massive netcdf file, if only -s is given, a netcdf file for only that day is given" + ". Formats allowed: \'YYYY-MM-DDTHH:MM:SS\', \'YYYY-MM-DD\'") - parser.add_argument('-e', '--end-time', type=_dt_convert, help='End time of massive netcdf file. Formats allowed:' + + argparser.add_argument('-e', '--end-time', type=_dt_convert, help='End time of massive netcdf file. Formats allowed:' + "\'YYYY-MM-DDTHH:MM:SS\', \'YYYY-MM-DD\'") - parser.add_argument('-cs', '--chunk-size', type=int, help='chunk Size for the netCDF file') - parser.add_argument('-z', '--zlib', action='store_true', help='compress netCDF file with zlib') + argparser.add_argument('-i', '--interval', type=float, + help='Width of the interval to average input data over in minutes.'+ + " If not specified, 1 is assumed. (Use 60 for one hour and 1440 for 1 day)") + argparser.add_argument('-cs', '--chunk-size', type=int, help='chunk Size for the netCDF file') + argparser.add_argument('-z', '--zlib', action='store_true', help='compress netCDF file with zlib') - parser.add_argument("input_files", nargs="+", + argparser.add_argument("input_files", nargs="+", help="aoss_tower level_00 paths") - parser.add_argument('-o', '--output', required=True, nargs="+", help="filename pattern or filename. " + + argparser.add_argument('-o', '--output', required=True, nargs="+", help="filename pattern or filename. " + "Should be along the lines of <filepath>/aoss_tower.YYYY-MM-DD.nc") - args = parser.parse_args() + args = argparser.parse_args() levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG] level=levels[min(3, args.verbosity)] logging.basicConfig(level=level) + database = avg_database.AOSS_VARS if args.interval else parser.database if(args.start_time and args.end_time): - result = createGiantNetCDF(args.start_time, args.end_time, args.input_files, args.output[0], args.zlib, args.chunk_size) + result = createGiantNetCDF(args.start_time, args.end_time, args.input_files, args.output[0], args.zlib, args.chunk_size, + args.interval, database) if(result == False): raise IOError('An empty ASCII file was found') elif(args.start_time): end_time = args.start_time.replace(hour=23, minute=59, second=59) - result = createGiantNetCDF(args.start_time, end_time, args.input_files, args.output[0], args.zlib, args.chunk_size) + result = createGiantNetCDF(args.start_time, end_time, args.input_files, args.output[0], args.zlib, args.chunk_size, + args.interval, database) if(result == False): raise IOError('An empty ASCII file was found') @@ -486,5 +515,6 @@ def main(): else: createMultiple(args.input_files, args.output, args.zlib, args.chunk_size) + if __name__ == "__main__": main() diff --git a/aosstower/level_b1/daily/nc.pyc b/aosstower/level_b1/daily/nc.pyc deleted file mode 100644 index 55c6b53922aadd2cd6e5fd6b266f2e5839dbdea0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8817 zcmcIpOK%*<cCPN>Ye)`BQ7=+2w;#M7Uu(#b?JG-)?Q5E(XgM;K+e3*q6Jgw(uHiH_ z(>?6!p(uw$04qp>EP}hai!72&a2ElR&HW8o1xS`zCO~jk*#=nz2;zL-sp%nw$Vd=G z<aE!es#C91=ke8<!oQZrCrW?0)lmIE7vG=aG5?NNh>xS%N^K8lM{PTKy{op}oGz&C zA)#H>9#)-UwLPL7_AjXRsOpTV?J-*(QthJZjH~T&TgLbkYI`ytr=+$^LJzC<l<G{Y z?P=v`SMALxeMUXU^s_1)QNNEhomJte5a(1lCd7Fa7KON=!f_!ks&GPxITcO{F|Wds z5SKV$IHjJrD*ZR5FRABBIkD`xR-VT0qwn#zQFcIPW0Xda=`8AKTQh>1Zjc(C+VUvM zt3ehRot;#eEXblHZg31f1w1Qw%v*RJDYYllK}^qG<@C!CX(2Dy%0pn>$oS^e**OrM z;IQ7-alJoyJq|iL^3YJ`;bmNUBh^^!Dtg2Q^GpAYBtEtDxN$dX>oRziyv%QcT52U7 zz4T-^NS0QTP%kxlX<F~5$)0XxW+`s0>sY54cxgE?W;1!J)1?4ZCRAG==ypA*f^V_= zOqTI9jZUq?oW&G|a#QY%GvhY6E3USLN2KxvYM^k(&?HwK533_bBm{Awazuqg>UdOz zFV<f?9#he%%D^hhh`mBT&;iN;wFog{k=g)a7sQ0R`4U#BpVhi46;9`@GcRSucu*TC zA=D4F9rfG~v7T2)L+WVw6zhcn>v?suR*VGJQFUZjY)8GQZidiXXzZo_0O94FxFWl` z`a^?ZylX0aE$7B~kidC$^SG!E|HU~fs@2E;>>Q6PP)Flx{&+$iO{maQ^XVhEPhnn~ z##Q$S?_WE3hp(&o@D253NXpP&oCJnIM>f2f4-vj4)B@^$q>c(Id|RQ{t^=hWI-XSf zHJ6(HzwV2P`yIdgKX#0-i><vdqg^95JXoX5Q@{eTr|_KY{V&|Gtd1sC9lH095Tgnn zf;;{(MD@2KDo|t@lj?VEp_sE>2ivcO4HVhN^Og!){!>~&v1KA}SwzcRfs=q@%VgfN zgqHWD1r%FK(t`C~koC5lqY+u=l-h4O_-q~iF&zS&=j!9%Q^zHB1U0V;Z?~}jRRNo^ z&8VM0p322k%FXFbDh<x_L{ak-9h}M$Oa@}Kdt;$5k28ve{;qNAUlq2(8=RmH+j&n2 zSkU`Iz=3}t1Sb255OCl(g@6P9PzV^pEwwkVvP)|3GJQ2o{Fb_yT~T{i)!sFgy~e7G zoWWy{TKP8o%7(d&_YT~$M+fQ!#_Pq=p%maiJ6-cS+Kp}`acDdf#SQJ<xcy<ZQoVtX zSFPTWkC_G6!q=6BPGun+)NY$gsFrb)W)q&@Yo<xZ<7{=-nyLRbb~OJV-+yn0QD3=n zt9m~xf@&x6Zj|*xolT+?>fKapQ`dpX#BmEBBM11*5PRN6M@FPJqKovS2hID%Nt(4X z@{>*a1zAS7+tSf)D>G+69_TcS8bR9&BS-<8hEqgj5%!>+`J6`9HrPux!76O3Znr8O z8a{U-WW{jt{g~u2CS_ceh(V_jHWY=jztdsMTmZ`^&*ilIbT?_V%o`l;e>#`#i0h8< zC<%etIH(W<O58J2Y)0*N{UB)fbR+?id;7K>?MP#z)r+5)=v!?vHuk!*neB*jLs-F! z>=CyaQHBxap0Q}PM`BtpZ3fVrWCv*AKq9_`xs|(>8;ds$M|rsA)oN=iG9^V)6OqW2 zLDifG2LtJ%MAAhB&~{K=Jg8QM0bkstSEi=QxtlC)aujKfyBTL_wv!-x|2?0ytWPR3 zYM?~}kAgG`cG_A-=q4u0q5~}o792tesE80!6xKUYe7Y1I%0W_1!XOO=ZVc4cmM9Kb zRc2%*jb=SfFb~Crl}7Qd&-LdRDxgY<Hig<W(x^*>WSNs*3QeS<!5j~C%oZc%2;We3 zVP$(0_@<ZY&1YS0j~vJgpa=+LP{h<a=wvfoMbgZk2C1$??FV>hFTp-}nq8PtGf6ul zwVr4$RUt7VoNfSXA{&=2jV#pb&?;T8%Z6JJUy?o((CCYPPz#$;YO)$#jDHP9|0-TG z;!f6%#n5)JA?n!<1jldIi-z|4GDy-eih;Lm^Ar8d9#@v7vZ90hL@9qrD}5$nMD=E` zTSnb0gmy3Ll=wl@<&=dH{7>UKJ5+Grb@Kl=oEwFbGwzN%GtLF)qr&aNZTEU%(kVKV zs4D>uqUD8W66G0Z(iw3}?yU1VcuLNhLJ>U%|Ap&rg94+@Q`=VXm|x-rhj8qw!{>|x ztB-$I9T!vv`3gn^B0-$M2QU+22R>sYaU>{yLCBeYLD&Rs*mYIy@Hfb@`uKg~QJ_*_ zxhJD4jnw|nB=$<>JisYIo~u5_7eXV#@JC?S+ApasoXH-59Z(FWA7kW#^>FAi&wN9C zmrv6!Q)}de+fXQLGt_mcaaeDr@TgV~fc;Ob{BF>OHBkuNP^^-7kh^4-QkDj>fw5ad z-%NYDEDnNGMRB*6@fKi2o8qLdJKgM=6vC*HaU>~F)kX8^BX)Hr!yD^lIFnsH+Fp^q zamARH>-iR>nd4N{3+KE$2c0U3mW(^2?yS`ZMmXAp=)+&Z`QY$*m$1j~D#&XO8UbVu zQ0u69o{Suy)r5I_0fW*&SHO#V2QD4>JTIs@Xblu%4_6H67&K^<hYDHS1IE<@_m1a| zLv4{u%OP~ig#vxXyUWSMWa9k<7e8EJ0Q)i3Tz^yyh7S}8T7(A^e+UKJ|4wZ|o%hBS zaXT<yQO)lGw~PKdWJevae3|4BfDnokEaH1oSP_2rN}ShW9S|S7vu<ioHzjqr0+A`4 zQMeaSbQ3HC8FRDK1eh!enB+ib0HPfcfeilc;VlY#^>I}ZQ98td^FW}K*K|IF%y2xT zj_|>5z){}M9PY1HCe-Si#R5n}kD5cT=vPn%iTR>_VTJ(U)71&8$+^6R&+jaJd1s;K zGqU;w8G>khA`+k4;qxLLeGb(yG==2{wVEdvTAzrnEG#}KnI(!teJC~-W?~QsEr?n$ zF<LMR!ezScD6^O+$%1yhk6rw?^I^og{7n{Zu(k2n2DreOXg7XDl;U$#VY&ZcWBuOd z!__sJ_5SjvEzlkU6QpTi9e$S3Vc11eSbm)Y*zik)D`PQyt!_Tp*^?!*MBF74N;~(n z(d*#m&fyzLFV1u-mfhbnMq+O+oxHXudFdViE+l64!QDpge^_2=Ax$FvSUQ;laZ32O z<Pf%fADa+17M8g?9_%afDcpvnC$Mt!Cn%VkoXnYa-x<E(OcpM?BSR%;PGZuOTXfGj z6VCL|Ip-W=&<Ntzbm1c6*^D!d9t}!?=ywo<zOp_C77t@g=lI6`7ls~>FbY{u0Ji|^ zrUMXzrz0|V8Ill3U|`=l5@3+CD8PPj=k8G(;U@a-h+X9hKi>C7PW(@F9AxK+2poSc z;S6snA_|Fb5?7ehuY(o0K%^ehzb8b)0e~A$1}ng)ziJ%$gxZ4xz|HU!1E&HLN5wAS zD_*ufKv0-C*x<Hs*`&F{Yr)~caUr1t2QtPBs`Z~}t)EfZtgsbjSaEjNGcuuAEA!|! z#0MB+s^!=d%kq|MEBEes)JE?qu6a$Ua2SEX-cO#2K@jKRHpHyc#@gn}>fI0OEBBXc zwR<ZscOZt5^}1Ik%=aJi^)+AL;3al)@4;G{CaK?Itzb9CokklG*Z&dgXzo5eiNC{F zgD>nt`I;|o#s3*!AM=GYtMZTWDpQ-RSIJ;m;%9Um3WEULq?aLh(|jzh<Un$jso^T) z|2BJ5DMQ^1dbrLB0Hs$Dc<Qf#dWE({?`iSjdKARjhRz`9zeA<Dju#{M2rO*God({U zW)z<+6~^63=MB_fb{E|9?j)XBv^OYCZiG&H1&={E5~Z$6V3I5Vst{$MU{4AvtupMc ze-0zFR>X|%;l~WTFbMFhK3-D417#g%LXoxxA))^WHJ!0)7^w{KT3k4)oc)j0mS`QF z@`l;R`(bT)eeI4H_c}W|#l_XLhas9rm}I<8&t#t7@8N!Z!X<F)6w}j|jy-t9H1lw? z@^HK!{i^YBQSss=^GsJaqGqJ65|=3o8$E_V=|cn?p1Q^il9cuThzt~#g?Mbtk(&{X zx9hyyM{`Cr$NvNN5{2Ooqwvsw%pvf+BF}qrBmEMp8Fky*$-}U6t~tX_$(?r^sCqGD z&U+kA8W;`-i8F@BsY$|n;|0sz%tODlo!Fbr$<wS5T_lZ<{CWBcw@Kt)53{=5WYcV# zd!&O<E`5N&Q?iIskD>uvf}XVjlAw;VPbe@P>~rNDi)u2xZGos@On6PGFEA{SE4-sk zaziN4alk$xK4!Fm?A##-dd<#AY|jv_!85RJ7Hw|chEyDqaD%q<r`wQ=LlzEguom>2 zJKbh;)0iHKX|!EA-F8)l_zc=4AARxEoF4PW$PBaNn^^Z?F>k4GK_M@TaCE|fByw4y zdqI|r&>&YlI75P1yb+x6Mts71O6?-DSon@zzr)}{1nMF(0<wbxDf}Nx1X2z&+k3Mr z{Tm_9lEBX_wvz9&LL)m58lqr^7t}>5BeH{@SI^-o;-5-kR+)duNBTzx3DEV=;{|3d z$(fJxk`adG=dfD20;SlHpPu}mfE1Ux!fXiqg#C%7TJ@ML@+4X05iEi1w;`ECC9515 znt!E&ZacjF5t#%HNuSV<2)y(E5R~|kM{E9P4{G-|zdlKjAva_Fiya}ty^wCFrQ2OY zZ=r)m%S#&kg-A1Jio_}IsRr(A(80T57v*KGH0kZOyfxo{;8(Vm{f&DY>y>*OcOO(% z*FJl=?v?v-%46L-b_YiCiIs|Y$O=89EP#MtQnoemoUEq{?hrp8z;EiT5jMrk+F`Wd z`$`4wRR%GyVexByU;9Hn=?N>1d<7C$E0QB*9F?NSI17l8VskPpBHXE2zm@!7r7}n} zGC_o#Fz(+*#z;EdRO1Y>-n^&xG|E~Yd-RtaClvuU<_cyF+U?|NALzslJC%p&5h$*k zJ7DmlrbG?K608X^Btj_3u@M8obtxV;v}D}aoCcB6VL!nJJRwOfd$Es$bk|7U@m({| z`Jr)jt8oJ@WNOS<F(-0Mt5j@=ksCvb02rk@lnv*B$U`68D`GA^uUquYr<kGEN_y?k z+tFUYgt!Q{9mUWnobN|u=6s*XFOGI4i(lif^5vJ6)m7w7C1op^NfsDkNCiASh}SmM zio%?Zw>DVsB#=F(9>m<fY)>l6{2f6y2Fb_|zmi*BPInQ1<LtNOr(j8N2VqzTk5mhC zL!Qz9l)V|8b$k$|NO)4mkks&hj&A{#a7+DPl-b*+QrpSyF7l_cE3qE20a}m{G#6<T zGm;^Zy!C&9f#gte%ZnOt9?fW1Mi&vtFB7s6*n|`;a`;pBqMxv5XmKUiBnn@r{tHg< zd0Vn*<j+CbJQaO|teOLe{Eh!-<iqoda0I991W|mJk+sTs04iq-RcFK*EnIQz|C}?1 z<l(4$%^5?Y@fuQ(Glf~_7V5?dB7=IJW7O+%y!MoMGUl1_e~E7qNSB#!<3B=~NZz7& zxke9u{a$QWt5y;`V{6qPoLmTi(Ynth6X1V>c5%9cz9;v6=)--SI9&Z0xayKzM~;~b zy8mGP)NSD#)W}7_9vx4_D$=h&oyc6M|My?^xwelwN!V-aPYBZtr<p_!j{o>!AOAwR Z)1$@F;y1-}#h(^GC{7iJif>Ok{{;%+j->zq diff --git a/aosstower/level_b1/monthly/nc_monthly.py b/aosstower/level_b1/monthly/nc_monthly.py index 8fd9c8c..b94edf2 100644 --- a/aosstower/level_b1/monthly/nc_monthly.py +++ b/aosstower/level_b1/monthly/nc_monthly.py @@ -1,7 +1,12 @@ import platform from netCDF4 import Dataset +from aosstower.l00 import parser +import calendar +import logging +from aosstower.level_b1.daily import nc from datetime import datetime as dt from collections import OrderedDict +import glob import os # CF Metadata 1.6 compliant structure for NetCDF file. This is essentially @@ -824,11 +829,58 @@ AOSS_VARS = OrderedDict({ }) def _dt_convert(date): - return None + return dt.strptime(date, '%Y-%m-%d') + +def files_fr_directories(directories,file_ending='.ascii'): + """gets a list of all raw data files in the given directories""" + files = [] + for directory in directories: + files += glob.glob(os.path.join(directory,'*%s'%file_ending)) + return files def createGiantNetCDF(start, end, directories, output, zlib, chunk_size): + default = False + + if(chunk_size): + chunksizes = [chunk_size] + + else: + default = True + inputFiles = files_fr_directories(directories) + return False + frame = getData(inputFiles) + + if(frame.empty): + return False + + else: + + frame = minuteAverages(frame) + + if(start and end): + frame = frame[start.strftime('%Y-%m-%d %H:%M:%S'): end.strftime('%Y-%m-%d %H:%M:%S')] + + if(default): + chunksizes = [len(list(frame.index))] + + firstStamp = dt.strptime(str(list(frame.index)[0]), '%Y-%m-%d %H:%M:%S') + + ncFile = Dataset(outputName, 'w', format='NETCDF4_CLASSIC') + + ncFile = writeDimensions(ncFile) + + ncFile = createVariables(ncFile, firstStamp, chunksizes, zlib) + + ncFile.inputFiles = ', '.join(inputFiles) + + ncFile = writeVars(ncFile, frame) + + ncFile.close() + + return True + def writeDimensions(ncFile): for name, size in BASIC_STATION['dimensions'].items(): ncFile.createDimension(name, size) @@ -920,12 +972,13 @@ def main(): if(args.start_time and args.end_time): - result = createGiantNetCDF(args.start_time, args.end_time, args.input_files, args.output[0], args.zlib, args.chunk_size) + result = createGiantNetCDF(args.start_time, args.end_time, args.input_directories, args.output[0], args.zlib, args.chunk_size) if(result == False): raise IOError('An empty ASCII file was found') elif(args.start_time): - end_time = args.start_time.replace(hour=23, minute=59, second=59) + last_day = calendar.monthrange(args.start_time.year, args.start_time.month)[1] + end_time = args.start_time.replace(day = last_day, hour=23, minute=59, second=59) result = createGiantNetCDF(args.start_time, end_time, args.input_directories, args.output[0], args.zlib, args.chunk_size) if(result == False): raise IOError('An empty ASCII file was found') @@ -937,9 +990,4 @@ def main(): createMultiple(args.input_files, args.output, args.zlib, args.chunk_size) if __name__ == "__main__": - # main() - ncFile = Dataset('aoss_tower.2013-06.SUMMARY.nc', 'w', format='NETCDF4_CLASSIC') - ncFile = writeDimensions(ncFile) - ncFile = createVariables(ncFile, dt(2003, 6, 1), [30], True) - - ncFile.close() + main() -- GitLab