From 3dd305e205e0817e49561fc0615f1edb11a139e3 Mon Sep 17 00:00:00 2001
From: mwestphall <westphall@wisc.edu>
Date: Thu, 12 Jan 2017 15:29:47 -0600
Subject: [PATCH] added --interval command line arg which adds function of
monthly to daily
---
aosstower/level_b1/__init__.py | 0
aosstower/level_b1/daily/autoRun.py | 3 -
aosstower/level_b1/daily/avg_database.py | 838 +++++++++++++++++++++++
aosstower/level_b1/daily/bundle.py | 139 ----
aosstower/level_b1/daily/bundle.pyc | Bin 3992 -> 0 bytes
aosstower/level_b1/daily/foo.txt | 163 +++++
aosstower/level_b1/daily/nc.py | 78 ++-
aosstower/level_b1/daily/nc.pyc | Bin 8817 -> 0 bytes
aosstower/level_b1/monthly/nc_monthly.py | 66 +-
9 files changed, 1112 insertions(+), 175 deletions(-)
create mode 100644 aosstower/level_b1/__init__.py
delete mode 100755 aosstower/level_b1/daily/autoRun.py
create mode 100644 aosstower/level_b1/daily/avg_database.py
delete mode 100644 aosstower/level_b1/daily/bundle.py
delete mode 100644 aosstower/level_b1/daily/bundle.pyc
create mode 100644 aosstower/level_b1/daily/foo.txt
delete mode 100644 aosstower/level_b1/daily/nc.pyc
diff --git a/aosstower/level_b1/__init__.py b/aosstower/level_b1/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/aosstower/level_b1/daily/autoRun.py b/aosstower/level_b1/daily/autoRun.py
deleted file mode 100755
index 0d15a20..0000000
--- a/aosstower/level_b1/daily/autoRun.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from bundle import createYesterdayFile
-
-createYesterdayFile()
diff --git a/aosstower/level_b1/daily/avg_database.py b/aosstower/level_b1/daily/avg_database.py
new file mode 100644
index 0000000..43b78fc
--- /dev/null
+++ b/aosstower/level_b1/daily/avg_database.py
@@ -0,0 +1,838 @@
+# CF Metadata 1.6 compliant structure for NetCDF file. This is essentially
+# a pythonified version of the CDL
+from collections import OrderedDict
+import aosstower.schema as schema
+BASIC_STATION = {
+
+ 'globals': {
+ 'source': 'surface observation',
+ 'institution': 'UW SSEC',
+ 'featureType': 'timeSeries',
+ 'Conventions': 'ARM-1.2 CF-1.6',
+ 'data_level': 'b1',
+ 'datastream' : 'aoss.tower.nc-1mo-1d.lb1.v00',
+ 'software_version': '00'
+ },
+
+ 'dimensions': {
+ 'max_len_station_name': 32,
+ 'time': None # Unlimited
+ },
+
+ 'variables': OrderedDict({
+ 'base_time': {
+ 'string': '', # Filled in by creator
+ 'standard_name': 'time',
+ 'long_name': 'base time as unix timestamp',
+ 'units': 'seconds since 1970-01-01 00:00:00 0:00',
+ '_type': 'd',
+ '_shape': tuple(),
+ },
+ 'time_offset': {
+ 'long_name': 'time offset from base_time',
+ 'standard_name': 'time',
+ 'units': '', # Filled in by creator
+ '_type': 'd',
+ '_shape': ('time',),
+ },
+ 'time': {
+ 'long_name': 'time offset from epoch',
+ 'standard_name': 'time',
+ 'units': 'seconds since 1970-01-01 00:00:00 0:00',
+ '_type': 'd',
+ '_shape': ('time',),
+ },
+ 'station_name': {
+ 'cf_role': 'timeseries_id',
+ 'long_name': 'station name',
+ '_type': 'S1',
+ '_shape': ('max_len_station_name',),
+ },
+ 'lat': {
+ 'standard_name': 'latitude',
+ 'units': 'degrees_north',
+ 'valid_min': -90,
+ 'valid_max': 90,
+ '_type': 'f',
+ '_shape': tuple(),
+ },
+ 'lon': {
+ 'standard_name': 'longitude',
+ 'units': 'degrees_east',
+ 'valid_min': -180,
+ 'valid_max': 180,
+ '_type': 'f',
+ '_shape': tuple(),
+ },
+ 'alt': {
+ 'long_name': 'vertical distance',
+ 'standard_name': 'height',
+ 'units': 'm',
+ 'positive': 'up',
+ 'axis': 'Z',
+ '_type': 'f',
+ '_shape': tuple(),
+ }
+ })
+}
+
+AOSS_VARS = OrderedDict({
+ 'box_temp_high': {
+ 'standard_name': 'air_temperature',
+ 'units': 'degC',
+ 'description': 'Auxillary Temperature',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'box_temp_low': {
+ 'standard_name': 'air_temperature',
+ 'units': 'degC',
+ 'description': 'Auxillary Temperature',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'box_temp_mean': {
+ 'standard_name': 'air_temperature',
+ 'units': 'degC',
+ 'description': 'Auxillary Temperature',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'box_presure_high': {
+ 'standard_name': 'air_pressure',
+ 'description': 'Pressure inside the data logger enclosure',
+ 'units': 'hpa',
+ 'valid_min': 850,
+ 'valid_max': '1100',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'box_presure_low': {
+ 'standard_name': 'air_pressure',
+ 'description': 'Pressure inside the data logger enclosure',
+ 'units': 'hpa',
+ 'valid_min': 850,
+ 'valid_max': '1100',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'box_presure_mean': {
+ 'standard_name': 'air_pressure',
+ 'description': 'Pressure inside the data logger enclosure',
+ 'units': 'hpa',
+ 'valid_min': 850,
+ 'valid_max': '1100',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'paro_air_temp_period_high': {
+ 'standard_name': '',
+ 'description': '',
+ 'units': '1',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'paro_air_temp_period_low': {
+ 'standard_name': '',
+ 'description': '',
+ 'units': '1',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'paro_air_temp_period_mean': {
+ 'standard_name': '',
+ 'description': '',
+ 'units': '1',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'paro_pressure_period_high': {
+ 'standard_name': '',
+ 'description': '',
+ 'units': '1',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'paro_pressure_period_low': {
+ 'standard_name': '',
+ 'description': '',
+ 'units': '1',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'paro_pressure_period_mean': {
+ 'standard_name': '',
+ 'description': '',
+ 'units': '1',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'paro_air_temp_high': {
+ 'standard_name': 'air_temperature',
+ 'description': '',
+ 'units': 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'paro_air_temp_low': {
+ 'standard_name': 'air_temperature',
+ 'description': '',
+ 'units': 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'paro_air_temp_mean': {
+ 'standard_name': 'air_temperature',
+ 'description': '',
+ 'units': 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'pressure_high': {
+ 'standard_name': 'air_pressure',
+ 'description': 'Air pressure as measured from the PAROSCI pressure sensor',
+ 'units': 'hpa',
+ 'valid_min': 850,
+ 'valid_max': 1100,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'pressure_low': {
+ 'standard_name': 'air_pressure',
+ 'description': 'Air pressure as measured from the PAROSCI pressure sensor',
+ 'units': 'hpa',
+ 'valid_min': 850,
+ 'valid_max': 1100,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'pressure_mean': {
+ 'standard_name': 'air_pressure',
+ 'description': 'Air pressure as measured from the PAROSCI pressure sensor',
+ 'units': 'hpa',
+ 'valid_min': 850,
+ 'valid_max': 1100,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'paro_cal_sig_high': {
+ 'standard_name': '',
+ 'description': '',
+ 'units': '',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'paro_cal_sig_low': {
+ 'standard_name': '',
+ 'description': '',
+ 'units': '',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'paro_cal_sig_mean': {
+ 'standard_name': '',
+ 'description': '',
+ 'units': '',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'box_rh_high': {
+ 'standard_name': 'relative humidity',
+ 'description': 'Relative humidity inside the data logger enclosure',
+ 'units': '%',
+ 'valid_min': 0,
+ 'valid_max': 100,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'box_rh_low': {
+ 'standard_name': 'relative humidity',
+ 'description': 'Relative humidity inside the data logger enclosure',
+ 'units': '%',
+ 'valid_min': 0,
+ 'valid_max': 100,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'box_rh_mean': {
+ 'standard_name': 'relative humidity',
+ 'description': 'Relative humidity inside the data logger enclosure',
+ 'units': '%',
+ 'valid_min': 0,
+ 'valid_max': 100,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'box_air_temp_high': {
+ 'standard_name': 'air_temperature',
+ 'description': 'Air temperature inside the data logger enclosure',
+ 'units': 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'box_air_temp_low': {
+ 'standard_name': 'air_temperature',
+ 'description': 'Air temperature inside the data logger enclosure',
+ 'units': 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'box_air_temp_mean': {
+ 'standard_name': 'air_temperature',
+ 'description': 'Air temperature inside the data logger enclosure',
+ 'units': 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'air_temp_2_high': {
+ 'standard_name': 'air_temperature',
+ 'description': 'Auxillary air temperature',
+ 'units': 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'air_temp_2_low': {
+ 'standard_name': 'air_temperature',
+ 'description': 'Auxillary air temperature',
+ 'units': 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'air_temp_2_mean': {
+ 'standard_name': 'air_temperature',
+ 'description': 'Auxillary air temperature',
+ 'units': 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'air_temp_3_high': {
+ 'standard_name': 'air_temperature',
+ 'description': 'Auxillary air temperature',
+ 'units' : 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'air_temp_3_low': {
+ 'standard_name': 'air_temperature',
+ 'description': 'Auxillary air temperature',
+ 'units' : 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'air_temp_3_mean': {
+ 'standard_name': 'air_temperature',
+ 'description': 'Auxillary air temperature',
+ 'units' : 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'air_temp_4_high': {
+ 'standard_name': 'air_temperature',
+ 'description' : 'Auxillary air temperature',
+ 'units' : 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'air_temp_4_low': {
+ 'standard_name': 'air_temperature',
+ 'description' : 'Auxillary air temperature',
+ 'units' : 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'air_temp_4_mean': {
+ 'standard_name': 'air_temperature',
+ 'description' : 'Auxillary air temperature',
+ 'units' : 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'air_temp_5_high': {
+ 'standard_name': 'air_temperature',
+ 'descripiton': 'Auxillary air temperature',
+ 'units': 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'air_temp_5_low': {
+ 'standard_name': 'air_temperature',
+ 'descripiton': 'Auxillary air temperature',
+ 'units': 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+},
+
+ 'air_temp_5_mean': {
+ 'standard_name': 'air_temperature',
+ 'descripiton': 'Auxillary air temperature',
+ 'units': 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'wind_speed_high': {
+ 'standard_name': 'wind_speed',
+ 'description': 'wind_speed',
+ 'units': 'm*s^-1',
+ 'valid_min': 0,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'wind_speed_low': {
+ 'standard_name': 'wind_speed',
+ 'description': 'wind_speed',
+ 'units': 'm*s^-1',
+ 'valid_min': 0,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'wind_speed_mean': {
+ 'standard_name': 'wind_speed',
+ 'description': 'wind_speed',
+ 'units': 'm*s^-1',
+ 'valid_min': 0,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'wind_dir_high': {
+ 'standard_name': 'wind_direction',
+ 'description': 'wind_direction',
+ 'units': 'degrees',
+ 'valid_min': 0,
+ 'valid_max': 360,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'wind_dir_low': {
+ 'standard_name': 'wind_direction',
+ 'description': 'wind_direction',
+ 'units': 'degrees',
+ 'valid_min': 0,
+ 'valid_max': 360,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'wind_dir_mean': {
+ 'standard_name': 'wind_direction',
+ 'description': 'wind_direction',
+ 'units': 'degrees',
+ 'valid_min': 0,
+ 'valid_max': 360,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'rh_shield_freq_high': {
+ 'standard_name': '',
+ 'descrption' : '',
+ 'units': 'hz',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'rh_shield_freq_low': {
+ 'standard_name': '',
+ 'descrption' : '',
+ 'units': 'hz',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'rh_shield_freq_mean': {
+ 'standard_name': '',
+ 'descrption' : '',
+ 'units': 'hz',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'rh_high': {
+ 'standard_name': 'relative_humidity',
+ 'descripiton': 'Relative humidity',
+ 'units': '%',
+ 'valid_min': 0,
+ 'valid_max': 100,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'rh_low': {
+ 'standard_name': 'relative_humidity',
+ 'descripiton': 'Relative humidity',
+ 'units': '%',
+ 'valid_min': 0,
+ 'valid_max': 100,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'rh_mean': {
+ 'standard_name': 'relative_humidity',
+ 'descripiton': 'Relative humidity',
+ 'units': '%',
+ 'valid_min': 0,
+ 'valid_max': 100,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'air_temp_6_3m_high': {
+ 'standard_name': 'air_temperature',
+ 'description': 'Air temperature 6.3m from tower base',
+ 'units': 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'air_temp_6_3m_low': {
+ 'standard_name': 'air_temperature',
+ 'description': 'Air temperature 6.3m from tower base',
+ 'units': 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'air_temp_6_3m_mean': {
+ 'standard_name': 'air_temperature',
+ 'description': 'Air temperature 6.3m from tower base',
+ 'units': 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'dewpoint_high': {
+ 'standard_name': 'dewpoint_temperature',
+ 'description': 'Calculated dewpoint temperature',
+ 'units': 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'dewpoint_low': {
+ 'standard_name': 'dewpoint_temperature',
+ 'description': 'Calculated dewpoint temperature',
+ 'units': 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'dewpoint_mean': {
+ 'standard_name': 'dewpoint_temperature',
+ 'description': 'Calculated dewpoint temperature',
+ 'units': 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'rtd_shield_freq_high': {
+ 'standard_name': '',
+ 'description': '',
+ 'units': '',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'rtd_shield_freq_low': {
+ 'standard_name': '',
+ 'description': '',
+ 'units': '',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'rtd_shield_freq_mean': {
+ 'standard_name': '',
+ 'description': '',
+ 'units': '',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'air_temp_high': {
+ 'standard_name': 'air_temperature',
+ 'description': 'Air temperature',
+ 'units': 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'air_temp_low': {
+ 'standard_name': 'air_temperature',
+ 'description': 'Air temperature',
+ 'units': 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'air_temp_mean': {
+ 'standard_name': 'air_temperature',
+ 'description': 'Air temperature',
+ 'units': 'degC',
+ 'valid_min': -50,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'solar_flux_high': {
+ 'standard_name': 'solar_flux',
+ 'description': 'Solar flux',
+ 'units': 'w*m^-2',
+ 'valid_min': 0,
+ 'valid_max': 3000,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'solar_flux_low': {
+ 'standard_name': 'solar_flux',
+ 'description': 'Solar flux',
+ 'units': 'w*m^-2',
+ 'valid_min': 0,
+ 'valid_max': 3000,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'solar_flux_mean': {
+ 'standard_name': 'solar_flux',
+ 'description': 'Solar flux',
+ 'units': 'w*m^-2',
+ 'valid_min': 0,
+ 'valid_max': 3000,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'precip_high': {
+ 'standard_name': '',
+ 'description': 'Precipitation',
+ 'units': 'mm',
+ 'valid_min': 0,
+ 'valid_max': 254,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'precip_low': {
+ 'standard_name': '',
+ 'description': 'Precipitation',
+ 'units': 'mm',
+ 'valid_min': 0,
+ 'valid_max': 254,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'precip_mean': {
+ 'standard_name': '',
+ 'description': 'Precipitation',
+ 'units': 'mm',
+ 'valid_min': 0,
+ 'valid_max': 254,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'accum_precip_high': {
+ 'standard_name': 'axxumulated_precipitation',
+ 'description': 'Precipitation accumulated since 0Z',
+ 'units': 'mm',
+ 'valid_min': 0,
+ 'valid_max': 254,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'accum_precip_low': {
+ 'standard_name': 'axxumulated_precipitation',
+ 'description': 'Precipitation accumulated since 0Z',
+ 'units': 'mm',
+ 'valid_min': 0,
+ 'valid_max': 254,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'accum_precip_mean': {
+ 'standard_name': 'axxumulated_precipitation',
+ 'description': 'Precipitation accumulated since 0Z',
+ 'units': 'mm',
+ 'valid_min': 0,
+ 'valid_max': 254,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'altimeter_high': {
+ 'standard_name': '',
+ 'description': '',
+ 'units': 'inHg',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'altimeter_low': {
+ 'standard_name': '',
+ 'description': '',
+ 'units': 'inHg',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'altimeter_mean': {
+ 'standard_name': '',
+ 'description': '',
+ 'units': 'inHg',
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'gust_high':
+ {
+ 'standard_name': 'wind_speed_of_gust',
+ 'descripiton': 'Wind gust over the previous 2 minutes',
+ 'units': 'm/s',
+ 'valid_min': 0,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'gust_low':
+ {
+ 'standard_name': 'wind_speed_of_gust',
+ 'descripiton': 'Wind gust over the previous 2 minutes',
+ 'units': 'm/s',
+ 'valid_min': 0,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ },
+
+ 'peak_wind':
+ {
+ 'standard_name': 'wind_speed',
+ 'description': 'fastest 5-second wind_speed',
+ 'units': 'm/s',
+ 'valid_min': 0,
+ 'valid_max': 50,
+ '_type': 'f',
+ '_shape': ('time',),
+ }
+})
+
+def make_Var_db(aoss_vars):
+ var_dict = {}
+ for key in aoss_vars:
+ #there's a typo in some var names
+ entry = aoss_vars[key]
+ key = key.replace('presure','pressure')
+ var_dict[key] = schema.Var(
+ entry.get('_type','f'),
+ entry.get('standard_name',''),
+ key,
+ entry.get('description',''),
+ entry.get('units',''),
+ entry.get('valid_min',''),
+ entry.get('valid_max',''),
+ )
+ return var_dict
+AOSS_VARS = make_Var_db(AOSS_VARS)
diff --git a/aosstower/level_b1/daily/bundle.py b/aosstower/level_b1/daily/bundle.py
deleted file mode 100644
index 66729cd..0000000
--- a/aosstower/level_b1/daily/bundle.py
+++ /dev/null
@@ -1,139 +0,0 @@
-from nc import getData, createGiantNetCDF, writeDimensions, createVariables, writeVars, minuteAverages
-from datetime import datetime as dt
-from datetime import timedelta as delta
-import pandas as pd
-import time
-from netCDF4 import Dataset
-import os
-
-def writeBack(frame, filename, cur_dt):
- #get start
- startString = str(frame.index[0])
- startObj = dt.strptime(startString.split(" ")[0], '%Y-%m-%d')
- createDirectory(startObj)
-
- #get end
- endObj = startObj.replace(hour=23, minute=59, second=59, microsecond=59)
-
- #input file
- inputFiles = []
- inputFiles.append(filename)
- inputFiles.append(createFilename(cur_dt - delta(days=1)))
-
- #output filename
- outputFilename = 'aoss_tower.' + startString.split(" ")[0] + '.day.nc'
-
- #create file
- createGiantNetCDF(startObj, endObj, inputFiles, outputFilename, True, None)
-
-def createDirectory(stamp):
- if os.path.exists("/mnt/inst-data/regen_netcdf/aoss/tower/" + stamp.strftime('%Y/%m')):
- os.chdir("/mnt/inst-data/regen_netcdf/aoss/tower/" + stamp.strftime('%Y/%m'))
-
- else:
- os.makedirs("/mnt/inst-data/regen_netcdf/aoss/tower/" + stamp.strftime('%Y/%m'))
- os.chdir("/mnt/inst-data/regen_netcdf/aoss/tower/" + stamp.strftime('%Y/%m'))
-
-
-def createFilename(date):
- filename = '/mnt/inst-data/aoss-tower/'
- filename += date.strftime('%Y/%m/')
- filename += 'rig_tower.' + date.strftime('%Y-%m-%d') + '.ascii'
-
- return filename
-
-def createNCFile(frame, filename):
- firstStamp = dt.strptime(str(list(frame.index)[0]), '%Y-%m-%d %H:%M:%S')
-
- createDirectory(firstStamp)
-
- outputName = 'aoss_tower.' + firstStamp.strftime('%Y-%m-%d') + '.day.nc'
-
- ncFile = Dataset(outputName, 'w', format='NETCDF4_CLASSIC')
-
- ncFile = writeDimensions(ncFile)
-
- ncFile = createVariables(ncFile, firstStamp, [len(list(frame.index))], True)
-
- ncFile.inputFiles = ', '.join([filename])
-
- ncFile = writeVars(ncFile, frame)
-
- ncFile.close()
-
-def create_files(start_date, end_date):
- # for each file's name based on date - does not rely on what dates are
- # in each file so should be similar code for buoy
- # get file, discover which full dates excluding the start are in it
- # from there we create the netcdf files
- # for the start, we always recreate the netcdf using the previous file
- # ignore the last unfulfilled date
-
- cur_dt = start_date
-
- for day in range((end_date - start_date).days + 1):
- filename = createFilename(cur_dt)
- print('grabbing ' + filename)
-
- df = getData([filename])
-
- if(df.empty):
- cur_dt += delta(days = 1)
- continue
-
- DFList = []
-
- for group in df.groupby(df.index.day):
- DFList.append(group[1])
-
-
- if(len(DFList) <= 1):
- frame = DFList[0]
-
- frame = minuteAverages(frame)
-
- createNCFile(frame, filename)
-
- elif(len(DFList) == 2):
- if(len(list(DFList[0].index)) > len(list(DFList[1].index))):
- frame = DFList[0]
-
- frame = minuteAverages(frame)
-
- createNCFile(frame, filename)
-
- #get second frame
- frame = DFList[1]
-
- writeBack(frame, filename, cur_dt)
-
- else:
- frame = DFList[1]
-
- frame = minuteAverages(frame)
-
- createNCFile(frame, filename)
-
- #get second frame
- frame = DFList[0]
-
- writeBack(frame, filename, cur_dt)
-
- else:
- print(len(DFList))
-
- for frame in DFList:
- print(len(list(frame.index)))
- print(list(frame.index)[0])
- exit(0)
-
-
-
- #increment day
- cur_dt += delta(days = 1)
-
-def createYesterdayFile():
- create_files(dt.today() - delta(days=1), dt.today() - delta(days=1))
-
-#createYesterdayFile()
-create_files(dt(2009,5,2), dt(2009,5,2))#dt.today() - delta(days=1))
diff --git a/aosstower/level_b1/daily/bundle.pyc b/aosstower/level_b1/daily/bundle.pyc
deleted file mode 100644
index 99b5245907dca7496a03515d81c8f008c7504070..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001
literal 3992
zcmcgvO>Y~?5v`sfDUlW}tz}!@_3lPNR#_fa;vz8Oz%i2bO0gWk;Q}Kgh6xzLXr@IE
zMb1z&tylsiB!F_wCBGr3{EFlr<dU3n%q2e{m)w%~s!2)fb&g(1Y<BfmRad=wT?+r*
zZvCPC$LEPme;WAy0-x%iXj1$c8HtRI^o@)I=?5|nr60<;CjFX>>(Z}_;krOZ4H-A2
zKj-C8<$38hy<C$~OZwOJO!OC|zo7lPjM_53F8%8l>lUTIC}~4}E7D(*bWS_(Njk5c
z8<IA)vn**#J2xf0rkz`2Z;G9h6i2ruy)Ne=#o=u^7qLxAaqJTM=FWKlZ8h2HyoYOM
z|HV(Ij-H!CGq7$ecCq7BQrg(r?JUk+&$`X6$8Hg$uglEYt!!-bDl76z$Nb7)#$^`o
zjcnzbvvnA%ItG%PN0Z_>hn;?eizuab<l>$mqY2bsS(k+9a~Icb;#2(<jT1Q)85+js
z{c|H{p}0VXp`3;?tT7T8<XjpG?s--dSC`WoM-9G)4Z+RRIz0^E!mDiIoH=-=c2<{x
z$XP>98#0`eQ{%U(fD4=kf}6qb@VNTX&%@zwDqleaR?g<cwW#|P7T`>XiU8M^bB~jq
z&UIX=m!T0yFRfO*@Y;*@wek8|>S_>P6-Q;Z!UaU<ILnWm&3-(SE1MK~igP)Yl*Lr$
z#xy>uG9FrSHZH1a*A=gAxlz%v8+oG29fiu3@Di|)r}h}`#Ac@f!^DIk(%EV<%G{N3
zZ)K%TTv47Va7sHF#feob<H^LrDqhYb<8d~!c|1Kw;~gm|Xy>!?$m&gdMeY?njz^X4
z&=ab6zvNDe`?F){isY!=g_*ou<>J!qxH8KJQ?ns|wKr5_U<tb|tjqGrk)th@+EyI7
zsZ85CSRmR|U!m81Rg7)-U=SDG%_6m3s4I6TWiiB^t8Sib+uWAGbN3NE`;3n5(#PuJ
z?#TYij&|eEyL(4@I<gy+lgpI&GENQ%>FNd*2FvC<rWJk?HcZ>BhIhk+j^T&CQP>0W
zggr1n3<z!FKb#_c1-LFB>NH^e%{i0b(@6mSEt5R;?b}iTJrEkb>Ql7caqhYql6@U1
z9Cu4Qu=y^cm8AQ6>#o0cMG&pM=&p@BjF2L!s5H2f*uC=MIL<0p&Cs=ffv)6Lnw2^{
zjt?yQQ{Lc_i0^pvZsQXzL(fOlMW6|S@=^GK(O@tiE`sqR2yo>Yyy+bZQ90+n9|G+#
zJ{VQbSpn6oOs96#DZT0g9Px%jiJbMR59vAeVYeb?%WN>qTUs{bhKkEJ;ws59WzL9U
z>d;hmH<>p2kVE*qJ5)!var*c1nR=~Zmd)KDp&5)DZBz*V2^|k%U;>mw7pZ^XO0qaz
z2(rc;s8Zc&=tvN|@-<edBT4q4Wq^Oi-sgcK#e<vmIQE-Lxj3rzHBC5UV93DdHNFbq
z{2_Q-2Qh{1yKMn0kgjF=a-ct41k>x=ywhPcJXcqI9cPhPU3>Ci?H3Q$cD}*0UyB!$
z38bqu|5`I)zbMDCC)D20p5b}Dx4ZfD(az4#H!C{uv(?TLz9Ma?Fe0C7bVo>`h;Ol?
zRzn~0<&3y5q9&zvaBg&m0?|7xvOM}O=P4<%CL_ET`t(P4IYGQAPwpf8z^6z$6+KV(
z$ij?ho{&P{EwJ1I6#2KKw@G#DkUe9XwV)NO1h;~P;8V1V9&TjR3%LCY{D~+6Psjf-
zz@1OkJHBclWp2Ivv+<RKM;ieVF!n3kPYBPNVg--_;fC@)>H@ss!@87*S_Jqa0<XWA
z`%rTDvE-<nv~oD!(4{~rHq6Vg$r57g<%*yhU{!&F00oCfy*d1yJO?5%jzt(e0Ga=t
zqAHyic;VDT8x>prAEI}<wk`*Ol)o_;K{+wq0ev_Jf0v_BbTN8Rlk%zgKc{mTi_f4G
zNuWyUno!K)Uv$rO@~TBwj(THH{`At;?I*|gXyMk&&xFT&i{YA?wQxDEFdN|2x1SvT
z-aJRC0Rh)Bums3s;da&M&-IaF(0jhlAqECzytjusU{#B%sEwhFS)sY4s4wGuV0A*-
zE5f8i<Yb+-!2C(1^%gT~P!>m%y%S9~%_&q>(Jk)yoXro}blH3ljpms>&RmC<XzFNX
zva5A3*#NbO3K3tL=m8r|I$D_SYj$ltev0f?TU5EG=C_t?yDYJ)FJMmf5RJ??f{)FD
zX$H50Cb+#3G=dM!t?-6v1xvvV^8?c|?XVRrhqLc%LEGFkw@s^_a6f-8((*n8WFsgL
zh0HlRAac+B_^0IUqewvhgEg&1d^lMgbx2~-BR1qmb(bp`9ER@FUD|sm^&IO3Pf#yV
zPTCT>$y2|^u+q1f>~BC&rgb%ge8qT$7h|3$9)7&iC3!1DN8ISipAPhpWN#AZX<TV-
zHSzx<XbnZQq9Wrn;@^T#xQnp#j|5+8k5~2DCdDt&d>g)+hn|PyB0U<}pYW<$0(Hz1
Wei!hIHfY{#Hk)(J<>pVDwf_M&n<xzc
diff --git a/aosstower/level_b1/daily/foo.txt b/aosstower/level_b1/daily/foo.txt
new file mode 100644
index 0000000..c2bf21f
--- /dev/null
+++ b/aosstower/level_b1/daily/foo.txt
@@ -0,0 +1,163 @@
+time
+lon
+station_name
+lat
+base_time
+time_offset
+alt
+rh_shield_freq_mean
+qc_rh_shield_freq_mean
+paro_cal_sig_low
+qc_paro_cal_sig_low
+air_temp_4_low
+qc_air_temp_4_low
+paro_air_temp_period_low
+qc_paro_air_temp_period_low
+air_temp_3_high
+qc_air_temp_3_high
+air_temp_2_low
+qc_air_temp_2_low
+pressure_mean
+qc_pressure_mean
+rh_mean
+qc_rh_mean
+rtd_shield_freq_mean
+qc_rtd_shield_freq_mean
+box_rh_high
+qc_box_rh_high
+box_air_temp_high
+qc_box_air_temp_high
+paro_air_temp_high
+qc_paro_air_temp_high
+air_temp_5_mean
+qc_air_temp_5_mean
+box_air_temp_low
+qc_box_air_temp_low
+precip_mean
+qc_precip_mean
+rh_shield_freq_high
+qc_rh_shield_freq_high
+rh_low
+qc_rh_low
+box_presure_high
+qc_box_presure_high
+paro_cal_sig_mean
+qc_paro_cal_sig_mean
+dewpoint_mean
+qc_dewpoint_mean
+paro_cal_sig_high
+qc_paro_cal_sig_high
+dewpoint_high
+qc_dewpoint_high
+paro_air_temp_period_mean
+qc_paro_air_temp_period_mean
+box_temp_low
+qc_box_temp_low
+altimeter_low
+qc_altimeter_low
+precip_low
+qc_precip_low
+rh_shield_freq_low
+qc_rh_shield_freq_low
+paro_air_temp_period_high
+qc_paro_air_temp_period_high
+wind_speed_low
+qc_wind_speed_low
+paro_pressure_period_low
+qc_paro_pressure_period_low
+paro_air_temp_low
+qc_paro_air_temp_low
+box_rh_low
+qc_box_rh_low
+air_temp_4_mean
+qc_air_temp_4_mean
+air_temp_6_3m_high
+qc_air_temp_6_3m_high
+gust_high
+qc_gust_high
+air_temp_5_low
+qc_air_temp_5_low
+wind_dir_high
+qc_wind_dir_high
+box_presure_low
+qc_box_presure_low
+pressure_low
+qc_pressure_low
+rh_high
+qc_rh_high
+rtd_shield_freq_low
+qc_rtd_shield_freq_low
+wind_speed_mean
+qc_wind_speed_mean
+air_temp_high
+qc_air_temp_high
+precip_high
+qc_precip_high
+accum_precip_high
+qc_accum_precip_high
+solar_flux_high
+qc_solar_flux_high
+paro_pressure_period_mean
+qc_paro_pressure_period_mean
+air_temp_4_high
+qc_air_temp_4_high
+peak_wind
+qc_peak_wind
+gust_low
+qc_gust_low
+rtd_shield_freq_high
+qc_rtd_shield_freq_high
+wind_dir_mean
+qc_wind_dir_mean
+box_temp_high
+qc_box_temp_high
+accum_precip_low
+qc_accum_precip_low
+pressure_high
+qc_pressure_high
+air_temp_6_3m_low
+qc_air_temp_6_3m_low
+wind_dir_low
+qc_wind_dir_low
+air_temp_3_low
+qc_air_temp_3_low
+air_temp_mean
+qc_air_temp_mean
+altimeter_mean
+qc_altimeter_mean
+air_temp_2_high
+qc_air_temp_2_high
+altimeter_high
+qc_altimeter_high
+box_presure_mean
+qc_box_presure_mean
+air_temp_low
+qc_air_temp_low
+box_air_temp_mean
+qc_box_air_temp_mean
+box_temp_mean
+qc_box_temp_mean
+paro_air_temp_mean
+qc_paro_air_temp_mean
+solar_flux_low
+qc_solar_flux_low
+wind_speed_high
+qc_wind_speed_high
+air_temp_2_mean
+qc_air_temp_2_mean
+accum_precip_mean
+qc_accum_precip_mean
+dewpoint_low
+qc_dewpoint_low
+box_rh_mean
+qc_box_rh_mean
+air_temp_3_mean
+qc_air_temp_3_mean
+solar_flux_mean
+qc_solar_flux_mean
+paro_pressure_period_high
+qc_paro_pressure_period_high
+air_temp_6_3m_mean
+qc_air_temp_6_3m_mean
+air_temp_5_high
+qc_air_temp_5_high
diff --git a/aosstower/level_b1/daily/nc.py b/aosstower/level_b1/daily/nc.py
index 7000340..f4a1d41 100644
--- a/aosstower/level_b1/daily/nc.py
+++ b/aosstower/level_b1/daily/nc.py
@@ -4,6 +4,7 @@ import logging
import pandas as pd
from datetime import datetime as dt
from aosstower.l00 import parser
+import avg_database
from netCDF4 import Dataset
import numpy as np
import platform
@@ -43,7 +44,7 @@ def writeDimensions(ncFile):
return ncFile
-def createVariables(ncFile, firstStamp, chunksizes, zlib):
+def createVariables(ncFile, firstStamp, chunksizes, zlib, database=parser.database):
#base_time long name
btln = 'base time as unix timestamp'
@@ -115,11 +116,11 @@ def createVariables(ncFile, firstStamp, chunksizes, zlib):
if 'time' in key:
variable.calendar = 'gregorian'
- for entry in parser.database:
+ for entry in database:
if(entry == 'stamp'):
continue
- varTup = parser.database[entry]
+ varTup = database[entry]
variable = ncFile.createVariable(entry, np.float32,
dimensions=('time'), fill_value=float(-99999), zlib=zlib, chunksizes=chunksizes)
@@ -265,6 +266,24 @@ def minuteAverages(frame):
return newFrame.fillna(-99999)
+def averageOverInterval(frame,interval_width):
+ """takes a frame and an interval to average it over, and returns a minimum,
+ maximum, and average dataframe for that interval"""
+ ts = frame.index
+ #round each timestamp to the nearest n minutes
+ frame['interval'] = (ts.astype(int)-ts.astype(int)%(interval_width*60e9)).astype('datetime64[ns]')
+ outFrames = {}
+ outFrames['low'] = frame.groupby('interval').min()
+ outFrames['high'] = frame.groupby('interval').max()
+ outFrames['mean'] = frame.groupby('interval').mean()
+ del frame['interval']
+ for key in outFrames:
+ #append the appropriate suffix to each column
+ columns = outFrames[key].columns
+ outFrames[key].columns = ['_'.join([col,key]) for col in columns]
+ outFrames = pd.concat(outFrames.values(),axis=1)
+ return outFrames
+
def getData(inputFiles):
dictData = {}
@@ -282,9 +301,8 @@ def getData(inputFiles):
return pd.DataFrame(dictData).transpose().replace(-99999, np.nan)
-def writeVars(ncFile, frame):
+def writeVars(ncFile, frame, database=parser.database):
stamps = list(frame.index)
-
baseDTObj = dt.strptime(str(stamps[0]).split(' ')[0], '%Y-%m-%d')
#find out how much time elapsed
@@ -308,7 +326,6 @@ def writeVars(ncFile, frame):
counter += 1
fileVar = ncFile.variables
-
fileVar['base_time'].assignValue(baseTimeValue)
fileVar['time_offset'][:] = timeNumpy
fileVar['time'][:] = timeNumpy + baseTimeValue
@@ -331,13 +348,16 @@ def writeVars(ncFile, frame):
#writes data into file
for varName in frame:
+ if varName not in fileVar:
+ logging.warn('Extraneous key: %s in frame'%varName)
+ continue
dataList = frame[varName].tolist()
dataArray = np.asarray(dataList)
fileVar[varName][:] = dataArray
- valid_min = parser.database[varName][5]
- valid_max = parser.database[varName][6]
+ valid_min = database[varName][5]
+ valid_max = database[varName][6]
fileVar['qc_' + varName][:] = filterArray(dataArray, valid_min, valid_max)
@@ -366,7 +386,8 @@ def writeVars(ncFile, frame):
# @param input filenames - list of filenames
# @param output filename - filename of the netcdf file
-def createGiantNetCDF(start, end, inputFiles, outputName, zlib, chunkSize):
+def createGiantNetCDF(start, end, inputFiles, outputName, zlib, chunkSize,
+ interval_width = None, database=parser.database):
default = False
if(chunkSize):
@@ -383,7 +404,9 @@ def createGiantNetCDF(start, end, inputFiles, outputName, zlib, chunkSize):
else:
frame = minuteAverages(frame)
-
+ if interval_width:
+ frame = averageOverInterval(frame,interval_width)
+
if(start and end):
frame = frame[start.strftime('%Y-%m-%d %H:%M:%S'): end.strftime('%Y-%m-%d %H:%M:%S')]
@@ -396,11 +419,11 @@ def createGiantNetCDF(start, end, inputFiles, outputName, zlib, chunkSize):
ncFile = writeDimensions(ncFile)
- ncFile = createVariables(ncFile, firstStamp, chunksizes, zlib)
+ ncFile = createVariables(ncFile, firstStamp, chunksizes, zlib,database)
ncFile.inputFiles = ', '.join(inputFiles)
- ncFile = writeVars(ncFile, frame)
+ ncFile = writeVars(ncFile, frame,database)
ncFile.close()
@@ -437,47 +460,53 @@ def _dt_convert(datetime_str):
return dt.strptime(datetime_str, '%Y-%m-%dT%H:%M:%S')
except:
return dt.strptime(datetime_str, '%Y-%m-%d')
-
+
def main():
import argparse
#argparse description
- parser = argparse.ArgumentParser(description="Convert level_00 aoss tower data to level_a0")
+ argparser = argparse.ArgumentParser(description="Convert level_00 aoss tower data to level_a0")
#argparse verbosity info
- parser.add_argument('-v', '--verbose', action="count", default=int(os.environ.get("VERBOSITY", 2)),
+ argparser.add_argument('-v', '--verbose', action="count", default=int(os.environ.get("VERBOSITY", 2)),
dest='verbosity',
help='each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG (default INFO)')
#argparse start and end times
- parser.add_argument('-s', '--start-time', type=_dt_convert,
+ argparser.add_argument('-s', '--start-time', type=_dt_convert,
help="Start time of massive netcdf file, if only -s is given, a netcdf file for only that day is given" +
". Formats allowed: \'YYYY-MM-DDTHH:MM:SS\', \'YYYY-MM-DD\'")
- parser.add_argument('-e', '--end-time', type=_dt_convert, help='End time of massive netcdf file. Formats allowed:' +
+ argparser.add_argument('-e', '--end-time', type=_dt_convert, help='End time of massive netcdf file. Formats allowed:' +
"\'YYYY-MM-DDTHH:MM:SS\', \'YYYY-MM-DD\'")
- parser.add_argument('-cs', '--chunk-size', type=int, help='chunk Size for the netCDF file')
- parser.add_argument('-z', '--zlib', action='store_true', help='compress netCDF file with zlib')
+ argparser.add_argument('-i', '--interval', type=float,
+ help='Width of the interval to average input data over in minutes.'+
+ " If not specified, 1 is assumed. (Use 60 for one hour and 1440 for 1 day)")
+ argparser.add_argument('-cs', '--chunk-size', type=int, help='chunk Size for the netCDF file')
+ argparser.add_argument('-z', '--zlib', action='store_true', help='compress netCDF file with zlib')
- parser.add_argument("input_files", nargs="+",
+ argparser.add_argument("input_files", nargs="+",
help="aoss_tower level_00 paths")
- parser.add_argument('-o', '--output', required=True, nargs="+", help="filename pattern or filename. " +
+ argparser.add_argument('-o', '--output', required=True, nargs="+", help="filename pattern or filename. " +
"Should be along the lines of <filepath>/aoss_tower.YYYY-MM-DD.nc")
- args = parser.parse_args()
+ args = argparser.parse_args()
levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
level=levels[min(3, args.verbosity)]
logging.basicConfig(level=level)
+ database = avg_database.AOSS_VARS if args.interval else parser.database
if(args.start_time and args.end_time):
- result = createGiantNetCDF(args.start_time, args.end_time, args.input_files, args.output[0], args.zlib, args.chunk_size)
+ result = createGiantNetCDF(args.start_time, args.end_time, args.input_files, args.output[0], args.zlib, args.chunk_size,
+ args.interval, database)
if(result == False):
raise IOError('An empty ASCII file was found')
elif(args.start_time):
end_time = args.start_time.replace(hour=23, minute=59, second=59)
- result = createGiantNetCDF(args.start_time, end_time, args.input_files, args.output[0], args.zlib, args.chunk_size)
+ result = createGiantNetCDF(args.start_time, end_time, args.input_files, args.output[0], args.zlib, args.chunk_size,
+ args.interval, database)
if(result == False):
raise IOError('An empty ASCII file was found')
@@ -486,5 +515,6 @@ def main():
else:
createMultiple(args.input_files, args.output, args.zlib, args.chunk_size)
+
if __name__ == "__main__":
main()
diff --git a/aosstower/level_b1/daily/nc.pyc b/aosstower/level_b1/daily/nc.pyc
deleted file mode 100644
index 55c6b53922aadd2cd6e5fd6b266f2e5839dbdea0..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001
literal 8817
zcmcIpOK%*<cCPN>Ye)`BQ7=+2w;#M7Uu(#b?JG-)?Q5E(XgM;K+e3*q6Jgw(uHiH_
z(>?6!p(uw$04qp>EP}hai!72&a2ElR&HW8o1xS`zCO~jk*#=nz2;zL-sp%nw$Vd=G
z<aE!es#C91=ke8<!oQZrCrW?0)lmIE7vG=aG5?NNh>xS%N^K8lM{PTKy{op}oGz&C
zA)#H>9#)-UwLPL7_AjXRsOpTV?J-*(QthJZjH~T&TgLbkYI`ytr=+$^LJzC<l<G{Y
z?P=v`SMALxeMUXU^s_1)QNNEhomJte5a(1lCd7Fa7KON=!f_!ks&GPxITcO{F|Wds
z5SKV$IHjJrD*ZR5FRABBIkD`xR-VT0qwn#zQFcIPW0Xda=`8AKTQh>1Zjc(C+VUvM
zt3ehRot;#eEXblHZg31f1w1Qw%v*RJDYYllK}^qG<@C!CX(2Dy%0pn>$oS^e**OrM
z;IQ7-alJoyJq|iL^3YJ`;bmNUBh^^!Dtg2Q^GpAYBtEtDxN$dX>oRziyv%QcT52U7
zz4T-^NS0QTP%kxlX<F~5$)0XxW+`s0>sY54cxgE?W;1!J)1?4ZCRAG==ypA*f^V_=
zOqTI9jZUq?oW&G|a#QY%GvhY6E3USLN2KxvYM^k(&?HwK533_bBm{Awazuqg>UdOz
zFV<f?9#he%%D^hhh`mBT&;iN;wFog{k=g)a7sQ0R`4U#BpVhi46;9`@GcRSucu*TC
zA=D4F9rfG~v7T2)L+WVw6zhcn>v?suR*VGJQFUZjY)8GQZidiXXzZo_0O94FxFWl`
z`a^?ZylX0aE$7B~kidC$^SG!E|HU~fs@2E;>>Q6PP)Flx{&+$iO{maQ^XVhEPhnn~
z##Q$S?_WE3hp(&o@D253NXpP&oCJnIM>f2f4-vj4)B@^$q>c(Id|RQ{t^=hWI-XSf
zHJ6(HzwV2P`yIdgKX#0-i><vdqg^95JXoX5Q@{eTr|_KY{V&|Gtd1sC9lH095Tgnn
zf;;{(MD@2KDo|t@lj?VEp_sE>2ivcO4HVhN^Og!){!>~&v1KA}SwzcRfs=q@%VgfN
zgqHWD1r%FK(t`C~koC5lqY+u=l-h4O_-q~iF&zS&=j!9%Q^zHB1U0V;Z?~}jRRNo^
z&8VM0p322k%FXFbDh<x_L{ak-9h}M$Oa@}Kdt;$5k28ve{;qNAUlq2(8=RmH+j&n2
zSkU`Iz=3}t1Sb255OCl(g@6P9PzV^pEwwkVvP)|3GJQ2o{Fb_yT~T{i)!sFgy~e7G
zoWWy{TKP8o%7(d&_YT~$M+fQ!#_Pq=p%maiJ6-cS+Kp}`acDdf#SQJ<xcy<ZQoVtX
zSFPTWkC_G6!q=6BPGun+)NY$gsFrb)W)q&@Yo<xZ<7{=-nyLRbb~OJV-+yn0QD3=n
zt9m~xf@&x6Zj|*xolT+?>fKapQ`dpX#BmEBBM11*5PRN6M@FPJqKovS2hID%Nt(4X
z@{>*a1zAS7+tSf)D>G+69_TcS8bR9&BS-<8hEqgj5%!>+`J6`9HrPux!76O3Znr8O
z8a{U-WW{jt{g~u2CS_ceh(V_jHWY=jztdsMTmZ`^&*ilIbT?_V%o`l;e>#`#i0h8<
zC<%etIH(W<O58J2Y)0*N{UB)fbR+?id;7K>?MP#z)r+5)=v!?vHuk!*neB*jLs-F!
z>=CyaQHBxap0Q}PM`BtpZ3fVrWCv*AKq9_`xs|(>8;ds$M|rsA)oN=iG9^V)6OqW2
zLDifG2LtJ%MAAhB&~{K=Jg8QM0bkstSEi=QxtlC)aujKfyBTL_wv!-x|2?0ytWPR3
zYM?~}kAgG`cG_A-=q4u0q5~}o792tesE80!6xKUYe7Y1I%0W_1!XOO=ZVc4cmM9Kb
zRc2%*jb=SfFb~Crl}7Qd&-LdRDxgY<Hig<W(x^*>WSNs*3QeS<!5j~C%oZc%2;We3
zVP$(0_@<ZY&1YS0j~vJgpa=+LP{h<a=wvfoMbgZk2C1$??FV>hFTp-}nq8PtGf6ul
zwVr4$RUt7VoNfSXA{&=2jV#pb&?;T8%Z6JJUy?o((CCYPPz#$;YO)$#jDHP9|0-TG
z;!f6%#n5)JA?n!<1jldIi-z|4GDy-eih;Lm^Ar8d9#@v7vZ90hL@9qrD}5$nMD=E`
zTSnb0gmy3Ll=wl@<&=dH{7>UKJ5+Grb@Kl=oEwFbGwzN%GtLF)qr&aNZTEU%(kVKV
zs4D>uqUD8W66G0Z(iw3}?yU1VcuLNhLJ>U%|Ap&rg94+@Q`=VXm|x-rhj8qw!{>|x
ztB-$I9T!vv`3gn^B0-$M2QU+22R>sYaU>{yLCBeYLD&Rs*mYIy@Hfb@`uKg~QJ_*_
zxhJD4jnw|nB=$<>JisYIo~u5_7eXV#@JC?S+ApasoXH-59Z(FWA7kW#^>FAi&wN9C
zmrv6!Q)}de+fXQLGt_mcaaeDr@TgV~fc;Ob{BF>OHBkuNP^^-7kh^4-QkDj>fw5ad
z-%NYDEDnNGMRB*6@fKi2o8qLdJKgM=6vC*HaU>~F)kX8^BX)Hr!yD^lIFnsH+Fp^q
zamARH>-iR>nd4N{3+KE$2c0U3mW(^2?yS`ZMmXAp=)+&Z`QY$*m$1j~D#&XO8UbVu
zQ0u69o{Suy)r5I_0fW*&SHO#V2QD4>JTIs@Xblu%4_6H67&K^<hYDHS1IE<@_m1a|
zLv4{u%OP~ig#vxXyUWSMWa9k<7e8EJ0Q)i3Tz^yyh7S}8T7(A^e+UKJ|4wZ|o%hBS
zaXT<yQO)lGw~PKdWJevae3|4BfDnokEaH1oSP_2rN}ShW9S|S7vu<ioHzjqr0+A`4
zQMeaSbQ3HC8FRDK1eh!enB+ib0HPfcfeilc;VlY#^>I}ZQ98td^FW}K*K|IF%y2xT
zj_|>5z){}M9PY1HCe-Si#R5n}kD5cT=vPn%iTR>_VTJ(U)71&8$+^6R&+jaJd1s;K
zGqU;w8G>khA`+k4;qxLLeGb(yG==2{wVEdvTAzrnEG#}KnI(!teJC~-W?~QsEr?n$
zF<LMR!ezScD6^O+$%1yhk6rw?^I^og{7n{Zu(k2n2DreOXg7XDl;U$#VY&ZcWBuOd
z!__sJ_5SjvEzlkU6QpTi9e$S3Vc11eSbm)Y*zik)D`PQyt!_Tp*^?!*MBF74N;~(n
z(d*#m&fyzLFV1u-mfhbnMq+O+oxHXudFdViE+l64!QDpge^_2=Ax$FvSUQ;laZ32O
z<Pf%fADa+17M8g?9_%afDcpvnC$Mt!Cn%VkoXnYa-x<E(OcpM?BSR%;PGZuOTXfGj
z6VCL|Ip-W=&<Ntzbm1c6*^D!d9t}!?=ywo<zOp_C77t@g=lI6`7ls~>FbY{u0Ji|^
zrUMXzrz0|V8Ill3U|`=l5@3+CD8PPj=k8G(;U@a-h+X9hKi>C7PW(@F9AxK+2poSc
z;S6snA_|Fb5?7ehuY(o0K%^ehzb8b)0e~A$1}ng)ziJ%$gxZ4xz|HU!1E&HLN5wAS
zD_*ufKv0-C*x<Hs*`&F{Yr)~caUr1t2QtPBs`Z~}t)EfZtgsbjSaEjNGcuuAEA!|!
z#0MB+s^!=d%kq|MEBEes)JE?qu6a$Ua2SEX-cO#2K@jKRHpHyc#@gn}>fI0OEBBXc
zwR<ZscOZt5^}1Ik%=aJi^)+AL;3al)@4;G{CaK?Itzb9CokklG*Z&dgXzo5eiNC{F
zgD>nt`I;|o#s3*!AM=GYtMZTWDpQ-RSIJ;m;%9Um3WEULq?aLh(|jzh<Un$jso^T)
z|2BJ5DMQ^1dbrLB0Hs$Dc<Qf#dWE({?`iSjdKARjhRz`9zeA<Dju#{M2rO*God({U
zW)z<+6~^63=MB_fb{E|9?j)XBv^OYCZiG&H1&={E5~Z$6V3I5Vst{$MU{4AvtupMc
ze-0zFR>X|%;l~WTFbMFhK3-D417#g%LXoxxA))^WHJ!0)7^w{KT3k4)oc)j0mS`QF
z@`l;R`(bT)eeI4H_c}W|#l_XLhas9rm}I<8&t#t7@8N!Z!X<F)6w}j|jy-t9H1lw?
z@^HK!{i^YBQSss=^GsJaqGqJ65|=3o8$E_V=|cn?p1Q^il9cuThzt~#g?Mbtk(&{X
zx9hyyM{`Cr$NvNN5{2Ooqwvsw%pvf+BF}qrBmEMp8Fky*$-}U6t~tX_$(?r^sCqGD
z&U+kA8W;`-i8F@BsY$|n;|0sz%tODlo!Fbr$<wS5T_lZ<{CWBcw@Kt)53{=5WYcV#
zd!&O<E`5N&Q?iIskD>uvf}XVjlAw;VPbe@P>~rNDi)u2xZGos@On6PGFEA{SE4-sk
zaziN4alk$xK4!Fm?A##-dd<#AY|jv_!85RJ7Hw|chEyDqaD%q<r`wQ=LlzEguom>2
zJKbh;)0iHKX|!EA-F8)l_zc=4AARxEoF4PW$PBaNn^^Z?F>k4GK_M@TaCE|fByw4y
zdqI|r&>&YlI75P1yb+x6Mts71O6?-DSon@zzr)}{1nMF(0<wbxDf}Nx1X2z&+k3Mr
z{Tm_9lEBX_wvz9&LL)m58lqr^7t}>5BeH{@SI^-o;-5-kR+)duNBTzx3DEV=;{|3d
z$(fJxk`adG=dfD20;SlHpPu}mfE1Ux!fXiqg#C%7TJ@ML@+4X05iEi1w;`ECC9515
znt!E&ZacjF5t#%HNuSV<2)y(E5R~|kM{E9P4{G-|zdlKjAva_Fiya}ty^wCFrQ2OY
zZ=r)m%S#&kg-A1Jio_}IsRr(A(80T57v*KGH0kZOyfxo{;8(Vm{f&DY>y>*OcOO(%
z*FJl=?v?v-%46L-b_YiCiIs|Y$O=89EP#MtQnoemoUEq{?hrp8z;EiT5jMrk+F`Wd
z`$`4wRR%GyVexByU;9Hn=?N>1d<7C$E0QB*9F?NSI17l8VskPpBHXE2zm@!7r7}n}
zGC_o#Fz(+*#z;EdRO1Y>-n^&xG|E~Yd-RtaClvuU<_cyF+U?|NALzslJC%p&5h$*k
zJ7DmlrbG?K608X^Btj_3u@M8obtxV;v}D}aoCcB6VL!nJJRwOfd$Es$bk|7U@m({|
z`Jr)jt8oJ@WNOS<F(-0Mt5j@=ksCvb02rk@lnv*B$U`68D`GA^uUquYr<kGEN_y?k
z+tFUYgt!Q{9mUWnobN|u=6s*XFOGI4i(lif^5vJ6)m7w7C1op^NfsDkNCiASh}SmM
zio%?Zw>DVsB#=F(9>m<fY)>l6{2f6y2Fb_|zmi*BPInQ1<LtNOr(j8N2VqzTk5mhC
zL!Qz9l)V|8b$k$|NO)4mkks&hj&A{#a7+DPl-b*+QrpSyF7l_cE3qE20a}m{G#6<T
zGm;^Zy!C&9f#gte%ZnOt9?fW1Mi&vtFB7s6*n|`;a`;pBqMxv5XmKUiBnn@r{tHg<
zd0Vn*<j+CbJQaO|teOLe{Eh!-<iqoda0I991W|mJk+sTs04iq-RcFK*EnIQz|C}?1
z<l(4$%^5?Y@fuQ(Glf~_7V5?dB7=IJW7O+%y!MoMGUl1_e~E7qNSB#!<3B=~NZz7&
zxke9u{a$QWt5y;`V{6qPoLmTi(Ynth6X1V>c5%9cz9;v6=)--SI9&Z0xayKzM~;~b
zy8mGP)NSD#)W}7_9vx4_D$=h&oyc6M|My?^xwelwN!V-aPYBZtr<p_!j{o>!AOAwR
Z)1$@F;y1-}#h(^GC{7iJif>Ok{{;%+j->zq
diff --git a/aosstower/level_b1/monthly/nc_monthly.py b/aosstower/level_b1/monthly/nc_monthly.py
index 8fd9c8c..b94edf2 100644
--- a/aosstower/level_b1/monthly/nc_monthly.py
+++ b/aosstower/level_b1/monthly/nc_monthly.py
@@ -1,7 +1,12 @@
import platform
from netCDF4 import Dataset
+from aosstower.l00 import parser
+import calendar
+import logging
+from aosstower.level_b1.daily import nc
from datetime import datetime as dt
from collections import OrderedDict
+import glob
import os
# CF Metadata 1.6 compliant structure for NetCDF file. This is essentially
@@ -824,11 +829,58 @@ AOSS_VARS = OrderedDict({
})
def _dt_convert(date):
- return None
+ return dt.strptime(date, '%Y-%m-%d')
+
+def files_fr_directories(directories,file_ending='.ascii'):
+ """gets a list of all raw data files in the given directories"""
+ files = []
+ for directory in directories:
+ files += glob.glob(os.path.join(directory,'*%s'%file_ending))
+ return files
def createGiantNetCDF(start, end, directories, output, zlib, chunk_size):
+ default = False
+
+ if(chunk_size):
+ chunksizes = [chunk_size]
+
+ else:
+ default = True
+ inputFiles = files_fr_directories(directories)
+
return False
+ frame = getData(inputFiles)
+
+ if(frame.empty):
+ return False
+
+ else:
+
+ frame = minuteAverages(frame)
+
+ if(start and end):
+ frame = frame[start.strftime('%Y-%m-%d %H:%M:%S'): end.strftime('%Y-%m-%d %H:%M:%S')]
+
+ if(default):
+ chunksizes = [len(list(frame.index))]
+
+ firstStamp = dt.strptime(str(list(frame.index)[0]), '%Y-%m-%d %H:%M:%S')
+
+ ncFile = Dataset(outputName, 'w', format='NETCDF4_CLASSIC')
+
+ ncFile = writeDimensions(ncFile)
+
+ ncFile = createVariables(ncFile, firstStamp, chunksizes, zlib)
+
+ ncFile.inputFiles = ', '.join(inputFiles)
+
+ ncFile = writeVars(ncFile, frame)
+
+ ncFile.close()
+
+ return True
+
def writeDimensions(ncFile):
for name, size in BASIC_STATION['dimensions'].items():
ncFile.createDimension(name, size)
@@ -920,12 +972,13 @@ def main():
if(args.start_time and args.end_time):
- result = createGiantNetCDF(args.start_time, args.end_time, args.input_files, args.output[0], args.zlib, args.chunk_size)
+ result = createGiantNetCDF(args.start_time, args.end_time, args.input_directories, args.output[0], args.zlib, args.chunk_size)
if(result == False):
raise IOError('An empty ASCII file was found')
elif(args.start_time):
- end_time = args.start_time.replace(hour=23, minute=59, second=59)
+ last_day = calendar.monthrange(args.start_time.year, args.start_time.month)[1]
+ end_time = args.start_time.replace(day = last_day, hour=23, minute=59, second=59)
result = createGiantNetCDF(args.start_time, end_time, args.input_directories, args.output[0], args.zlib, args.chunk_size)
if(result == False):
raise IOError('An empty ASCII file was found')
@@ -937,9 +990,4 @@ def main():
createMultiple(args.input_files, args.output, args.zlib, args.chunk_size)
if __name__ == "__main__":
- # main()
- ncFile = Dataset('aoss_tower.2013-06.SUMMARY.nc', 'w', format='NETCDF4_CLASSIC')
- ncFile = writeDimensions(ncFile)
- ncFile = createVariables(ncFile, dt(2003, 6, 1), [30], True)
-
- ncFile.close()
+ main()
--
GitLab