Commit b775046a authored by Marco Kurzynski's avatar Marco Kurzynski Committed by David Hoese
Browse files

Rewrite of NCDF and Quicklooks generation

parent 5380ca0e
include aossceilo/ceilo.ncml
include aossceilo/level_b1/ceilo.ncml
<?xml version="1.0" encoding="UTF-8"?>
<netcdf xmlns="http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2">
<dimension name="time" isUnlimited="true" />
<dimension name="range" length="256" />
<dimension name="string_len" length="8" />
<attribute name="Conventions" type="String" value="CF-1.3" />
<attribute name="title" type="String" value="Rooftop Instrument Group(RIG) Ceilometer obervations" />
<attribute name="institution" type="String" value="University of Wisconsin
Space Science and Engineering Center
1225 W. Dayton St.
Madison, WI 53706" />
<attribute name="source" type="String" value="surface observation" />
<attribute name="history" type="String" value="" />
<attribute name="references" type="String" value="http://metobs.ssec.wisc.edu" />
<attribute name="comment" type="String" value="" />
<attribute name="site_id" type="String" value="" />
<attribute name="facility_id" type="String" value="" />
<attribute name="sample_int" type="String" value="15 seconds" />
<attribute name="serial_number" type="String" value="" />
<attribute name="Mentor_QC_Field_Information" type="String" value="For each qc field interpret the values as follows:
Basic mentor QC checks (bit values):
==========================================
0x0 = value is within the specified range
0x1 = value is equal to 'missing_value'
0x2 = value is less than the 'valid_min'
0x4 = value is greater than the 'valid_max'
0x8 = value failed the 'valid_delta' check
If the value is a 'missing_value' no min, max, or delta checks are performed.
The delta checks are done by comparing each data value to the one just
prior to it in time. If a previous data value does not exist or is a
'missing_value' the delta check will not be performed.
Note that the delta computation for multi-dimensioned data compares the
absolute value between points in the same spatial location to the previous
point in time.
If the associated non-QC field does not contain any mentor-specified minimum,
maximum, or delta information a qc_field is not generated.
" />
<variable name="base_time" type="int">
<attribute name="long_name" type="String" value="Time of first record in Epoch" />
<attribute name="units" type="String" value="seconds since 1970-1-1 0:00:00 0:00" />
<attribute name="_FillValue" type="int" value="-9999" />
</variable>
<variable name="time_offset" shape="time" type="int">
<attribute name="long_name" type="String" value="Time offset from base_time" />
<attribute name="units" type="String" value="" />
<attribute name="_FillValue" type="int" value="-9999" />
</variable>
<variable name="time" shape="time" type="int">
<attribute name="long_name" type="String" value="Time offset from midnight" />
<attribute name="units" type="String" value="" />
<attribute name="_FillValue" type="int" value="-9999" />
</variable>
<variable name="range" shape="range" type="double">
<attribute name="long_name" type="String" value="Distance to the center of the corresponding range bin." />
<attribute name="units" type="String" value="m" />
<attribute name="_FillValue" type="double" value="-9999." />
<attribute name="accuracy" type="String" value="30 meters" />
</variable>
<variable name="first_cbh" shape="time" type="double">
<attribute name="long_name" type="String" value="Lowest cloud base height detected." />
<attribute name="standard_name" type="String" value="cloud_base_altitude" />
<attribute name="units" type="String" value="m" />
<attribute name="valid_min" type="double" value="0." />
<attribute name="valid_max" type="double" value="7620." />
<attribute name="_FillValue" type="double" value="-9999." />
<attribute name="valid_min" type="double" value="0" />
<attribute name="valid_max" type="double" value="7620." />
<attribute name="description" type="String" value=" If detection_status = 1, 2 or 3,
lowest cloud base height measured.
If detection_status = 4 this value
will be stored as -9999 and a value
for vertical_visibility will be stored.
If detection_status = 0 or 5, -9999 will
be stored for this value and vertical_visibility.
(5 slashes were read from field)." />
</variable>
<variable name="vertical_visibility" shape="time" type="double">
<attribute name="long_name" type="String" value="Vertical visibility" />
<attribute name="standard_name" type="String" value="visibility_in_air" />
<attribute name="units" type="String" value="m" />
<attribute name="valid_min" type="double" value="0." />
<attribute name="valid_max" type="double" value="7620." />
<attribute name="_FillValue" type="double" value="-9999." />
<attribute name="description" type="String" value="This field will only be stored if detection_status
is = 4. In all other cases, a value of -9999 will
be recorded." />
</variable>
<variable name="second_cbh" shape="time" type="double">
<attribute name="long_name" type="String" value="Second lowest cloud base height" />
<attribute name="standard_name" type="String" value="cloud_base_altitude" />
<attribute name="units" type="String" value="m" />
<attribute name="_FillValue" type="double" value="-9999." />
<attribute name="valid_min" type="double" value="0" />
<attribute name="valid_max" type="double" value="7620." />
<attribute name="description" type="String" value=" If detection_status = 2 or 3, a second
cloud base was detected. This value is
the distance from the ground to the second
cloud base.
If detection_status = 4, -9999 will be recorded
here and a value for alt_highest_signal
will be recorded.
If detection_status = 5, -9999 will be recorded
for this value and for alt_highest_signal" />
</variable>
<variable name="alt_highest_signal" shape="time" type="double">
<attribute name="long_name" type="String" value="Altitude of highest signal" />
<attribute name="units" type="String" value="m" />
<attribute name="_FillValue" type="double" value="-9999." />
<attribute name="valid_min" type="double" value="0" />
<attribute name="valid_max" type="double" value="7620." />
<attribute name="description" type="String" value=" If detection_status = 4, this value is the
distance from the ground to the highest signal
detected.
If detection_status = 0, 1, 2, 3 or 5, -9999
will be recorded." />
</variable>
<variable name="third_cbh" shape="time" type="double">
<attribute name="long_name" type="String" value="Third cloud base height" />
<attribute name="standard_name" type="String" value="cloud_base_altitude" />
<attribute name="units" type="String" value="m" />
<attribute name="_FillValue" type="double" value="-9999." />
<attribute name="valid_min" type="double" value="0" />
<attribute name="valid_max" type="double" value="7620." />
<attribute name="description" type="String" value=" If detection_status = 3, this is the distance
from the ground to the base of the highest
cloud.
If detection = 0, 1, 2, 4 or 5, -9999 will be
stored here. In this case, ///// was read from field." />
</variable>
<variable name="sum_backscatter" shape="time" type="double">
<attribute name="long_name" type="String" value="SUM of detected and normalized backscatter" />
<attribute name="units" type="String" value="1.0 sr-1" />
<attribute name="valid_min" type="double" value="0." />
<attribute name="valid_max" type="double" value="999." />
<attribute name="_FillValue" type="double" value="-9999." />
<attribute name="description" type="String" value="Multiplied by scaling factor
times 10^4. At scaling factor 100 the sum range,
which is 0 to 999, this corresponds to integrated
backscatter, which ranges from 0 to 0.0999 srad^-1" />
</variable>
<variable name="backscatter" shape="time range" type="double">
<attribute name="long_name" type="String" value="Backscatter" />
<attribute name="units" type="String" value="1.0E-7 m-1.sr-1" />
<attribute name="_FillValue" type="double" value="-9999." />
<attribute name="description" type="String" value="Data is range and sensitivity normalized backscatter, units, (10000 - srad - km)^-1 unless otherwise scaled by
scaling parameter.
The length of each range bin corresponds to
(c/2)*(200ns +/14 ns) = (approximately 30 m)." />
</variable>
<variable name="detection_status" shape="time" type="int">
<attribute name="long_name" type="String" value="Detection status." />
<attribute name="_FillValue" type="int" value="-9999" />
<attribute name="valid_min" type="int" value="0" />
<attribute name="valid_max" type="int" value="5" />
<attribute name="description" type="String" value="0=No significant backscatter
1=One cloud base detected
2=Two cloud bases detected
3=Three cloud bases detected
4=Full obscuration determined but no cloud
base detected.
5=Some obscuration detected but determined
to be transparent." />
</variable>
<variable name="status_flag" shape="time" type="char">
<attribute name="long_name" type="String" value="Ceilometer status indicator." />
<attribute name="description" type="String" value="Warning and alarm information as follows
0 = Self-check OK
W = At least one Warning active, no Alarms
A = At least one Alarm active" />
</variable>
<variable name="laser_pulse_energy" shape="time" type="double">
<attribute name="long_name" type="String" value="Laser pulse energy" />
<attribute name="units" type="String" value="%" />
<attribute name="valid_min" type="double" value="0." />
<attribute name="valid_max" type="double" value="999." />
<attribute name="_FillValue" type="double" value="-9999." />
<attribute name="comment" type="String" value="Percentage of nominal factory setting." />
</variable>
<variable name="laser_temperature" shape="time" type="double">
<attribute name="long_name" type="String" value="Laser temperature" />
<attribute name="units" type="String" value="degC" />
<attribute name="valid_min" type="double" value="-50." />
<attribute name="valid_max" type="double" value="99." />
<attribute name="_FillValue" type="double" value="-9999." />
</variable>
<variable name="receiver_sensitivity" shape="time" type="double">
<attribute name="long_name" type="String" value="Receiver sensitivity" />
<attribute name="units" type="String" value="%" />
<attribute name="valid_min" type="double" value="0." />
<attribute name="valid_max" type="double" value="999." />
<attribute name="_FillValue" type="double" value="-9999." />
<attribute name="comment" type="String" value="Percentage of nominal factory setting." />
</variable>
<variable name="window_contamination" shape="time" type="double">
<attribute name="long_name" type="String" value="Window contamination" />
<attribute name="units" type="String" value="0.0010 V" />
<attribute name="valid_min" type="double" value="0." />
<attribute name="valid_max" type="double" value="2500." />
<attribute name="_FillValue" type="double" value="-9999." />
<attribute name="comment" type="String" value="Millivolts measured at the internal ADC input" />
</variable>
<variable name="tilt_angle" shape="time" type="double">
<attribute name="long_name" type="String" value="Tilt angle" />
<attribute name="units" type="String" value="degrees" />
<attribute name="valid_min" type="double" value="-15." />
<attribute name="valid_max" type="double" value="90." />
<attribute name="valid_delta" type="double" value="1." />
<attribute name="_FillValue" type="double" value="-9999." />
<attribute name="description" type="String" value="If Angle Correction is Off detection values are distances not altitudes" />
</variable>
<variable name="background_light" shape="time" type="double">
<attribute name="long_name" type="String" value="Background light" />
<attribute name="units" type="String" value="0.0010 V" />
<attribute name="valid_min" type="double" value="0." />
<attribute name="valid_max" type="double" value="2500." />
<attribute name="_FillValue" type="double" value="-9999." />
<attribute name="comment" type="String" value="Millivolts measured at the internal ADC input" />
</variable>
<variable name="measurement_parameters" shape="time string_len" type="char">
<attribute name="long_name" type="String" value="6 character string describing instrument measurement parameters" />
<attribute name="comment" type="String" value="(L)ong/(S)hort - (F)req - pulse qty r ^(7) + 1
- gain (H)igh/(L)ow - bandwidth (N)arrow/(W)ide -
sampling (5) / (1)0 / (2)0 Mhz" />
</variable>
<variable name="status_string" shape="time string_len" type="char">
<attribute name="long_name" type="String" value="Warning and alarm status bits" />
<attribute name="comment1" type="String" value="Has Form FEDCBA98. Each character is a hexadecimal representation of
four bits, altogether 32 bits. See additional comments for interpretation." />
<attribute name="commentF" type="String" value="(8000 0000) = Laser temperature shut-off(A)
(4000 0000)= Laser failure(A)
(2000 0000)= Receiver failure(A)
(1000 0000)= Voltage failure(A)" />
<attribute name="commentE" type="String" value="(0800 0000) = Profile error(A)" />
<attribute name="commentD" type="String" value="(0080 0000) = Window contaminated(W)
(0040 0000)= Battery low(W)
(0020 0000)= Laser power low(W)
(0010 0000)= Laser temperature high or low(W)" />
<attribute name="commentC" type="String" value="(0008 0000) = Internal temperature high or low(W)
(0004 0000)= Voltage high or low(W)
(0002 0000)= Relative humidity is > 85 % (W)
(0001 0000)= Receiver optical cross-talk compensation poor(W)" />
<attribute name="commentB" type="String" value="(0000 8000) = Fan suspect(W)
(0000 4000)= Profile warning(W)" />
<attribute name="commentA" type="String" value="(0000 0800) = Blower is ON
(0000 0400)= Blower heater is ON
(0000 0200)= Internal heater is ON
(0000 0100)= Units are METERS if ON, else FEET." />
<attribute name="comment9" type="String" value="(0000 0080) = Polling mode is ON
(0000 0040)= Working from battery
(0000 0020)= Single sequence mode is ON
(0000 0010)= Manual Settings are effective." />
<attribute name="comment8" type="String" value="(0000 0008) = Tilt angle is > 45 degrees." />
</variable>
<variable name="lat" type="double">
<attribute name="standard_name" type="String" value="latitude" />
<attribute name="long_name" type="String" value="north latitude" />
<attribute name="units" type="String" value="degrees_north" />
<attribute name="valid_min" type="double" value="-90." />
<attribute name="valid_max" type="double" value="90." />
<attribute name="_FillValue" type="double" value="-9999." />
</variable>
<variable name="lon" type="double">
<attribute name="standard_name" type="String" value="longitude" />
<attribute name="long_name" type="String" value="east longitude" />
<attribute name="units" type="String" value="degrees_east" />
<attribute name="valid_min" type="double" value="-180." />
<attribute name="valid_max" type="double" value="180." />
<attribute name="_FillValue" type="double" value="-9999." />
</variable>
<variable name="alt" type="double">
<attribute name="standard_name" type="String" value="altitude" />
<attribute name="long_name" type="String" value="altitude" />
<attribute name="positive" type="String" value="up" />
<attribute name="units" type="String" value="m" />
<attribute name="_FillValue" type="double" value="-9999." />
</variable>
<variable name="qc_time" shape="time" type="int">
<attribute name="long_name" type="String" value="Results of quality checks on sample time" />
<attribute name="description" type="String" value="The qc_time values are calculated by comparing each sample
time with the previous time (i.e. delta_t = t[n] - t[n-1]).
If the 'qc_check_prior' flag is set the first sample time
from a new raw file will be compared against the time just
previous to it in the stored data. If the 'qc_check_prior'
flag is not set the qc_time value for the first sample time
will be set to 0
The qc_time bit values are as follows:
=========================================================
0x0 = delta time is within the specified range
0x1 = delta time is equal to 0, duplicate sample times
0x2 = delta time is less than the 'delta_t_lower_limit'
0x4 = delta time is greater than the 'delta_t_upper_limit'
" />
<attribute name="delta_t_lower_limit" type="int" value="12" />
<attribute name="delta_t_upper_limit" type="int" value="18" />
<attribute name="prior_sample_flag" type="int" value="1" />
<attribute name="_FillValue" type="int" value="-9999" />
</variable>
<variable name="qc_first_cbh" shape="time" type="int">
<attribute name="long_name" type="String" value="Quality check results on field: Lowest cloud base height detected." />
<attribute name="_FillValue" type="int" value="-9999" />
</variable>
<variable name="qc_vertical_visibility" shape="time" type="int">
<attribute name="long_name" type="String" value="Quality check results on field: Vertical visibility" />
<attribute name="_FillValue" type="int" value="-9999" />
</variable>
<variable name="qc_second_cbh" shape="time" type="int">
<attribute name="long_name" type="String" value="Quality check results on field: Second lowest cloud base height" />
<attribute name="_FillValue" type="int" value="-9999" />
</variable>
<variable name="qc_alt_highest_signal" shape="time" type="int">
<attribute name="long_name" type="String" value="Quality check results on field: Altitude of highest signal" />
<attribute name="_FillValue" type="int" value="-9999" />
</variable>
<variable name="qc_third_cbh" shape="time" type="int">
<attribute name="long_name" type="String" value="Quality check results on field: Third cloud base height" />
<attribute name="_FillValue" type="int" value="-9999" />
</variable>
<variable name="qc_laser_pulse_energy" shape="time" type="int">
<attribute name="long_name" type="String" value="Quality check results on field: Laser pulse energy" />
<attribute name="_FillValue" type="int" value="-9999" />
</variable>
<variable name="qc_laser_temperature" shape="time" type="int">
<attribute name="long_name" type="String" value="Quality check results on field: Laser temperature" />
<attribute name="_FillValue" type="int" value="-9999" />
</variable>
<variable name="qc_tilt_angle" shape="time" type="int">
<attribute name="long_name" type="String" value="Quality check results on field: Tilt angle" />
<attribute name="_FillValue" type="int" value="-9999" />
</variable>
</netcdf>
"""
For processing Ceilometer CT25K Messages
"""
import logging
from calendar import timegm
from datetime import datetime
from numpy import array
LOG = logging.getLogger(__name__)
#: Value to use for NetCDF missing_value attribute
missing_value = -9999
#: Function to determine if a value is a missing_value or not
def is_missing(val):
return val == missing_value
#: number of allowable missing values before backscatter is considered invalid
BK_THRESH = 256 * 0.25
START_OF_HEADING = "\x01"
START_OF_TEXT = "\x02"
END_OF_TEXT = "\x03"
MANUAL_BLOWER_CONTROL = 2
HIGH_BACKGROUND_RADIANCE = 4
TILT_ANGLE_45_PLUS = 8
MANUAL_SETTINGS_EFF = 16
SINGLE_SEQ_ON = 32
WORKING_FROM_BATT = 64
POLLING_ON = 128
UNITS_IN_METERS = 256
INTERNAL_HEATER_ON = 512
BLOWER_HEATER_ON = 1024
BLOWER_ON = 2048
BLOWER_SUSPECT = 32768
RCV_OPTICAL_XTLK_POOR = 65536
REL_HUMIDITY_HIGH = 131072
VOLTAGE_ABNORMAL = 262144
INT_TEMP_HIGH = 524288
LASER_TEMP_HIGH = 1048576
LASER_PWR_LOW = 2097152
BATTERY_LOW = 4194304
WINDOW_CONTAMINATED = 8388608
VOLTAGE_FAILURE = 268435456
RECEIVER_FAILURE = 536870912
LASER_FAILURE = 1073741824
LASER_TEMP_SHUTOFF = 2147483648
WARNING_FLAGS = [
BLOWER_SUSPECT,
RCV_OPTICAL_XTLK_POOR,
REL_HUMIDITY_HIGH,
VOLTAGE_ABNORMAL,
INT_TEMP_HIGH,
LASER_TEMP_HIGH,
LASER_PWR_LOW,
BATTERY_LOW,
WINDOW_CONTAMINATED,
]
WARNING_MSGS = [
"Blower suspect",
"Receiver optical cross-talk poor",
"Relative humidity is greater than 85%",
"Voltage is low or high",
"Internal temperature is high or low",
"Laser temperature is high",
"Laser power is low",
"Battery is low",
"Window is contaminated",
]
INFO_FLAGS = [
MANUAL_BLOWER_CONTROL,
HIGH_BACKGROUND_RADIANCE,
TILT_ANGLE_45_PLUS,
MANUAL_SETTINGS_EFF,
SINGLE_SEQ_ON,
WORKING_FROM_BATT,
POLLING_ON,
UNITS_IN_METERS,
INTERNAL_HEATER_ON,
BLOWER_HEATER_ON,
BLOWER_ON,
]
INFO_MSGS = [
"Using manual blower control",
"Background radiance is high",
"Tilt angle is greater than 45 degrees",
"Manual settings are effective",
"Single sequence mode is ON",
"Working from the battery",
"Polling mode is ON",
"Units are in Meters",
"Internal heater is ON",
"Blower heater is ON",
"Blower is ON",
]
ALARM_FLAGS = [
VOLTAGE_FAILURE,
RECEIVER_FAILURE,
LASER_FAILURE,
LASER_TEMP_SHUTOFF,
]
ALARM_MSGS = [
"VOLTAGE FAILURE",
"RECEIVER FAILURE",
"LASER FAILURE",
"LASER TEMPERATURE SHUT_OFF",
]
class MessageError(RuntimeError):
"""General message error."""
class Message2(object):
NUM_LINES = 20
def __init__(
self, lines, stamp,
):
"""Initialize this message, disecting the various message components
according to the CT25K Message 2 format.
The first line in a message is the header and it must start with a
SOH and end with SOT, and the entire message must end with EOT
ASCII chars to be valid. Assertion errors are raised if these
conditions are not met.
The backscatter is initially filled with L{missing_value} so if a
line is the incorrect length all of it's values will be missing_value.
Likewise, if a value cannot be parsed as a HEX string it's value will
be missing_value.
@param lines: Exactly the 20 lines that comprise a valid message
@type lines: list
@param stamp: The message time. If the 'stamp' is a naieve datetime the
tzinfo will be set to metobs.util.time.UTC, otherwise time operations
will proceed with provided tzinfo.
@type stamp: datetime
@raises MessageError: If this instance cannot be created due to an error
parsing.
"""
assert len(lines) == self.NUM_LINES, (
"A Message2 must contain %s lines" % self.NUM_LINES
)
self._epoch = timegm(stamp.utctimetuple())
self._lines = lines
self._stamp = stamp
self._header = lines[0]
# strip non-printables
self._header = self._header.replace(START_OF_HEADING, "",)
self._header = self._header.replace(START_OF_TEXT, "",)
self._msg_num = int(self.header[5])
if self._msg_num != 2:
raise MessageError(
"Invalid message number", self.header,
)
self._status_string = lines[1][21:29]
self._status_flag = lines[1][1]
self._detection_status = _int(lines[1][0])
self._first_cbh = missing_value
self._second_cbh = missing_value
self._third_cbh = missing_value
if self.detection_status == 3:
self._third_cbh = _int(lines[1][15:20])
if self.detection_status == 2:
self._second_cbh = _int(lines[1][9:14])
if self.detection_status == 1:
self._first_cbh = _int(lines[1][3:8])
if self.detection_status == 4:
self._vertical_visibility = _int(lines[1][3:8])
self._alt_highest_signal = _int(lines[1][9:14])
else:
self._vertical_visibility = missing_value
self._alt_highest_signal = missing_value
meas_params = lines[2].split()
if len(meas_params) < 10:
LOG.warn(
"Invalid measurement parameters for message with time %s",
self.epoch,
)
self._scale = _int(meas_params[0])
self._measurement_mode = _str(meas_params[1])
self._laser_pulse_energy = _float(meas_params[2])
self._laser_temperature = _float(meas_params[3])