-
Coda Phillips authoredCoda Phillips authored
parse_isd.py 39.21 KiB
import numpy as np
import pandas as pd
from datetime import datetime
import re
splits = {
'total_char':1,
'usaf_id':5,
'ncei_id':11,
'date':16,
'time':24,
'source':28,
'latitude':29,
'longitude':35,
'obs_code':42,
'elevation':47,
'call':52,
'qc_name':57,
'wind_dir':61,
'wind_qc':64,
'wind_type':65,
'wind_spd':66,
'wind_spd_qc':70,
'ceiling':71,
'ceiling_qc':76,
'ceiling_method':77,
'ceiling_cavok':78,
'visibility':79,
'visibility_qc':85,
'visibility_var':86,
'visibility_var_qc':87,
'air_temp':88,
'air_temp_qc':93,
'air_dpt':94,
'air_dpt_qc':99,
'slp':100,
'slp_qc':105,
'additional':106
}
additional_data = {
'AA[1-4]':'liquid_episode',
'AB1':'liquid_month',
'AC1':'precip_history',
'AD1':'precip_max24h',
'AE1':'liquid_days',
'AG1':'precip_estim',
'AH[1-6]':'liquid_max_short',
'AI[1-6]':'liquid_max_short2',
'AJ1':'snow_depth',
'AK1':'max_snow_depth',
'AL1':'snow_acc',
'AM1':'snow_acc_month',
'AN1':'snow',
'AO[1-4]':'liquid_precip_occur',
'AP[1-4]':'',
'AT[1-8]':'present_weather',
'AU[1-9]':'',
'AW[1-4]':'present_weather',
'AX[1-6]':'past_weather',
'AY[1-2]':'past_weather',
'AZ[1-2]':'past_weather',
}
aw_codes = {
'00':'No significant weather observed',
'01':'Clouds generally dissolving or becoming less developed',
'02':'State of sky on the whole unchanged during the past hour',
'03':'Clouds generally forming or developing during the past hour',
'04':'Haze, smoke, or dust in suspension in the air, visibility equal to or greater than 1km',
'05':'Smoke',
'07':'Dust or sand raised by wind at or near the station at the time of observation, but no well-developed dust irl(s)whirls(s) or sand whirl(s), and no duststorm or sandstorm seen or, in the case of ships, blowing spray at thestation',
'10':'Mist',
'11':'Diamond dust',
'12':'Distant lightning',
'18':'Squalls',
'20':'Fog',
'21':'Precipitation',
'22':'Drizzle (not freezing) or snow grains',
'23':'Rain (not freezing)',
'24':'Snow',
'25':'Freezing drizzle or freezing rain',
'26':'Thunderstorm (with or without precipitation)',
'27':'Blowing or drifting snow or sand',
'28':'Blowing or drifting snow or sand, visibility equal to or greater than 1 km',
'29':'Blowing or drifting snow or sand, visibility less than 1 km',
'30':'Fog',
'31':'Fog or ice fog in patches',
'32':'Fog or ice fog, has become thinner during the past hour',
'33':'Fog or ice fog, no appreciable change during the past hour',
'34':'Fog or ice fog, has begun or become thicker during the past hour',
'35':'Fog, depositing rime',
'40':'Precipitation',
'41':'Precipitation, slight or moderate',
'42':'Precipitation, heavy',
'43':'Liquid precipitation, slight or moderate',
'44':'Liquid precipitation, heavy',
'45':'Solid precipitation, slight or moderate',
'46':'Solid precipitation, heavy',
'47':'Freezing precipitation, slight or moderate',
'48':'Freezing precipitation, heavy',
'50':'Drizzle',
'51':'Drizzle, not freezing, slight',
'52':'Drizzle, not freezing, moderate',
'53':'Drizzle, not freezing, heavy',
'54':'Drizzle, freezing, slight',
'55':'Drizzle, freezing, moderate',
'56':'Drizzle, freezing, heavy',
'57':'Drizzle and rain, slight',
'58':'Drizzle and rain, moderate or heavy',
'60':'Rain',
'61':'Rain, not freezing, slight',
'62':'Rain, not freezing, moderate',
'63':'Rain, not freezing, heavy',
'64':'Rain, freezing, slight',
'65':'Rain, freezing, moderate',
'66':'Rain, freezing, heavy',
'67':'Rain or drizzle and snow, slight',
'68':'Rain or drizzle and snow, moderate or heavy',
'70':'Snow',
'71':'Snow, slight',
'72':'Snow, moderate',
'73':'Snow, heavy',
'74':'Ice pellets, slight',
'75':'Ice pellets, moderate',
'76':'Ice pellets, heavy',
'77':'Snow grains',
'78':'Ice crystals',
'80':'Showers or intermittent precipitation',
'81':'Rain showers or intermittent rain, slight',
'82':'Rain showers or intermittent rain, moderate',
'83':'Rain showers or intermittent rain, heavy',
'84':'Rain showers or intermittent rain, violent',
'85':'Snow showers or intermittent snow, slight',
'86':'Snow showers or intermittent snow, moderate',
'87':'Snow showers or intermittent snow, heavy',
'89':'Hail',
'90':'Thunderstorm',
'91':'Thunderstorm, slight or moderate, with no precipitation',
'92':'Thunderstorm, slight or moderate, with rain showers and/or snow showers',
'93':'Thunderstorm, slight or moderate, with hail',
'94':'Thunderstorm, heavy, with no precipitation',
'95':'Thunderstorm, heavy, with rain showers and/or snow',
'96':'Thunderstorm, heavy, with hail',
'99':'Tornado'
}
AW_CODE_CAT = pd.CategoricalDtype(categories=sorted(set(aw_codes.values())), ordered=False)
at_codes = {
'01':'Fog, ice fog or freezing fog (may include heavy fog)',
'02':'Heavy fog or heavy freezing fog (not always distinguished from fog)',
'03':'Thunder',
'04':'Ice pellets, sleet, snow pellets or small hail',
'05':'Hail (may include small hail)',
'06':'Glaze or rime',
'07':'Dust, volcanic ash, blowing dust, blowing sand or blowing obstruction',
'08':'Smoke or haze',
'09':'Blowing or drifting snow',
'10':'Tornado, water spout or funnel cloud',
'11':'High or damaging winds',
'12':'Blowing spray',
'13':'Mist',
'14':'Drizzle',
'15':'Freezing drizzle',
'16':'Rain',
'17':'Freezing rain',
'18':'Snow, snow pellets, snow grains or ice crystals',
'19':'Unknown precipitation',
'20':'Ground fog',
'22':'Ice fog or freezing fog'
}
AT_CODE_CAT = pd.CategoricalDtype(categories=sorted(at_codes.values()), ordered=False)
au_int_code = {
'0':'Not Reported',
'1':'Light (-)',
'2':'Moderate or Not Reported (no entry in original observation)',
'3':'Heavy (+)',
'4':'Vicinity (VC)',
'9':'Missing'
}
AU_INT_CAT = pd.CategoricalDtype(categories=sorted(au_int_code.values()), ordered=False)
au_des_code = {
'0':'No Descriptor',
'1':'Shallow (MI)',
'2':'Partial (PR)',
'3':'Patches (BC)',
'4':'Low Drifting (DR)',
'5':'Blowing (BL)',
'6':'Shower(s) (SH)',
'7':'Thunderstorm (TS)',
'8':'Freezing (FZ)',
'9':'Missing'
}
AU_DES_CAT = pd.CategoricalDtype(categories=sorted(set(au_des_code.values())), ordered=False)
au_precip_code = {
'00':'No Precipitation',
'01':'Drizzle (DZ)',
'02':'Rain (RA)',
'03':'Snow (SN)',
'04':'Snow Grains (SG)',
'05':'Ice Crystals (IC)',
'06':'Ice Pellets (PL)',
'07':'Hail (GR)',
'08':'Small Hail and/or Snow Pellets (GS)',
'09':'Unknown Precipitation (UP)',
'99':'Missing'
}
AU_PCP_CAT = pd.CategoricalDtype(categories=sorted(set(au_precip_code.values())), ordered=False)
au_obsc_code = {
'0':'No Obscuration',
'1':'Mist (BR)',
'2':'Fog (FG)',
'3':'Smoke (FU)',
'4':'Volcanic Ash (VA)',
'5':'Widespread Dust (DU)',
'6':'Sand (SA)',
'7':'Haze (HZ)',
'8':'Spray (PY)',
'9':'Missing'
}
AU_OBSC_CAT = pd.CategoricalDtype(categories=sorted(set(au_obsc_code.values())), ordered=False)
au_other_code = {
'0':'None Reported',
'1':'Well-Developed Dust/Sand Whirls (PO)',
'2':'Squalls (SQ)',
'3':'Funnel Cloud, Tornado, Waterspout (FC)',
'4':'Sandstorm (SS)',
'5':'Duststorm (DS)',
'9':'Missing'
}
AU_OTHER_CAT = pd.CategoricalDtype(categories=sorted(set(au_other_code.values())), ordered=False)
au_combo_code = {
'1':'Not part of combined weather elements',
'2':'Beginning element of combined weather elements',
'3':'Combined with previous weather element to form a single weather report',
'9':'Missing'
}
AU_COMBO_CAT = pd.CategoricalDtype(categories=sorted(set(au_combo_code.values())), ordered=False)
ay_code = {
'0':'Cloud covering 1/2 or less of the sky throughout the appropriate period',
'1':'Cloud covering more than ½ of the sky duringpart of the appropriate period and covering ½ or less during part of the period',
'2':'Cloud covering more than 1/2 of the sky throughout the appropriate period',
'3':'Sandstorm, duststorm or blowing snow',
'4':'Fog or ice fog or thick haze',
'5':'Drizzle',
'6':'Rain',
'7':'Snow, or rain and snow mixed',
'8':'Shower(s)',
'9':'Thunderstorm(s) with or without precipitation'
}
AY_CODE_CAT = pd.CategoricalDtype(categories=sorted(set(ay_code.values())), ordered=False)
mw_code = {
'00':'Cloud development not observed or not observable',
'01':'Clouds generally dissolving or becoming less developed',
'02':'State of sky on the whole unchanged',
'03':'Clouds generally forming or developing',
'04':'Visibility reduced by smoke, e.g. veldt or forest fires, industrial smoke or volcanic ashes',
'05':'Haze',
'06':'Widespread dust in suspension in the air, not raised by wind at or near the station at the time of observation',
'07': 'Dust or sand raised by wind at or near the station at the time of observation, but no well-developed dust whirl(s) sand whirl(s), and no duststorm or sandstorm seen or, in the case of ships, blowing spray at the station',
'08':' Well developed dust whirl(s) or sand whirl(s) seen at or near the station during the preceding hour or at the time of observation, but no duststorm or sandstorm',
'09':'Duststorm or sandstorm within sight at the time of observation, or at the station during the preceding hour',
'10':'Mist',
'11':'Patches of shallow fog or ice fog at the station, whether on land or sea, not deeper than about 2 meters on land or 10 meters at sea',
'12':'More or less continuous shallow fog or ice fog at the station, whether on land or sea, not deeper than about 2 meters on land or 10 meters at sea',
'13':'Lightning visible, no thunder heard',
'14':'Precipitation within sight, not reaching the ground or the surface of the sea',
'15':' Precipitation within sight, reaching the ground or the surface of the sea, but distant, i.e., estimated to be more than 5 km from the station',
'16':'Precipitation within sight, reaching the ground or the surface of the sea, near to, but not at the station',
'17':'Thunderstorm, but no precipitation at the time of observation',
'18':'Squalls at or within sight of the station during the preceding hour or at the time of observation',
'19':'Funnel cloud(s) (Tornado cloud or waterspout) at or within sight of the station during the preceding hour or at the time of observation',
'20':'Drizzle (not freezing) or snow grains not falling as shower(s)',
'21':'Rain (not freezing) not falling as shower(s)',
'22':'Snow not falling as shower(s)',
'23':'Rain and snow or ice pellets not falling as shower(s)',
'24':'Freezing drizzle or freezing rain not falling as shower(s)',
'25':'Shower(s) of rain',
'26':'Shower(s) of snow or of rain and snow',
'27':'Shower(s) of hail (Hail, small hail, snow pellets), or rain and hail',
'28':'Fog or ice fog',
'29':'Thunderstorm (with or without precipitation)',
'30':'Slight or moderate duststorm or sandstorm has decreased during the preceding hour',
'31':'Slight or moderate duststorm or sandstorm no appreciable change during the preceding hour',
'32':'Slight or moderate duststorm or sandstorm has begun or has increased during the preceding hour',
'33':'Severe duststorm or sandstorm has decreased during the preceding hour',
'34':'Severe duststorm or sandstorm no appreciable change during the preceding hour',
'35':'Severe duststorm or sandstorm has begun or has increased during the preceding hour',
'36':'Slight or moderate drifting snow generally low (below eye level)',
'37':'Heavy drifting snow generally low (below eye level)',
'38':'Slight or moderate blowing snow generally high (above eye level)',
'39':'Heavy blowing snow generally high (above eye level)',
'40':'Fog or ice fog at a distance at the time of observation, but not at the station during the preceding hour, the fog or ice fog extending to a level above that of the observer',
'41':'Fog or ice fog in patches',
'42':'Fog or ice fog, sky visible, has become thinner during the preceding hour',
'43':'Fog or ice fog, sky invisible, has become thinner during the preceding hour',
'44':'Fog or ice fog, sky visible, no appreciable change during the preceding hour',
'45':'Fog or ice fog, sky invisible, no appreciable change during the preceding hour',
'46':'Fog or ice fog, sky visible, has begun or has become thicker during the preceding hour',
'47':'Fog or ice fog, sky invisible, has begun or has become thicker during the preceding hour',
'48':'Fog, depositing rime, sky visible',
'49':'Fog, depositing rime, sky invisible',
'50':'Drizzle, not freezing, intermittent, slight at time of observation',
'51':'Drizzle, not freezing, continuous, slight at time of observation',
'52':'Drizzle, not freezing, intermittent, moderate at time of observation',
'53':'Drizzle, not freezing, continuous, moderate at time of observation',
'54':'Drizzle, not freezing, intermittent, heavy (dense) at time of observation',
'55':'Drizzle, not freezing, continuous, heavy (dense) at time of observation',
'56':'Drizzle, freezing, slight',
'57':'Drizzle, freezing, moderate or heavy (dense)',
'58':'Drizzle and rain, slight',
'59':'Drizzle and rain, moderate or heavy',
'60':'Rain, not freezing, intermittent, slight at time of observation',
'61':'Rain, not freezing, continuous, slight at time of observation',
'62':'Rain, not freezing, intermittent, moderate at time of observation',
'63':'Rain, not freezing, continuous, moderate at time of observation',
'64':'Rain, not freezing, intermittent, heavy at time of observation',
'65':'Rain, not freezing, continuous, heavy at time of observation',
'66':'Rain, freezing, slight',
'67':'Rain, freezing, moderate or heavy',
'68':'Rain or drizzle and snow, slight',
'69':'Rain or drizzle and snow, moderate or heavy',
'70':'Intermittent fall of snowflakes, slight at time of observation',
'71':'Continuous fall of snowflakes, slight at time of observation',
'72':'Intermittent fall of snowflakes, moderate at time of observation',
'73':'Continuous fall of snowflakes, moderate at time of observation',
'74':'Intermittent fall of snowflakes, heavy at time of observation',
'75':'Continuous fall of snowflakes, heavy at time of observation',
'76':'Diamond dust (with or without fog)',
'77':'Snow grains (with or without fog)',
'78':'Isolated star-like snow crystals (with or without fog)',
'79':'Ice pellets',
'80':'Rain shower(s), slight',
'81':'Rain shower(s), moderate or heavy',
'82':'Rain shower(s), violent',
'83':'Shower(s) of rain and snow mixed, slight',
'84':'Shower(s) of rain and snow mixed, moderate or heavy',
'85':'Show shower(s), slight',
'86':'Snow shower(s), moderate or heavy',
'87':'Shower(s) of snow pellets or small hail, with or without rain or rain and snow mixed, slight',
'88':'Shower(s) of snow pellets or small hail, with or without rain or rain and snow mixed, moderate or heavy',
'89':'Shower(s) of hail (hail, small hail, snow pellets), with or without rain or rain and snow mixed, not associated with thunder, slight',
'90':'Shower(s) of hail (hail, small hail, snow pellets), with or without rain or rain and snow mixed, not associated with thunder, moderate or heavy',
'91':'Slight rain at time of observation, thunderstorm during the preceding hour but not at time of observation',
'92':'Moderate or heavy rain at time of observation, thunderstorm during the preceding hour but not at time of observation',
'93':'Slight snow, or rain and snow mixed or hail (Hail, small hail, snow pellets), at time of observation, thunderstorm during the preceding hour but not at time of observation',
'94':'Moderate or heavy snow, or rain and snow mixed or hail(Hail, small hail, snow pellets) at time of observation, thunderstorm during the preceding hour but not at time of observation',
'95':'Thunderstorm, slight or moderate, without hail (Hail, small hail, snow pellets), but with rain and/or snow at time of observation, thunderstorm at time of observation',
'96':'Thunderstorm, slight or moderate, with hail (hail, small hail, snow pellets) at time of observation, thunderstorm at time of observation',
'97':'Thunderstorm, heavy, without hail (Hail, small hail, snow pellets), but with rain and/or snow at time of observation, thunderstorm at time of observation',
'98':'Thunderstorm combined with duststorm or sandstorm at time of observation, thunderstorm at time of observation',
'99':'Thunderstorm, heavy, with hail (Hail, small hail, snow pellets) at time of observation, thunderstorm at time of observation'
}
MW_CODE_CAT = pd.CategoricalDtype(categories=sorted(set(mw_code.values())), ordered=False)
gd_codes1 = {
'0':'Clear',
'1':'FEW',
'2':'SCATTERED',
'3':'BROKEN',
'4':'OVERCAST',
'5':'OBSCURED',
'6':' PARTIALLY OBSCURED',
'9':'MISSING'
}
GD_CODES1_CAT = pd.CategoricalDtype(categories=sorted(set(gd_codes1.values())), ordered=False)
gd_codes2 = {
'00':'None',
'01':'One okta - 1/10 or less but not zero',
'02':'Two oktas - 2/10 - 3/10, or FEW',
'03':'Three oktas - 4/10',
'04':'Four oktas - 5/10, or SCT',
'05':'Five oktas - 6/10',
'06':'Six oktas - 7/10 - 8/10',
'07':'Seven oktas - 9/10 or more but not 10/10, or BKN',
'08':'Eight oktas - 10/10, or OVC',
'09':'Sky obscured, or cloud amount cannot be estimated',
'10':'Partial Obscuration',
'11':'Thin Scattered',
'12':'Scattered',
'13':'Dark Scattered',
'14':'Thin Broken',
'15':'Broken',
'16':'Dark Broken',
'17':'Thin Overcast',
'18':'Overcast',
'19':'Dark overcast',
'99':'Missing'
}
GD_CODES2_CAT = pd.CategoricalDtype(categories=sorted(set(gd_codes2.values())), ordered=False)
ga_codes = {
'00':'None, SKC or CLR',
'01':'One okta - 1/10 or less but not zero',
'02':'Two oktas - 2/10 - 3/10, or FEW',
'03':'Three oktas - 4/10',
'04':'Four oktas - 5/10, or SCT',
'05':'Five oktas - 6/10',
'06':'Six oktas - 7/10 - 8/10',
'07':'Seven oktas - 9/10 or more but not 10/10, or BKN',
'08':'Eight oktas - 10/10, or OVC',
'09':'Sky obscured, or cloud amount cannot be estimated',
'10':'Partial obscuration',
'99':'Missing'
}
GA_CODES_CAT = pd.CategoricalDtype(categories=sorted(set(ga_codes.values())), ordered=False)
QC_CAT = pd.CategoricalDtype(categories=['0','1','2','3','4','5','6','7','8','M','9'])
def split_line_mandatory(line):
start = None
old_k = None
slices = {}
for new_k,v in splits.items():
v = v-1
if start is not None:
slices[old_k] = line[start:v]
start = v
old_k = new_k
slices[old_k] = line[start:]
return slices
def parse_fields(fields):
def nan_if_equal(i,f):
if i == f:
return np.nan
return i
date = fields.pop('date')
time = fields.pop('time')
fields['dt'] = datetime.strptime(f'{date}{time}','%Y%m%d%H%M')
lat = fields.pop('latitude')
if lat == '+99999':
fields['lat'] = np.nan
else:
fields['lat'] = float(lat) / 1000
lon = fields.pop('longitude')
if lon == '+999999':
fields['lon'] = np.nan
else:
fields['lon'] = float(lon) / 1000
fields['elevation'] = float(fields['elevation'])
fields['wind_dir'] = float(fields['wind_dir'])
fields['wind_spd'] = float(fields['wind_spd']) / 10
fields['visibility'] = float(fields['visibility'])
if (fields['visibility'] == 999999) or (fields['visibility'] == 9999):
fields['visibility'] = np.nan
for k in ['air_dpt','air_temp']:
if int(fields[k]) == 9999:
fields[k] = np.nan
else:
fields[k] = float(fields[k])/10
return fields
def read_mandatory(f):
with open(f) as fp:
return _read_mandatory(fp)
def _read_mandatory(fp):
df = []
for l in fp:
fields = split_line_mandatory(l)
parse_fields(fields)
df.append(fields)
df = pd.DataFrame(df)
df['visibility'] /= 1000
return df
def get_aw(x):
s = x.getvalue()
i = 0
skipped = 0
aw = []
gex = re.compile('AW1([0-9]{2})([0-9])')
for l in s.splitlines():
cat = np.nan
splits = l.split('ADD')
if len(splits) == 2:
_,add = splits
match = gex.search(add)
if match:
code,qc = gex.search(add).groups()
if qc in ['0','1','4','5','M','9']:
cat = aw_codes[code]
aw.append(cat)
return pd.Series(aw, dtype='category')
def get_at1(x):
s = x.getvalue()
at1 = []
for l in s.splitlines():
splits = l.split('ADD')
cat = np.nan
if len(splits)==2:
_,add = splits
match = re.compile('AT1(AU|AW|MW)([0-9]{2})(.{4})([0-9M])').search(add)
if match:
#counts[k] += 1
source,code,code2,qc = match.groups()
if qc in ['0','1','4','5','M','9']:
cat = at_codes[code]
at1.append(cat)
at1 = pd.Series(at1, dtype='category')
return at1
def get_au1(x):
s = x.getvalue()
au1 = []
for l in s.splitlines():
splits = l.split('ADD')
cat = [np.nan,np.nan,np.nan,np.nan,np.nan]
if len(splits)==2:
_,add = splits
match = re.compile('AU1([0-9])([0-9])([09][0-9])([0-9])([0-9])([0-9])([0-9M])').search(add)
if match:
#counts[k] += 1
intensity,desc,precip,obscur,other,combo,qc = match.groups()
if qc in ['0','1','4','5','M','9']:
intensity = au_int_code[intensity]
desc = au_des_code[desc]
precip = au_precip_code[precip]
obscur = au_obsc_code[obscur]
other = au_other_code[other]
cat = [intensity,desc,precip,obscur,other]
au1.append(cat)
au1 = pd.DataFrame(au1, dtype='category', columns=['intens','desc','precip','obsc','other'])
return au1
def get_ay1(x):
good_qc = ['0','1','4','5','9']
s = x.getvalue()
ay1 = []
for l in s.splitlines():
splits = l.split('ADD')
cat = None
if len(splits)==2:
_,add = splits
match = re.compile('AY1([0-9])([0-9])([0-9]{2})([0-9])').search(add)
if match:
code, qc1, quantity ,qc2 = match.groups()
if qc1 in good_qc and qc2 in good_qc:
cat = ay_code[code]
ay1.append(cat)
ay1 = pd.DataFrame(ay1, dtype='category', columns=['ay1'])
return ay1
def get_mw1(x):
return _get_mwn(x,1)
def _get_mwn(x,n):
good_qc = ['0','1','4','5','M','9']
s = x.getvalue()
mw1 = []
for l in s.splitlines():
splits = l.split('ADD')
cat = None
if len(splits)==2:
_,add = splits
match = re.compile(f'MW{n}'+'([0-9]{2})([0-9])').search(add)
if match:
code, qc = match.groups()
if qc in good_qc:
cat = mw_code[code]
mw1.append(cat)
mw1 = pd.DataFrame(mw1, dtype='category', columns=['mw1'])
return mw1
def _get_add(s, gex, func, columns, dtypes):
x = []
for l in s.splitlines():
row = [np.nan]*len(columns)
splits = l.split('ADD')
if len(splits)==2:
_,add = splits
add = add.split('REM', maxsplit=1)[0]
match = gex.search(add)
if match:
tokens = match.groups()
ret = func(tokens)
if ret is not None:
row = ret
x.append(row)
x = pd.DataFrame(x, columns=columns)
for k in dtypes:
if k in columns:
x[k] = x.astype(dtypes[k])
return x
def get_mw(s):
good_qc = ['0','1','4','5','M','9']
def func(tokens):
code, qc = tokens
if qc in good_qc:
return [mw_code[code]]
dtypes = {}
mw = []
for i in range(1,8):
dtypes[f'wx_mw{i}'] = MW_CODE_CAT
gex = re.compile(f'MW{i}'+'([0-9]{2})([0-9])')
mw.append(_get_add(s, gex, func, [f'wx_mw{i}'], dtypes))
mw = pd.concat(mw, axis=1)
for k,v in dtypes.items():
mw[k] = mw[k].astype(v)
return mw
def get_ay(s):
good_qc = ['0','1','4','5','M','9']
def func(tokens):
code, qc1, quantity ,qc2 = tokens
if qc1 in good_qc and qc2 in good_qc:
return [ay_code[code], int(quantity)]
return [np.nan, np.nan]
ay = []
dtypes = {}
for i in range(1,3):
dtypes[f'past_wx_ay{i}'] = AY_CODE_CAT
gex = re.compile(f'AY{i}'+'([0-9])([0-9])([0-9]{2})([0-9])')
ay.append(_get_add(s, gex, func, [f'past_wx_ay{i}',f'quantity_ay{i}'], dtypes))
ay = pd.concat(ay, axis=1)
for k,v in dtypes.items():
ay[k] = ay[k].astype(v)
return ay
def get_au(s):
good_qc = ['0','1','4','5','M','9']
def func(tokens):
intensity,desc,precip,obscur,other,combo,qc = tokens
if qc in good_qc:
intensity = au_int_code[intensity]
desc = au_des_code[desc]
precip = au_precip_code[precip]
obscur = au_obsc_code[obscur]
other = au_other_code[other]
return [intensity,desc,precip,obscur,other]
au = []
dtypes = {}
for i in range(1,10):
dtypes[f'wx_int_au{i}'] = AU_INT_CAT
dtypes[f'wx_desc_au{i}'] = AU_DES_CAT
dtypes[f'wx_precip_au{i}'] = AU_PCP_CAT
dtypes[f'wx_obscur_au{i}'] = AU_OBSC_CAT
dtypes[f'wx_other_au{i}'] = AU_OTHER_CAT
gex = re.compile(f'AU{i}'+'([0-9])([0-9])([09][0-9])([0-9])([0-9])([0-9])([0-9M])')
au.append(_get_add(s, gex, func, [f'wx_int_au{i}',f'wx_desc_au{i}',f'wx_precip_au{i}',f'wx_obscur_au{i}',f'wx_other_au{i}'], dtypes))
au = pd.concat(au, axis=1)
for k,v in dtypes.items():
au[k] = au[k].astype(v)
return au
def get_at(s):
good_qc = ['0','1','4','5','M','9']
def func(tokens):
source,code,code2,qc = tokens
if qc in good_qc:
return [at_codes[code]]
at = []
dtypes = {}
for i in range(1,9):
dtypes[f'wx_at{i}'] = AT_CODE_CAT
gex = re.compile(f'AT{i}'+'(AU|AW|MW)([0-9]{2})(.{4})([0-9M])')
at.append(_get_add(s, gex, func, [f'wx_at{i}'], dtypes))
at = pd.concat(at, axis=1)
for k,v in dtypes.items():
at[k] = at[k].astype(v)
return at
def get_aw(s):
good_qc = ['0','1','4','5','M','9']
def func(tokens):
code, qc = tokens
if qc in good_qc:
return [aw_codes[code]]
aw = []
dtypes = {}
for i in range(1,5):
dtypes[f'wx_aw{i}'] = AW_CODE_CAT
gex = re.compile(f'AW{i}'+'([0-9]{2})([0-9])')
aw.append(_get_add(s, gex, func, [f'wx_aw{i}'], dtypes))
aw = pd.concat(aw, axis=1)
for k,v in dtypes.items():
aw[k] = aw[k].astype(v)
return aw
def get_ga(s):
good_qc = ['0','1','4','5','M','9']
def func(tokens):
cov, cov_qc, height, height_qc, type, type_qc = tokens
if cov_qc in good_qc:
return [ga_codes[cov], int(height), height_qc]
ga = []
dtypes = {}
for i in range(1,7):
dtypes[f'cld_cov_ga{i}'] = GA_CODES_CAT
dtypes[f'cld_height_qc_ga{i}'] = QC_CAT
gex = re.compile(f'GA{i}'+'([0-9]{2})([0-9])([+-][0-9]{5})([0-9M])([0-9]{2})([0-9M])')
ga.append(_get_add(s, gex, func, [f'cld_cov_ga{i}',f'cld_height_ga{i}',f'cld_height_qc_ga{i}'], dtypes))
ga = pd.concat(ga, axis=1)
for k,v in dtypes.items():
ga[k] = ga[k].astype(v)
return ga
def get_gd(s):
good_qc = ['0','1','4','5','M','9']
def func(tokens):
cov, cov2, cov_qc, height, height_qc, char = tokens
if cov_qc in good_qc:
return [gd_codes2[cov2], int(height), height_qc]
gd = []
dtypes = {}
for i in range(1,7):
dtypes[f'cld_cov_gd{i}'] = GD_CODES2_CAT
dtypes[f'cld_height_qc_gd{i}'] = QC_CAT
gex = re.compile(f'GD{i}'+'([0-9]{1})([0-9]{2})([0-9])([-+][0-9]{5})([0-9])([0-9])')
gd.append(_get_add(s, gex, func, [f'cld_cov_gd{i}',f'cld_height_gd{i}',f'cld_height_qc_gd{i}'], dtypes))
gd = pd.concat(gd, axis=1)
for k,v in dtypes.items():
gd[k] = gd[k].astype(v)
return gd
mw_rename = {'HeavyRain': ['Drizzle, freezing, moderate or heavy (dense)',
'Rain, freezing, moderate or heavy',
'Shower(s) of hail (hail, small hail, snow pellets), with or without rain or rain and snow mixed, not associated with thunder, moderate or heavy',
'Moderate or heavy snow, or rain and snow mixed or hail(Hail, small hail, snow pellets) at time of observation, thunderstorm during the preceding hour but not at time of observation',
'Thunderstorm, heavy, with hail (Hail, small hail, snow pellets) at time of observation, thunderstorm at time of observation',
'Drizzle and rain, moderate or heavy',
'Shower(s) of hail (hail, small hail, snow pellets), with or without rain or rain and snow mixed, not associated with thunder, slight',
'Rain or drizzle and snow, moderate or heavy',
'Shower(s) of snow pellets or small hail, with or without rain or rain and snow mixed, moderate or heavy',
'Ice pellets',
'Drizzle, not freezing, continuous, heavy (dense) at time of observation',
'Intermittent fall of snowflakes, heavy at time of observation',
'Snow grains (with or without fog)',
'Shower(s) of rain and snow mixed, moderate or heavy',
'Shower(s) of hail (Hail, small hail, snow pellets), or rain and hail',
'Drizzle, not freezing, intermittent, heavy (dense) at time of observation',
'Moderate or heavy rain at time of observation, thunderstorm during the preceding hour but not at time of observation',
'Continuous fall of snowflakes, heavy at time of observation',
'Heavy blowing snow generally high (above eye level)',
'Isolated star-like snow crystals (with or without fog)',
'Rain and snow or ice pellets not falling as shower(s)',
'Rain, not freezing, intermittent, heavy at time of observation',
'Rain, not freezing, continuous, heavy at time of observation',
'Thunderstorm, heavy, without hail (Hail, small hail, snow pellets), but with rain and/or snow at time of observation, thunderstorm at time of observation',
'Continuous fall of snowflakes, moderate at time of observation',
'Shower(s) of snow or of rain and snow',
'Snow not falling as shower(s)',
'Rain shower(s), violent',
'Snow shower(s), moderate or heavy',
'Rain, not freezing, continuous, moderate at time of observation',
'Shower(s) of rain',
'Rain shower(s), moderate or heavy',
'Thunderstorm, slight or moderate, without hail (Hail, small hail, snow pellets), but with rain and/or snow at time of observation, thunderstorm at time of observation',
'Rain (not freezing) not falling as shower(s)'],
'LightRain': ['Slight snow, or rain and snow mixed or hail (Hail, small hail, snow pellets), at time of observation, thunderstorm during the preceding hour but not at time of observation',
'Freezing drizzle or freezing rain not falling as shower(s)',
'Drizzle, freezing, slight',
'Rain, freezing, slight',
'Intermittent fall of snowflakes, moderate at time of observation',
'Drizzle, not freezing, intermittent, moderate at time of observation',
'Shower(s) of snow pellets or small hail, with or without rain or rain and snow mixed, slight',
'Drizzle and rain, slight',
'Rain or drizzle and snow, slight',
'Slight rain at time of observation, thunderstorm during the preceding hour but not at time of observation',
'Drizzle, not freezing, continuous, moderate at time of observation',
'Precipitation within sight, not reaching the ground or the surface of the sea',
'Drizzle (not freezing) or snow grains not falling as shower(s)',
'Shower(s) of rain and snow mixed, slight',
'Rain, not freezing, intermittent, moderate at time of observation',
'Drizzle, not freezing, intermittent, slight at time of observation',
'Intermittent fall of snowflakes, slight at time of observation',
'Drizzle, not freezing, continuous, slight at time of observation',
' Precipitation within sight, reaching the ground or the surface of the sea, but distant, i.e., estimated to be more than 5 km from the station',
'Precipitation within sight, reaching the ground or the surface of the sea, near to, but not at the station',
'Show shower(s), slight',
'Continuous fall of snowflakes, slight at time of observation',
'Rain, not freezing, intermittent, slight at time of observation',
'Rain, not freezing, continuous, slight at time of observation',
'Rain shower(s), slight'],
'Haze': ['Thunderstorm combined with duststorm or sandstorm at time of observation, thunderstorm at time of observation',
'Severe duststorm or sandstorm has decreased during the preceding hour',
'Severe duststorm or sandstorm has begun or has increased during the preceding hour',
'Severe duststorm or sandstorm no appreciable change during the preceding hour',
'Slight or moderate duststorm or sandstorm has decreased during the preceding hour',
' Well developed dust whirl(s) or sand whirl(s) seen at or near the station during the preceding hour or at the time of observation, but no duststorm or sandstorm',
'Slight or moderate duststorm or sandstorm has begun or has increased during the preceding hour',
'Slight or moderate duststorm or sandstorm no appreciable change during the preceding hour',
'Duststorm or sandstorm within sight at the time of observation, or at the station during the preceding hour',
'Widespread dust in suspension in the air, not raised by wind at or near the station at the time of observation',
'Dust or sand raised by wind at or near the station at the time of observation, but no well-developed dust whirl(s) sand whirl(s), and no duststorm or sandstorm seen or, in the case of ships, blowing spray at the station',
'Slight or moderate blowing snow generally high (above eye level)',
'Visibility reduced by smoke, e.g. veldt or forest fires, industrial smoke or volcanic ashes',
'Haze'],
'Cloudy': ['Funnel cloud(s) (Tornado cloud or waterspout) at or within sight of the station during the preceding hour or at the time of observation',
'Squalls at or within sight of the station during the preceding hour or at the time of observation',
'Thunderstorm, slight or moderate, with hail (hail, small hail, snow pellets) at time of observation, thunderstorm at time of observation',
'Thunderstorm (with or without precipitation)',
'Thunderstorm, but no precipitation at the time of observation',
'Clouds generally forming or developing'],
'Fog': ['Fog, depositing rime, sky invisible',
'Fog or ice fog, sky invisible, has become thinner during the preceding hour',
'Fog, depositing rime, sky visible',
'Fog or ice fog, sky visible, has become thinner during the preceding hour',
'Diamond dust (with or without fog)',
'More or less continuous shallow fog or ice fog at the station, whether on land or sea, not deeper than about 2 meters on land or 10 meters at sea',
'Fog or ice fog, sky invisible, has begun or has become thicker during the preceding hour',
'Fog or ice fog',
'Fog or ice fog, sky visible, has begun or has become thicker during the preceding hour',
'Fog or ice fog at a distance at the time of observation, but not at the station during the preceding hour, the fog or ice fog extending to a level above that of the observer',
'Fog or ice fog in patches',
'Fog or ice fog, sky invisible, no appreciable change during the preceding hour',
'Patches of shallow fog or ice fog at the station, whether on land or sea, not deeper than about 2 meters on land or 10 meters at sea',
'Fog or ice fog, sky visible, no appreciable change during the preceding hour',
'Mist'],
'Clear': ['Heavy drifting snow generally low (below eye level)',
'Lightning visible, no thunder heard',
'Slight or moderate drifting snow generally low (below eye level)',
'Clouds generally dissolving or becoming less developed',
'Cloud development not observed or not observable',
'State of sky on the whole unchanged']}
at_rename = {'HeavyRain': ['Freezing rain',
'Ice pellets, sleet, snow pellets or small hail',
'Snow, snow pellets, snow grains or ice crystals',
'Rain'],
'Clear': ['Blowing or drifting snow', 'Thunder'],
'Haze': ['Dust, volcanic ash, blowing dust, blowing sand or blowing obstruction',
'Smoke or haze'],
'LightRain': ['Unknown precipitation', 'Drizzle'],
'Fog': ['Fog, ice fog or freezing fog (may include heavy fog)', 'Mist']}
aw_rename = {'Clear': ['Clouds generally dissolving or becoming less developed',
'State of sky on the whole unchanged during the past hour',
'Squalls',
'Blowing or drifting snow or sand',
'Distant lightning',
'No significant weather observed'],
'Cloudy': ['Clouds generally forming or developing during the past hour',
'Thunderstorm (with or without precipitation)',
'Thunderstorm'],
'LightRain': ['Rain, freezing, slight',
'Drizzle and rain, slight',
'Showers or intermittent precipitation',
'Ice pellets, slight',
'Snow showers or intermittent snow, slight',
'Drizzle (not freezing) or snow grains',
'Thunderstorm, slight or moderate, with no precipitation',
'Thunderstorm, slight or moderate, with rain showers and/or snow showers',
'Drizzle, not freezing, slight',
'Rain or drizzle and snow, slight',
'Precipitation, slight or moderate',
'Snow, slight',
'Rain showers or intermittent rain, slight',
'Rain, not freezing, slight'],
'HeavyRain': ['Ice pellets, moderate',
'Snow showers or intermittent snow, moderate',
'Solid precipitation, slight or moderate',
'Snow grains',
'Hail',
'Thunderstorm, slight or moderate, with hail',
'Rain',
'Rain showers or intermittent rain, violent',
'Snow, heavy',
'Snow, moderate',
'Snow',
'Thunderstorm, heavy, with rain showers and/or snow',
'Drizzle, not freezing, heavy',
'Rain or drizzle and snow, moderate or heavy',
'Precipitation',
'Precipitation, heavy',
'Drizzle, not freezing, moderate',
'Rain showers or intermittent rain, heavy',
'Rain showers or intermittent rain, moderate',
'Rain, not freezing, heavy',
'Rain, not freezing, moderate',
'Rain (not freezing)'],
'Fog': ['Fog, depositing rime',
'Fog or ice fog in patches',
'Fog or ice fog, no appreciable change during the past hour',
'Fog or ice fog, has become thinner during the past hour',
'Fog or ice fog, has begun or become thicker during the past hour',
'Mist'],
'Haze': ['Haze, smoke, or dust in suspension in the air, visibility equal to or greater than 1km',
'Smoke']}
au_obscr_rename = {'Fog': ['Fog (FG)', 'Mist (BR)'],
'Haze': ['Haze (HZ)'],
'Clear': ['No Obscuration']}
ay_rename = {'LightRain': ['Drizzle', 'Shower(s)'],
'Haze': ['Sandstorm, duststorm or blowing snow'],
'HeavyRain': ['Thunderstorm(s) with or without precipitation',
'Snow, or rain and snow mixed',
'Rain'],
'Cloudy': ['Cloud covering more than ½ of the sky duringpart of the appropriate period and covering ½ or less during part of the period',
'Cloud covering more than 1/2 of the sky throughout the appropriate period'],
'Fog': ['Fog or ice fog or thick haze'],
'Clear': ['Cloud covering 1/2 or less of the sky throughout the appropriate period']}