Skip to content
Snippets Groups Projects
Commit 199daffe authored by Max Drexler's avatar Max Drexler
Browse files

move relavent code from sds_grib

parents
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 482 additions and 0 deletions
grib_pipeline.py 0 → 100644
+ 482
− 0
View file @ 199daffe
"""
Ingest grib files and publish metadata for files to RabbitMQ.
"""
__author__ = 'Max Drexler'
__email__ = 'mndrexler@wisc.edu'
from collections import defaultdict
from datetime import datetime
import json
import os
from pathlib import Path
from socket import gethostname
import sys
import time
from typing import Generator
import grib2io
from watchfiles import watch, Change
from ssec_amqp import AmqpClient, AmqpExchange
from dotenv import load_dotenv
if sys.version < (3,7):
raise SystemError('Python version too low')
HOSTNAME = gethostname()
GRIB_ENDINGS = [".grb", ".grib", ".grb2", ".grib2"]
# (first_lat, first_lon, rows, cols, gen_proc_id)
XCD_MODELS = {
("44.196", "174.759", "237", "377", "115"): ["DGEX-AKPS", "DGEX"],
("19.943", "234.907", "303", "491", "115"): ["DGEX-USLC", "DGEX"],
("0.0", "0.0", "181", "720", "30"): ["FGF-NHME", "FGF"],
("90.0", "0.0", "181", "360", "107"): ["GEFS-GLME", "GEFS"],
("40.5301", "181.429", "1105", "1649", "96"): ["GFS-AKPSDRES", "GFS"],
("19.132", "174.163", "156", "180", "81"): ["GFS-AKPSHRES", "GFS"],
("19.132", "174.163", "156", "180", "96"): ["GFS-AKPSHRES", "GFS"],
("0.0", "0.0", "91", "360", "96"): ["GFS-NHME", "GFS"],
("-20.826", "210.0", "65", "65", "81"): ["GFS-NHPS", "GFS"],
("-20.826", "210.0", "65", "65", "96"): ["GFS-NHPS", "GFS"],
("12.19", "226.541", "129", "185", "81"): ["GFS-USLC2", "GFS"],
("12.19", "226.541", "129", "185", "96"): ["GFS-USLC2", "GFS"],
("20.192", "238.446", "689", "1073", "96"): ["GFS-USLCAWI4", "GFS"],
("20.192", "238.446", "1377", "2145", "96"): ["GFS-USLCDRS1", "GFS"],
("7.838", "218.972", "85", "129", "81"): ["GFS-USPS", "GFS"],
("7.838", "218.972", "85", "129", "96"): ["GFS-USPS", "GFS"],
("49.1", "267.799984", "235", "327", "131"): ["GLWM", "GLWM"],
("89.958", "0.042", "2160", "4320", "120"): ["ICA-GLME", "ICA"],
("16.281", "233.862", "337", "451", "191"): ["ICN-USLC13KM", "ICNG"],
("20.192", "238.446", "689", "1073", "108"): ["LMP-USLCAWI4", "LMP"],
("20.192", "238.446", "1377", "2145", "108"): ["LMP-USLCDRES", "LMP"],
("74.0", "165.0", "391", "548", "11"): ["MGWM-AKME", "MGWM"],
("30.0", "130.0", "301", "511", "11"): ["MGWM-EPME", "MGWM"],
("90.0", "0.0", "336", "720", "11"): ["MGWM-GLME", "MGWM"],
("75.0", "140.0", "187", "401", "11"): ["MGWM-NPME", "MGWM"],
("55.0", "260.0", "331", "301", "11"): ["MGWM-WAME", "MGWM"],
("50.0", "195.0", "526", "736", "11"): ["MGWM-WCHIME", "MGWM"],
("50.0", "210.0", "151", "241", "11"): ["MGWM-WCME", "MGWM"],
("30.0", "187.0", "425", "553", "84"): ["NAM-AKPSAWI4", "NAM"],
("1.0", "214.5", "277", "349", "111"): ["NAM-USLC32KM", "NAM"],
("12.19", "226.541", "428", "614", "84"): ["NAM-USLCAW12", "NAM"],
("20.190001", "238.449996", "689", "1073", "180"): ["NCE-USLCAWI4", "NCE"],
("40.530094", "181.429031", "553", "825", "183"): ["NDF-AKPS2", "NDF"],
("40.530096", "181.429024", "553", "825", "183"): ["NDF-AKPS2", "NDF"],
("40.529998", "181.429993", "553", "825", "183"): ["NDF-AKPS", "NDF"],
("40.530101", "181.429", "1105", "1649", "0"): ["NDF-AKPSDRS", "NDF"],
("20.191999", "238.445999", "1377", "2145", "183"): ["NDF-USLCDRES", "NDF"],
("20.192", "238.446", "1377", "2145", "183"): ["NDF-USLCDRES", "NDF"],
("40.53", "181.429", "553", "825", "89"): ["NMMB-AKPS", "NMMB"],
("40.53", "181.429", "1105", "1649", "89"): ["NMMB-AKPSDRS", "NMMB"],
("20.192", "238.45", "689", "1073", "89"): ["NMMB-USLCAW4", "NMMB"],
("20.192", "238.446", "1377", "2145", "89"): ["NMMB-USLCDRS", "NMMB"],
("40.530101", "181.429", "553", "825", "12"): ["PSS-AKPS", "PSS"],
("16.281", "233.862", "337", "451", "105"): ["RAP-USLC13KM", "RAP"],
("40.530101", "181.429", "553", "825", "109"): ["RTM-AKPS", "RTMA"],
("40.530101", "181.429", "1105", "1649", "109"): ["RTM-AKPSDRES", "RTMA"],
("20.191999", "238.445999", "689", "1073", "109"): ["RTM-USLCAWI4", "RTMA"],
("20.191999", "238.445999", "1377", "2145", "109"): ["RTM-USLCDRES", "RTMA"],
("-40.0", "130.0", "80", "120", "85"): ["RTOF-APME", "RTOF"],
("40.0", "140.0", "150", "350", "85"): ["RTOF-ARCTME", "RTOF"],
("0.0", "180.0", "500", "625", "85"): ["RTOF-HIME", "RTOF"],
("40.0", "251.0", "152", "328", "85"): ["RTOF-NATLME", "RTOF"],
("40.0", "195.0", "45", "84", "85"): ["RTOF-NEPME", "RTOF"],
("40.0", "155.0", "340", "700", "85"): ["RTOF-NPME", "RTOF"],
("-30.0", "170.0", "375", "560", "85"): ["RTOF-SCPME", "RTOF"],
("10.0", "260.0", "435", "575", "85"): ["RTOF-USECME", "RTOF"],
("10.0", "210.0", "625", "625", "85"): ["RTOF-USWCME", "RTOF"],
("0.0", "130.0", "375", "625", "85"): ["RTOF-WPME", "RTOF"],
("30.0", "187.0", "107", "139", "113"): ["SREF-AKPS", "SREF"],
("12.19", "226.541", "129", "185", "113"): ["SREF-USLC2", "SREF"],
("10.0", "190.0", "101", "126", "113"): ["SREF-USME", "SREF"],
("89.958", "0.042", "2160", "4320", "44"): ["SST-GLME", "SST"],
("20.191999", "238.445999", "1377", "2145", "118"): ["URMA-USLCDR1", "URMA"],
("37.979684", "234.042704", "795", "709", "118"): ["URMA-USLCDR2", "URMA"],
("44.8", "185.5", "603", "825", "116"): ["WRFE-AKPS", "WRFE"],
("16.4", "197.65", "170", "223", "116"): ["WRFE-HIME", "WRFE"],
("11.7", "141.0", "170", "223", "116"): ["WRFE-WPME", "WRFE"],
("13.5", "283.41", "208", "340", "116"): ["WRFE-PRME", "WRFE"],
("1.0", "214.5", "277", "349", "116"): ["WRFE-USLC32K", "WRFE"],
("22.1", "250.2", "614", "884", "116"): ["WRFE-USLCSE", "WRFE"],
("24.5", "230.8", "614", "884", "116"): ["WRFE-USLCSW", "WRFE"],
("44.8", "185.5", "603", "825", "112"): ["WRFN-AKPS", "WRFN"],
("16.4", "197.65", "170", "223", "112"): ["WRFN-HIME", "WRFN"],
("11.7", "141.0", "170", "223", "112"): ["WRFN-WPME", "WRFN"],
("13.5", "283.41", "208", "340", "112"): ["WRFN-PRME", "WRFN"],
("1.0", "214.5", "277", "349", "112"): ["WRFN-USLC32K", "WRFN"],
("22.1", "250.2", "614", "884", "112"): ["WRFN-USLCSE", "WRFN"],
("24.5", "230.8", "614", "884", "112"): ["WRFN-USLCSW", "WRFN"],
("20.191999", "238.445999", "1377", "2145", "83"): ["HRR-USLCDRES", "HRR"],
("90.0", "0.0", "73", "144", "96"): ["GFS-GLME2P5D", "GFS"],
("90.0", "0.0", "73", "144", "81"): ["GFS-GLME2P5D", "GFS"],
("12.19", "226.541", "129", "185", "89"): ["NMMB-USLCA40", "NMMB"],
("90.0", "0.0", "361", "720", "96"): ["GFS-GLME0P5D", "GFS"],
("90.0", "0.0", "361", "720", "81"): ["GFS-GLME0P5D", "GFS"],
("12.19", "226.541", "129", "185", "116"): ["WRFE-USLC40K", "WRFE"],
("50.75", "271.75", "205", "275", "96"): ["GFS-PRMEP25D", "GFS"],
("30.0", "187.0", "213", "277", "96"): ["GFS-AKPSAWI2", "GFS"],
("-0.268", "220.525", "110", "147", "84"): ["NAM-USPS", "NAM"],
("30.0", "187.0", "107", "139", "84"): ["NAM-AKPS45KM", "NAM"],
("12.19", "226.541", "129", "185", "112"): ["WRFN-USLC40K", "WRFN"],
("30.0", "187.0", "213", "277", "81"): ["GFS-AKPSAWI2", "GFS"],
("41.530708", "267.364016", "361", "581", "131"): ["GLW-USLC", "GLWM"],
("49.099998", "267.799988", "235", "327", "131"): ["GLW-MWUSME", "GLWM"],
("90.0", "0.0", "145", "288", "96"): ["GFS-GLME1P25", "GFS"],
("90.0", "0.0", "145", "288", "81"): ["GFS-GLME1P25", "GFS"],
("90.0", "0.0", "721", "1440", "96"): ["GFS-GLMEP25D", "GFS"],
("90.0", "0.0", "721", "1440", "81"): ["GFS-GLMEP25D", "GFS"],
("20.191999", "238.445999", "1377", "2145", "14"): ["ESSA-USLCDRS", "ESSA"],
("20.191999", "238.445999", "1377", "2145", "17"): ["ESSP-USLCDRS", "ESSP"],
("40.530101", "181.429", "553", "825", "17"): ["ESSP-AKPS", "ESSP"],
("20.191999", "238.445999", "689", "1073", "12"): ["PSS-USLCAWI4", "PSS"],
("20.191999", "238.445999", "1377", "2145", "12"): ["PSS-USLCDRES", "PSS"],
("12.19", "226.541", "129", "185", "84"): ["NAM-USLC2", "NAM"],
("50.75", "271.75", "102", "137", "96"): ["GFS-WAME", "GFS"],
("50.75", "271.75", "102", "137", "81"): ["GFS-WAME", "GFS"],
("40.5301", "181.429", "1105", "1649", "81"): ["GFS-AKPSDRES", "GFS"],
("60.0", "160.0", "250", "950", "85"): ["RTOF-SARCME", "RTOF"],
("16.281", "233.861999", "337", "451", "105"): ["RAP-USLC13KM", "RAP"],
("20.191999", "238.445999", "1377", "2145", "114"): ["NAEF-USLCDRS", "NAEF"],
("20.192", "238.446", "1597", "2145", "104"): ["NBM-USLCDRS2", "NBM"],
("20.191999", "238.445999", "1377", "2145", "222"): ["WPC-USLCDRES", "WPC"],
("20.191999", "238.445999", "1377", "2145", "223"): ["WPC-USLCDRES", "WPC"],
("12.19", "226.541", "129", "185", "111"): ["NAM-USLC2", "NAM"],
("35.0", "170.0", "225", "277", "96"): ["GFS-NPPS", "GFS"],
("35.0", "170.0", "225", "277", "81"): ["GFS-NPPS", "GFS"],
("20.191999", "238.445999", "1377", "2145", "96"): ["GFS-USLCDRS1", "GFS"],
("20.192", "238.446", "1597", "2145", "96"): ["GFS-USLCDRS2", "GFS"],
("20.190001", "238.449996", "689", "1073", "224"): ["WPC-USLCAWI4", "WPC"],
("20.190001", "238.449996", "689", "1073", "221"): ["WPC-USLCAWI4", "WPC"],
("54.995", "230.005", "3500", "7000", "97"): ["MRMS-USME1", "OAR"],
("54.9975", "230.0025", "7000", "14000", "97"): ["MRMS-USME2", "OAR"],
("54.95", "230.05", "350", "700", "97"): ["MRMS-USME3", "OAR"],
("35.8", "282.7", "138", "92", "15"): ["NWPS-VAMEWF2", "NWPS"],
("25.45", "275.2", "263", "185", "15"): ["NWPS-FLMETB2", "NWPS"],
("35.8", "282.7", "229", "153", "15"): ["NWPS-VAMEWF1", "NWPS"],
("25.45", "275.2", "119", "83", "15"): ["NWPS-FLMETB3", "NWPS"],
("24.1", "276.46", "134", "174", "15"): ["NWPS-FLMESF1", "NWPS"],
("24.1", "276.46", "201", "261", "15"): ["NWPS-FLMESF2", "NWPS"],
("33.85", "282.0", "170", "167", "15"): ["NWPS-NCMEMC1", "NWPS"],
("36.0", "284.32", "112", "90", "15"): ["NWPS-NCMEKDH", "NWPS"],
("36.0", "284.2", "67", "54", "15"): ["NWPS-NCMENOB", "NWPS"],
("33.85", "282.0", "62", "61", "15"): ["NWPS-NCMEMC2", "NWPS"],
("39.75", "285.762", "62", "91", "15"): ["NWPS-NYMENY1", "NWPS"],
("39.75", "285.55", "102", "162", "15"): ["NWPS-NYMENY2", "NWPS"],
("40.83", "287.25", "61", "90", "15"): ["NWPS-NYMELI1", "NWPS"],
("40.4", "285.7", "37", "41", "15"): ["NWPS-LIME4", "NWPS"],
("40.54", "286.22", "29", "85", "15"): ["NWPS-NYMELI2", "NWPS"],
("27.41", "264.97", "74", "114", "15"): ["NWPS-LATXME1", "NWPS"],
("27.41", "264.97", "179", "253", "15"): ["NWPS-LAMELC2", "NWPS"],
("25.26", "262.359", "84", "57", "15"): ["NWPS-TXME3", "NWPS"],
("25.26", "262.0", "186", "145", "15"): ["NWPS-TXMEBR2", "NWPS"],
("26.0", "262.345", "90", "67", "15"): ["NWPS-TXME1", "NWPS"],
("26.0", "261.5", "217", "195", "15"): ["NWPS-TXMECC2", "NWPS"],
("26.13", "278.1", "75", "84", "15"): ["NWPS-SWFLME1", "NWPS"],
("26.18", "279.89", "65", "59", "15"): ["NWPS-SEFLME1", "NWPS"],
("26.89", "279.93", "47", "42", "15"): ["NWPS-SEFLME2", "NWPS"],
("25.85", "279.87", "167", "101", "15"): ["NWPS-SEFLME3", "NWPS"],
("28.5", "270.9", "61", "79", "15"): ["NWPS-MSALME1", "NWPS"],
("28.5", "270.9", "155", "174", "15"): ["NWPS-ALMEMP2", "NWPS"],
("40.45", "287.5", "104", "103", "15"): ["NWPS-MAMEBT1", "NWPS"],
("40.45", "287.5", "156", "155", "15"): ["NWPS-MAMEBT2", "NWPS"],
("41.9", "288.8", "289", "149", "15"): ["NWPS-MAMEBT3", "NWPS"],
("43.47", "288.5", "289", "251", "15"): ["NWPS-NHMELKW", "NWPS"],
("43.76", "289.36", "189", "145", "15"): ["NWPS-MEMELKS", "NWPS"],
("42.4", "289.021", "67", "74", "15"): ["NWPS-MEME7", "NWPS"],
("42.4", "288.25", "103", "129", "15"): ["NWPS-MEMEGP2", "NWPS"],
("27.0", "262.6", "217", "222", "15"): ["NWPS-TXMEHG2", "NWPS"],
("27.0", "262.6", "84", "100", "15"): ["NWPS-TXMEHG4", "NWPS"],
("30.93", "278.53", "81", "42", "15"): ["NWPS-GAMESAS", "NWPS"],
("30.62", "278.51", "89", "111", "15"): ["NWPS-GAMEKBY", "NWPS"],
("38.5", "284.3", "151", "92", "15"): ["NWPS-DEMEDB1", "NWPS"],
("24.3", "276.8", "123", "315", "15"): ["NWPS-FLMEFKY", "NWPS"],
("23.0", "276.5", "84", "116", "15"): ["NWPS-FLMEKW1", "NWPS"],
("23.0", "276.5", "96", "132", "15"): ["NWPS-FLMEKW2", "NWPS"],
("27.48", "276.8", "182", "232", "15"): ["NWPS-FLMETB1", "NWPS"],
("38.2", "284.424", "89", "65", "15"): ["NWPS-PAMEMH1", "NWPS"],
("28.2", "279.2", "134", "99", "15"): ["NWPS-FLMECPC", "NWPS"],
("26.5", "278.681", "98", "70", "15"): ["NWPS-NFLAME1", "NWPS"],
("26.5", "278.6", "217", "160", "15"): ["NWPS-FLMEMB2", "NWPS"],
("29.87", "278.68", "34", "26", "15"): ["NWPS-FLMESTA", "NWPS"],
("30.38", "278.57", "28", "34", "15"): ["NWPS-FLMEMPT", "NWPS"],
("17.0", "292.0", "93", "142", "15"): ["NWPS-PRMESJ1", "NWPS"],
("17.55", "292.6", "140", "319", "15"): ["NWPS-PRMESJ3", "NWPS"],
("18.33", "292.7", "212", "212", "15"): ["NWPS-PRMERIN", "NWPS"],
("17.0", "292.0", "155", "237", "15"): ["NWPS-PRMESJ2", "NWPS"],
("29.0", "264.82", "178", "141", "15"): ["NWPS-TXMEHG3", "NWPS"],
("28.35", "272.65", "62", "116", "15"): ["NWPS-FLMETL3", "NWPS"],
("28.35", "272.65", "140", "259", "15"): ["NWPS-FLMETL1", "NWPS"],
("30.15", "271.55", "104", "103", "15"): ["NWPS-ALMEMBY", "NWPS"],
("30.15", "273.2", "60", "94", "15"): ["NWPS-FLMECBY", "NWPS"],
("30.1", "272.68", "82", "70", "15"): ["NWPS-FLMEESP", "NWPS"],
("30.08", "272.32", "70", "62", "15"): ["NWPS-FLMEPBY", "NWPS"],
("38.2", "284.25", "149", "117", "15"): ["NWPS-PAMEMH2", "NWPS"],
("30.67", "278.3", "180", "192", "15"): ["NWPS-SCMECH1", "NWPS"],
("30.67", "278.384", "81", "85", "15"): ["NWPS-SCARME2", "NWPS"],
("20.192", "238.446", "1377", "2145", "0"): ["MOS-USLCDRES", "MOS"],
("40.53", "181.429", "1105", "1649", "0"): ["MOS-AKPSDRES", "MOS"],
("28.35", "272.65", "62", "115", "15"): ["NWPS-FLMETL4", "NWPS"],
("36.75", "282.2", "164", "107", "15"): ["NWPS-MDMEBW1", "NWPS"],
("35.8", "282.7", "138", "91", "15"): ["NWPS-DMVAME4", "NWPS"],
("43.3", "290.8", "109", "131", "15"): ["NWPS-MEMECB1", "NWPS"],
("43.3", "290.8", "49", "59", "15"): ["NWPS-MEMECB2", "NWPS"],
("44.16", "291.58", "156", "160", "15"): ["NWPS-MEMEBHR", "NWPS"],
("44.27", "291.83", "167", "168", "15"): ["NWPS-MEMEWHR", "NWPS"],
("32.5", "279.6", "103", "147", "15"): ["NWPS-NCMEWL1", "NWPS"],
("34.4", "282.73", "83", "68", "15"): ["NWPS-NCMESOB", "NWPS"],
("34.6", "282.85", "99", "153", "15"): ["NWPS-NCMEEMI", "NWPS"],
("32.5", "279.6", "85", "122", "15"): ["NWPS-NCMEWL2", "NWPS"],
("29.0", "276.45", "89", "104", "15"): ["NWPS-FLMECKY", "NWPS"],
("29.85", "275.5", "56", "137", "15"): ["NWPS-FLMESTM", "NWPS"],
("29.6", "274.54", "65", "99", "15"): ["NWPS-FLMECSB", "NWPS"],
("29.95", "274.13", "78", "81", "15"): ["NWPS-FLMEPCY", "NWPS"],
("29.3", "269.4", "98", "140", "15"): ["NWPS-LAMELPN", "NWPS"],
("27.5", "268.2", "87", "109", "15"): ["NWPS-LAMENB1", "NWPS"],
("27.5", "268.2", "192", "242", "15"): ["NWPS-LAMENB2", "NWPS"],
("16.201", "285.72", "94", "108", "105"): ["RAP-PRLC16KM", "RAP"],
("16.281", "233.862", "113", "151", "105"): ["RAP-USLC40KM", "RAP"],
("16.281", "233.862", "225", "301", "105"): ["RAP-USLC20KM", "RAP"],
("20.19", "238.45", "689", "1073", "0"): ["SPC-USLCAWI4", "SPC"],
("32.6", "236.64", "98", "142", "15"): ["NWPS-CAMELA1", "NWPS"],
("32.6", "236.64", "112", "163", "15"): ["NWPS-CAMELA2", "NWPS"],
("32.08", "241.0", "97", "118", "15"): ["NWPS-CAMESD1", "NWPS"],
("32.08", "241.0", "65", "79", "15"): ["NWPS-CAMESD2", "NWPS"],
("32.6", "236.64", "78", "114", "15"): ["NWPS-CAMELA3", "NWPS"],
("33.52", "241.59", "56", "112", "15"): ["NWPS-CAMEPLB", "NWPS"],
("31.95", "240.8", "37", "44", "15"): ["NWPS-CAMESD3", "NWPS"],
("32.68", "242.66", "195", "79", "15"): ["NWPS-CAMECLJ", "NWPS"],
("35.0", "234.2", "123", "116", "15"): ["NWPS-CAMEMR1", "NWPS"],
("35.0", "234.2", "109", "103", "15"): ["NWPS-CAMEMR2", "NWPS"],
("35.0", "234.2", "98", "93", "15"): ["NWPS-CAMEMR3", "NWPS"],
("37.3", "236.7", "123", "89", "15"): ["NWPS-CAMESFB", "NWPS"],
("36.2", "237.8", "178", "90", "15"): ["NWPS-CAMEMRB", "NWPS"],
("28.35", "272.65", "63", "117", "15"): ["NWPS-FLMETL2", "NWPS"],
("55.0", "202.0", "62", "81", "15"): ["NWPS-AKMEAN1", "NWPS"],
("53.9", "215.5", "177", "237", "15"): ["NWPS-AKMEJN1", "NWPS"],
("53.9", "215.5", "59", "79", "15"): ["NWPS-AKMEJN2", "NWPS"],
("57.0", "222.6", "300", "297", "15"): ["NWPS-AKMEGBY", "NWPS"],
("54.5", "224.0", "306", "387", "15"): ["NWPS-AKMEWLS", "NWPS"],
("54.995", "230.005992", "3500", "7000", "97"): ["MRMS-USME4", "OAR"],
("55.0", "202.0", "147", "193", "15"): ["NWPS-AKMEAN2", "NWPS"],
("27.0", "262.6", "98", "100", "15"): ["NWPS-TXMEHG1", "NWPS"],
("20.191999", "238.445999", "1597", "2145", "180"): ["NCE-USLCDRS", "NCEP"],
("40.53", "181.429", "1105", "1649", "104"): ["NBM-AKPSDRS", "NBM"],
("53.15", "183.0", "82", "136", "15"): ["NWPS-AKMEAI1", "NWPS"],
("58.8", "205.6", "311", "317", "15"): ["NWPS-AKMEAN3", "NWPS"],
("41.0", "232.7", "192", "154", "15"): ["NWPS-ORMEMD1", "NWPS"],
("41.0", "232.7", "77", "62", "15"): ["NWPS-ORMEMD2", "NWPS"],
("32.61", "279.99", "49", "75", "15"): ["NWPS-SCMECHB", "NWPS"],
("31.88", "278.87", "112", "131", "15"): ["NWPS-GAMESHB", "NWPS"],
("39.4", "285.4", "112", "69", "15"): ["NWPS-NJMELBI", "NWPS"],
("40.4", "285.7", "74", "82", "15"): ["NWPS-DEMEDB2", "NWPS"],
("42.4", "288.25", "74", "92", "15"): ["NWPS-MEMEGP1", "NWPS"],
("28.5", "270.9", "70", "79", "15"): ["NWPS-ALMEMP1", "NWPS"],
("46.1", "233.0", "93", "99", "15"): ["NWPS-WAMESE1", "NWPS"],
("46.1", "233.0", "123", "131", "15"): ["NWPS-WAMESE2", "NWPS"],
("46.1", "233.0", "74", "79", "15"): ["NWPS-WAMESE3", "NWPS"],
("38.4", "233.73", "106", "64", "15"): ["NWPS-CAMEEU1", "NWPS"],
("38.4", "233.73", "141", "85", "15"): ["NWPS-CAMEEU2", "NWPS"],
("38.4", "233.73", "85", "51", "15"): ["NWPS-CAMEEU3", "NWPS"],
("18.07", "198.5", "207", "298", "15"): ["NWPS-HIME1", "NWPS"],
("18.07", "198.5", "70", "101", "15"): ["NWPS-HIME2", "NWPS"],
("21.55", "199.96", "190", "197", "15"): ["NWPS-HIMEKAN", "NWPS"],
("19.4", "204.8", "112", "126", "15"): ["NWPS-HIMEHIL", "NWPS"],
("20.4", "203.1", "223", "209", "15"): ["NWPS-HIMEMAU", "NWPS"],
("21.2", "201.65", "136", "156", "15"): ["NWPS-HIMEOAH", "NWPS"],
("41.0", "232.7", "128", "103", "15"): ["NWPS-ORMEMD3", "NWPS"],
("53.15", "183.0", "132", "218", "15"): ["NWPS-AKMEAI2", "NWPS"],
("12.34", "143.67", "125", "126", "15"): ["NWPS-GUME1", "NWPS"],
("12.34", "143.67", "67", "67", "15"): ["NWPS-GUME2", "NWPS"],
("14.72", "145.32", "74", "72", "15"): ["NWPS-GUMENMI", "NWPS"],
("14.02", "145.02", "29", "39", "15"): ["NWPS-GUMEROT", "NWPS"],
("13.12", "144.52", "74", "61", "15"): ["NWPS-GUMEISL", "NWPS"],
("23.018", "275.667", "574", "480", "84"): ["NAM-FLLC", "NWPS"],
("40.5301", "181.429", "1105", "1649", "114"): ["NAEF-AKPSDRS", "NAEF"],
("23.097", "240.964", "881", "1121", "190"): ["AWC-UNKNPS", "AWC"],
("40.5301", "181.429", "1105", "1649", "0"): ["MOS-AKPSDRES", "MOS"],
("19.229", "233.7234", "1597", "2345", "104"): ["NBM-USLCDRS3", "NBM"],
("65.92", "193.82", "143", "155", "15"): ["NWPS-AKMEKBS", "NWPS"],
("62.24", "193.33", "176", "180", "15"): ["NWPS-AKMENTS", "NWPS"],
("61.0", "183.5", "178", "266", "15"): ["NWPS-AKMEFBK", "NWPS"],
("59.65", "211.23", "189", "193", "15"): ["NWPS-AKMEPWS", "NWPS"],
("50.75", "271.75", "103", "137", "96"): ["GFS-PRMEP5D", "GFS"],
("43.5", "233.72", "136", "81", "15"): ["NWPS-ORMEPT2", "NWPS"],
("46.84", "235.74", "38", "25", "15"): ["NWPS-WAME4", "NWPS"],
("43.5", "233.72", "82", "49", "15"): ["NWPS-ORMEPT3", "NWPS"],
("43.5", "233.72", "203", "121", "15"): ["NWPS-ORMEPT1", "NWPS"],
("19.229", "233.7234", "1597", "2345", "113"): ["SREF-USLCDRS", "SREF"],
("12.19", "226.541", "257", "369", "96"): ["GFS-USLCAW20", "GFS"],
("16.281", "233.862", "337", "451", "193"): ["ICI-USLC13KM", "ICI"],
("41.530708", "267.364016", "361", "581", "133"): ["GLSW-USLC", "GLSW"],
("43.6", "286", "91", "31", "133"): ["GLSW", "GLSW"],
("20.191999", "238.445999", "5505", "8577", "16"): ["ETSS-USLC", "ETSS"],
("20.191999", "238.445999", "1377", "2145", "16"): ["ETSS-USLCDRS", "ETSS"],
("13.5", "283.41", "208", "340", "132"): ["HREF-PRME", "HREF"],
("16.293825", "233.8932", "337", "451", "191"): ["ICN-USLC13KM", "ICNG"],
("12.19", "226.541", "1025", "1473", "132"): ["HREF-USLCEST", "HREF"],
("14.3515", "195.0305", "561", "625", "104"): ["NBM-TRUEME1", "NBM"],
("44.8", "185.5", "603", "825", "132"): ["HREF-AKPS", "HREF"],
("30.67", "278.3", "81", "87", "15"): ["NWPS-SCMECH2", "NWPS"],
("40.530101", "181.429", "1105", "1649", "89"): ["NMMB-AKPSDRS", "NMMB"],
("16.828685", "291.804687", "129", "177", "89"): ["NMMB-TRUEME1", "NMMB"],
("26.5", "278.6", "98", "72", "15"): ["NWPS-FLMEMB1", "NWPS"],
("20.191999", "238.445999", "1597", "2145", "221"): ["WPC-USLCPD10", "WPC"],
("12.349884", "143.686538", "193", "193", "109"): ["RTM-TRUEME4", "RTMA"],
("18.072699", "198.474999", "225", "321", "109"): ["RTM-TRUEME1", "RTMA"],
("-30.4192", "129.9058", "1817", "2517", "104"): ["NBM-TRUEME2", "NBM"],
("12.19", "226.541", "257", "369", "81"): ["GFS-USLCAW20", "GFS"],
("-45.0", "110.0", "725", "837", "96"): ["GFS-TRUEME", "GFS"],
("-45.0", "110.0", "725", "837", "81"): ["GFS-TRUEME", "GFS"],
("18.072699", "198.474999", "225", "321", "17"): ["ESSP-TRUEME", "ESSP"],
("16.9775", "291.9722", "225", "339", "104"): ["NBM-TRUEME3", "NBM"],
("30.000002", "130.0", "301", "511", "11"): ["MGWM-EPME", "MGWM"],
("25.26", "262.0", "84", "66", "15"): ["NWPS-TXMEBR1", "NWPS"],
("20.191999", "238.445999", "1377", "2145", "18"): ["PETS-USLCDRS", "PETS"],
("40.530101", "181.429", "1105", "1649", "18"): ["PETS-AKPSDRS", "PETS"],
("12.349884", "143.686538", "193", "193", "104"): ["NBM-TRUEME4", "NBM"],
("32.5", "279.6", "86", "122", "15"): ["NWPS-NCMEWL2", "NWPS"],
("16.4", "197.65", "170", "223", "132"): ["HREF-HIME", "HREF"],
("16.828685", "291.804687", "129", "177", "109"): ["RTM-TRUEME3", "RTMA"],
("50.75", "271.75", "205", "275", "81"): ["GFS-PRMEP25D", "GFS"],
("40.530101", "181.429", "1105", "1649", "16"): ["ETSS-AKPSDRS", "ETSS"],
("26.0", "261.5", "98", "88", "15"): ["NWPS-TXMECC1", "NWPS"],
("46.13", "235.8", "56", "39", "15"): ["NWPS-ORMECRB", "NWPS"],
("18.072699", "198.474999", "225", "321", "89"): ["NMMB-TRUEME2", "NMMB"],
("20.191999", "238.445999", "5505", "8577", "18"): ["PETS-PD10LC", "PETS"],
("39.75", "285.55", "62", "97", "15"): ["NWPS-NYMENY1", "NWPS"],
("27.41", "264.97", "81", "114", "15"): ["NWPS-LAMELC1", "NWPS"],
("43.15", "235.3", "167", "122", "15"): ["NWPS-ORMECWB", "NWPS"],
("89.277", "0.0", "190", "384", "98"): ["CFS-UNKNGAUS", "CFS"],
("16.977485", "291.972167", "225", "339", "14"): ["ESSA-TRUEME", "ESSA"],
("45.37", "235.8", "89", "47", "15"): ["NWPS-ORMETKB", "NWPS"],
("52.870833", "235.270833", "3351", "6935", "25"): ["SNOW-UNKNME", "SNOW"],
("26.72", "279.86", "156", "80", "15"): ["NWPS-FLMEPAB", "NWPS"],
("38.2", "284.25", "89", "70", "15"): ["NWPS-PAMEMH1", "NWPS"],
("28.7", "278.3", "105", "67", "15"): ["NWPS-FLMEJK2", "NWPS"],
("28.7", "278.3", "204", "131", "15"): ["NWPS-FLMEJK1", "NWPS"],
("26.67", "278.87", "125", "112", "15"): ["NWPS-FLMELKO", "NWPS"],
("25.72", "279.8", "156", "86", "15"): ["NWPS-FLMEMIB", "NWPS"],
("26.0", "279.86", "178", "70", "15"): ["NWPS-FLMEFTL", "NWPS"],
("42.22", "234.9", "181", "140", "15"): ["NWPS-ORMECPB", "NWPS"],
("42.0", "235.55", "56", "75", "15"): ["NWPS-ORMEBGS", "NWPS"],
("90.0", "0.0", "181", "360", "96"): ["GFS-GLME1P0D", "GFS"],
("90.0", "0.0", "181", "360", "81"): ["GFS-GLME1P0D", "GFS"],
("16.8287", "291.8047", "257", "353", "109"): ["RTM-TRUEME2", "RTMA"],
("47.0", "261.0", "481", "586", "11"): ["MGWM-ECGMME", "MGWM"],
("16.8287", "291.8047", "257", "353", "89"): ["NMMB-TRUEME3", "NMMB"],
("21.138", "237.28", "635", "1079", "84"): ["NAM-USLCUNKN", "NAM"],
("19.229", "233.7234", "1597", "2345", "108"): ["LMP-USLCDRS2", "LMP"],
("21.138", "237.28", "635", "1079", "116"): ["WRFE-USLCUNK", "WRFE"],
("36.75", "282.2", "110", "72", "15"): ["NWPS-MDMEBW2", "NWPS"],
("16.8287", "291.8047", "257", "353", "118"): ["URMA-TRUEME1", "URMA"],
("71.995", "184.005", "2200", "5000", "98"): ["OAR-UNKNME1", "CFS"],
("35.349744", "315.0", "1006", "1006", "11"): ["MGWM-UNKNPS", "MGWM"],
("11.7", "141.0", "170", "223", "84"): ["NAM-WPME", "NAM"],
("16.4", "197.65", "170", "223", "84"): ["NAM-HIME", "NAM"],
("25.998", "196.001996", "2200", "2600", "99"): ["OAR-UNKNME2", "TEST"],
("44.8", "185.5", "603", "825", "84"): ["NAM-AKPS", "NAM"],
("20.191924", "238.445993", "1377", "2145", "118"): ["URMA-USLCDR1", "URMA"],
("53.82", "193.25", "300", "296", "15"): ["NWPS-AKMEUNK", "NWPS"],
("54.995", "230.005004", "3500", "7000", "97"): ["MRMS-USME1", "OAR"],
("13.5", "283.41", "208", "340", "84"): ["NAM-PRME", "NAM"],
("57.8", "200.8", "296", "296", "15"): ["NWPS-AKMEUN2", "NWPS"],
("35.0", "234.2", "196", "185", "15"): ["NWPS-CAMEMR4", "NWPS"],
("35.0", "234.2", "140", "132", "15"): ["NWPS-CAMEMR5", "NWPS"],
("37.979669", "234.042695", "795", "709", "118"): ["URMA-USLCDR2", "URMA"],
("40.53", "181.429", "1105", "1649", "118"): ["URMA-AKPSDRS", "URMA"],
("20.191999", "238.445999", "1377", "2145", "0"): ["NDF-USLCDRES", "NDFD"],
}
def watch_filter(change: Change, path: str) -> bool:
if change == Change.deleted:
return False
if Path(path).suffix in GRIB_ENDINGS:
return True
return False
def date_type_from_dt(dt: datetime) -> str:
return dt.strftime("%Y-%m-%d %H:%M:%S.%f")[:-5]
def date_type_from_str(date_str: str, format: str) -> str:
return date_type_from_dt(datetime.strptime(date_str, format))
def amqp_grib(grib_path: str, start_message: int) -> Generator[dict, None, None]:
with grib2io.open(grib_path) as grib_file: # Grib2Message._msg_class_store might lead to memory leakage
for msg in grib_file[start_message:]:
f_lat = getattr(msg, "latitudeFirstGridpoint", None)
f_lon = getattr(msg, "longitudeFirstGridpoint", None)
rows = msg.ny
cols = msg.nx
gen_proc = msg.generatingProcess
xcd_info = XCD_MODELS.get((str(f_lat), str(f_lon), str(rows), str(cols), str(gen_proc.value)), ['UKNWN', 'UKNWN'])
yield {
"__payload_gen_time__": date_type_from_dt(datetime.now()),
"path": grib_path,
"directory": os.path.dirname(grib_path),
"file_name": os.path.basename(grib_path),
"server_ip": HOSTNAME,
"server_type": "realtime",
"first_lat": f_lat,
"last_lat": getattr(msg, "latitudeLastGridpoint", None),
"first_lon": f_lon,
"last_long": getattr(msg, "longitudeLastGridpoint", None),
"forecast_hour": int(msg.valueOfForecastTime),
"run_hour": int(msg.hour) * 100 + int(msg.minute),
"model_time": date_type_from_dt(msg.refDate),
"projection": msg.gridDefinitionTemplateNumber.definition,
"center_id": int(msg.originatingCenter.value),
"center_desc": msg.originatingCenter.definition,
"level": msg.level,
"parameter": msg.shortName,
"param_long": msg.fullName,
"param_units": msg.units,
"grib_number": 2,
"size": "%d x %d" % (rows, cols),
"ensemble": getattr(msg, "pertubatyionNumber", None),
"model_id": int(gen_proc.value),
"model": gen_proc.definition,
"xcd_model": xcd_info[1],
"xcd_model_id": xcd_info[0],
# "resolution": msg._xydivisor, THIS IS NOT CORRECT!
"grib_record_number": msg._msgnum,
"title": str(msg),
}
def main():
if not load_dotenv():
print("Couldn't find .env file")
sys.exit(1)
cl = AmqpClient()
cl.connect(
AmqpExchange(
"mq1.ssec.wisc.edu", os.getenv("RMQ_USER"), os.getenv("RMQ_PASS"), "model"
)
)
mem = defaultdict(int)
avg_time = 0
for changes in watch("/data/xcd/grib/", watch_filter=watch_filter, debounce=5000):
for _, path in changes:
start_time = time.time()
i = 0
for i, amqp in enumerate(amqp_grib(path, mem[path])):
cl.publish(amqp, route_key=f"{amqp['xcd_model']}.{amqp['xcd_model_id']}.realtime.reserved.reserved.2")
print(json.dumps(amqp, indent=4))
mem[path] += i
print(path, mem[path])
avg_time -= avg_time / 100
avg_time += (time.time() - start_time) / 100
print(avg_time)
if __name__ == "__main__":
sys.exit(main())
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment