diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..32dabf3dba443d05684f0c2012464809855c65aa
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,19 @@
+repos:
+ - repo: 'https://github.com/pre-commit/pre-commit-hooks'
+ rev: v4.6.0
+ hooks:
+ - id: check-yaml
+ - id: debug-statements
+ - id: end-of-file-fixer
+ - id: trailing-whitespace
+ - repo: 'https://github.com/pre-commit/mirrors-mypy'
+ rev: v1.10.0
+ hooks:
+ - id: mypy
+ - repo: 'https://github.com/astral-sh/ruff-pre-commit'
+ rev: v0.5.2
+ hooks:
+ - id: ruff
+ args:
+ - '--fix'
+ - id: ruff-format
diff --git a/CONTRIBUTING b/CONTRIBUTING
index 7b8155e8d7a2a9ceda60853393fb43e136ccd893..ab4c2b2dfd84b4c7b498863839753d7d83b4fb6d 100644
--- a/CONTRIBUTING
+++ b/CONTRIBUTING
@@ -2,16 +2,62 @@
How to set up your dev environment to work on the grib processor.
-First, create a virtual environment like in the [readme](/README.md#setup).
+## Goals
-Then, install ruff and mypy.
+The main goal of this package is to provide a realtime event processor for the SDS that publishes metadata about incoming grib files. As this package is made for the SDS, final say on creative decision goes to the SDS managers.
-Before pushing commits run:
+The main goal can be accomplished using the `--realtime` CLI option.
-`ruff format grib_processor.py`
+Additional (secondary) goals include:
-`ruff check grib_processor.py`
+* Providing different ways to ingest grib files; stdin, specifying one file, etc.
+* Providing a python interface to the event processor's components.
+* Providing an interface to extract metadata from grib files.
-`mypy grib_processor.py`
+## Setup
->Note: mypy will give an error about untyped imports, this can be ignored.
+First, clone the repo locally.
+
+```bash
+git clone https://gitlab.ssec.wisc.edu/mdrexler/grib_rmq_stream.git
+cd grib_rmq_stream
+```
+
+Next, install the package from source and create a virtual environment like in the [readme](/README.md#setup).
+
+Then, install the dev dependencies.
+
+```bash
+python -m pip install -r requirements_dev.txt
+```
+
+Finally, install the package in editable mode.
+
+```bash
+python -m pip install -e .
+```
+
+### Pre-commit
+
+[pre-commit](https://pre-commit.com/) can be used to automatically format/lint/type check any new commits you make. It is recommended, but not necessary as the CI will also check that for you.
+
+```bash
+pre-commit install
+pre-commit run --all-files
+```
+
+## Testing
+
+When adding a new feature, please add corresponding unit tests under [tests/](/tests/).
+
+Tests can be run using pytest.
+
+```bash
+pytest tests/
+```
+
+To get the code coverage use the coverage command.
+
+```bash
+coverage run -m pytest
+```
diff --git a/LICENSE b/LICENSE
index 96a53d6f8e82a5c31fabee4c77e7d0762d47d574..0e1fe8c75da00595623b1e7dc0e18a28a407173d 100644
--- a/LICENSE
+++ b/LICENSE
@@ -4,4 +4,4 @@ Permission is hereby granted, free of charge, to any person obtaining a copy of
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
\ No newline at end of file
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/payload.md b/docs/payload.md
similarity index 97%
rename from payload.md
rename to docs/payload.md
index 54f43ee8a575d79cd52c4ef5476283e98a106317..41c5359da244541e5f27bd999ca0d6b04934bc03 100644
--- a/payload.md
+++ b/docs/payload.md
@@ -46,6 +46,7 @@ This is the code that generates the payload. The payload is then converted into
| Name | Type | Meaning |
| :--: | :--: | :-----: |
| `__payload_gen_time__` | datetime (str) | when the payload was generated. |
+| `__injector_script__` | str | the user@server:path/to/script that generated the payload. |
| `path` | str | fully qualified path to the grib file with this message. |
| `directory` | str | directory of `path`. |
| `file_name` | str | file name of `path`. |
@@ -64,7 +65,7 @@ This is the code that generates the payload. The payload is then converted into
| `center_desc` | str | name of the generating center. |
| `level` | str | level of the grid message. |
| `parameter` | str | short name of the grid message parameter. |
-| `param_long` | str | long name of the grid message paramter. |
+| `param_long` | str | long name of the grid message paramter. |
| `param_units` | str | units of the grid message parameter. |
| `grib_number` | int literal | grid edition number, e.g. 1 or 2. |
| `size` | str | number of rows in cols in grid message formatted as "{rows} x {cols}". |
@@ -74,4 +75,4 @@ This is the code that generates the payload. The payload is then converted into
| `xcd_model` | str | name of the xcd grib model. |
| `xcd_model_id` | str | name of the xcd grib model id. |
| `grib_record_number` | int | the index of this grid message in the file. |
-| `title` | str | the wgrib2 formatted string for this grid message. |
\ No newline at end of file
+| `title` | str | the wgrib2 formatted string for this grid message. |
diff --git a/grib_processor b/grib_processor
deleted file mode 100755
index 4b96fb219bf36bc2010d4baada74c127645503f6..0000000000000000000000000000000000000000
--- a/grib_processor
+++ /dev/null
@@ -1,9 +0,0 @@
-#! /bin/bash
-
-SOURCE="${BASH_SOURCE[0]}"
-while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
-BASE="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-
-PYTHON=${PYTHON:-python3}
-
-exec $PYTHON $BASE/grib_processor.py "$@"
diff --git a/grib_processor.py b/grib_processor.py
deleted file mode 100644
index d8f8935bf35d232838c857b55b34794cd0e1271a..0000000000000000000000000000000000000000
--- a/grib_processor.py
+++ /dev/null
@@ -1,1143 +0,0 @@
-"""
-Ingest grib files and publish metadata for files to RabbitMQ.
-"""
-
-from __future__ import annotations
-
-__author__ = "Max Drexler"
-__email__ = "mndrexler@wisc.edu"
-
-
-import argparse
-from collections import defaultdict
-from datetime import datetime
-import os
-import logging
-import logging.handlers
-import sys
-import time
-import threading
-from typing import DefaultDict, Generator
-
-import grib2io
-from watchfiles import watch, Change
-import ssec_amqp.api as mq
-import ssec_amqp.utils as amqp_utils
-from dotenv import load_dotenv
-
-
-if sys.version_info < (3, 8):
- raise SystemError("Python version too low")
-
-# logging stuff
-LOG = logging.getLogger("grib_ingest")
-LOG_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "logs")
-LOG_LEVELS = [
- logging.CRITICAL,
- logging.ERROR,
- logging.WARNING,
- logging.INFO,
- logging.DEBUG,
-]
-LOG_NAME = os.path.splitext(os.path.basename(sys.argv[0]))[0] + ".log"
-
-# Where we're publishing data to
-DEFAULT_AMQP_SERVERS = ["mq1.ssec.wisc.edu", "mq2.ssec.wisc.edu", "mq3.ssec.wisc.edu"]
-
-# Where on the servers we're putting the data.
-DEFAULT_AMQP_EXCHANGE = "model"
-
-# RabbitMQ default login details (shouldn't be authenticated to publish).
-DEFAULT_AMQP_USER = "guest"
-DEFAULT_AMQP_PASS = "guest"
-
-# Format of the route key to publish to
-ROUTE_KEY_FMT = "{xcd_model}.{xcd_model_id}.realtime.reserved.reserved.2"
-
-# What file extensions to parse as grib files
-GRIB_ENDINGS = [".grb", ".grib", ".grb2", ".grib2"]
-
-# How long grib files stay in the message cache.
-CACHE_TTL = 43200 # (12 hours)
-
-# How often to clean the cache of previous messages.
-CACHE_CLEAN_INTERVAL = 5400 # (1.5 hours)
-
-# Table containing XCD model and model ids for grib messages
-# [first_lat][first_lon][rows][cols][gen_proc_id]
-XCD_MODELS = {
- "44.196": {"174.759": {"237": {"377": {"115": ["DGEX-AKPS", "DGEX"]}}}},
- "19.943": {"234.907": {"303": {"491": {"115": ["DGEX-USLC", "DGEX"]}}}},
- "0.0": {
- "0.0": {
- "181": {"720": {"30": ["FGF-NHME", "FGF"]}},
- "91": {"360": {"96": ["GFS-NHME", "GFS"]}},
- },
- "180.0": {"500": {"625": {"85": ["RTOF-HIME", "RTOF"]}}},
- "130.0": {"375": {"625": {"85": ["RTOF-WPME", "RTOF"]}}},
- },
- "90.0": {
- "0.0": {
- "181": {
- "360": {
- "107": ["GEFS-GLME", "GEFS"],
- "96": ["GFS-GLME1P0D", "GFS"],
- "81": ["GFS-GLME1P0D", "GFS"],
- }
- },
- "336": {"720": {"11": ["MGWM-GLME", "MGWM"]}},
- "73": {
- "144": {"96": ["GFS-GLME2P5D", "GFS"], "81": ["GFS-GLME2P5D", "GFS"]}
- },
- "361": {
- "720": {"96": ["GFS-GLME0P5D", "GFS"], "81": ["GFS-GLME0P5D", "GFS"]}
- },
- "145": {
- "288": {"96": ["GFS-GLME1P25", "GFS"], "81": ["GFS-GLME1P25", "GFS"]}
- },
- "721": {
- "1440": {"96": ["GFS-GLMEP25D", "GFS"], "81": ["GFS-GLMEP25D", "GFS"]}
- },
- }
- },
- "40.5301": {
- "181.429": {
- "1105": {
- "1649": {
- "96": ["GFS-AKPSDRES", "GFS"],
- "81": ["GFS-AKPSDRES", "GFS"],
- "114": ["NAEF-AKPSDRS", "NAEF"],
- "0": ["MOS-AKPSDRES", "MOS"],
- }
- }
- }
- },
- "19.132": {
- "174.163": {
- "156": {
- "180": {"81": ["GFS-AKPSHRES", "GFS"], "96": ["GFS-AKPSHRES", "GFS"]}
- }
- }
- },
- "-20.826": {
- "210.0": {"65": {"65": {"81": ["GFS-NHPS", "GFS"], "96": ["GFS-NHPS", "GFS"]}}}
- },
- "12.19": {
- "226.541": {
- "129": {
- "185": {
- "81": ["GFS-USLC2", "GFS"],
- "96": ["GFS-USLC2", "GFS"],
- "113": ["SREF-USLC2", "SREF"],
- "89": ["NMMB-USLCA40", "NMMB"],
- "116": ["WRFE-USLC40K", "WRFE"],
- "112": ["WRFN-USLC40K", "WRFN"],
- "84": ["NAM-USLC2", "NAM"],
- "111": ["NAM-USLC2", "NAM"],
- }
- },
- "428": {"614": {"84": ["NAM-USLCAW12", "NAM"]}},
- "257": {
- "369": {"96": ["GFS-USLCAW20", "GFS"], "81": ["GFS-USLCAW20", "GFS"]}
- },
- "1025": {"1473": {"132": ["HREF-USLCEST", "HREF"]}},
- }
- },
- "20.192": {
- "238.446": {
- "689": {
- "1073": {"96": ["GFS-USLCAWI4", "GFS"], "108": ["LMP-USLCAWI4", "LMP"]}
- },
- "1377": {
- "2145": {
- "96": ["GFS-USLCDRS1", "GFS"],
- "108": ["LMP-USLCDRES", "LMP"],
- "183": ["NDF-USLCDRES", "NDF"],
- "89": ["NMMB-USLCDRS", "NMMB"],
- "0": ["MOS-USLCDRES", "MOS"],
- }
- },
- "1597": {
- "2145": {"104": ["NBM-USLCDRS2", "NBM"], "96": ["GFS-USLCDRS2", "GFS"]}
- },
- },
- "238.45": {"689": {"1073": {"89": ["NMMB-USLCAW4", "NMMB"]}}},
- },
- "7.838": {
- "218.972": {
- "85": {"129": {"81": ["GFS-USPS", "GFS"], "96": ["GFS-USPS", "GFS"]}}
- }
- },
- "49.1": {"267.799984": {"235": {"327": {"131": ["GLWM", "GLWM"]}}}},
- "89.958": {
- "0.042": {
- "2160": {"4320": {"120": ["ICA-GLME", "ICA"], "44": ["SST-GLME", "SST"]}}
- }
- },
- "16.281": {
- "233.862": {
- "337": {
- "451": {
- "191": ["ICN-USLC13KM", "ICNG"],
- "105": ["RAP-USLC13KM", "RAP"],
- "193": ["ICI-USLC13KM", "ICI"],
- }
- },
- "113": {"151": {"105": ["RAP-USLC40KM", "RAP"]}},
- "225": {"301": {"105": ["RAP-USLC20KM", "RAP"]}},
- },
- "233.861999": {"337": {"451": {"105": ["RAP-USLC13KM", "RAP"]}}},
- },
- "74.0": {"165.0": {"391": {"548": {"11": ["MGWM-AKME", "MGWM"]}}}},
- "30.0": {
- "130.0": {"301": {"511": {"11": ["MGWM-EPME", "MGWM"]}}},
- "187.0": {
- "425": {"553": {"84": ["NAM-AKPSAWI4", "NAM"]}},
- "107": {
- "139": {"113": ["SREF-AKPS", "SREF"], "84": ["NAM-AKPS45KM", "NAM"]}
- },
- "213": {
- "277": {"96": ["GFS-AKPSAWI2", "GFS"], "81": ["GFS-AKPSAWI2", "GFS"]}
- },
- },
- },
- "75.0": {"140.0": {"187": {"401": {"11": ["MGWM-NPME", "MGWM"]}}}},
- "55.0": {
- "260.0": {"331": {"301": {"11": ["MGWM-WAME", "MGWM"]}}},
- "202.0": {
- "62": {"81": {"15": ["NWPS-AKMEAN1", "NWPS"]}},
- "147": {"193": {"15": ["NWPS-AKMEAN2", "NWPS"]}},
- },
- },
- "50.0": {
- "195.0": {"526": {"736": {"11": ["MGWM-WCHIME", "MGWM"]}}},
- "210.0": {"151": {"241": {"11": ["MGWM-WCME", "MGWM"]}}},
- },
- "1.0": {
- "214.5": {
- "277": {
- "349": {
- "111": ["NAM-USLC32KM", "NAM"],
- "116": ["WRFE-USLC32K", "WRFE"],
- "112": ["WRFN-USLC32K", "WRFN"],
- }
- }
- }
- },
- "20.190001": {
- "238.449996": {
- "689": {
- "1073": {
- "180": ["NCE-USLCAWI4", "NCE"],
- "224": ["WPC-USLCAWI4", "WPC"],
- "221": ["WPC-USLCAWI4", "WPC"],
- }
- }
- }
- },
- "40.530094": {"181.429031": {"553": {"825": {"183": ["NDF-AKPS2", "NDF"]}}}},
- "40.530096": {"181.429024": {"553": {"825": {"183": ["NDF-AKPS2", "NDF"]}}}},
- "40.529998": {"181.429993": {"553": {"825": {"183": ["NDF-AKPS", "NDF"]}}}},
- "40.530101": {
- "181.429": {
- "1105": {
- "1649": {
- "0": ["NDF-AKPSDRS", "NDF"],
- "109": ["RTM-AKPSDRES", "RTMA"],
- "89": ["NMMB-AKPSDRS", "NMMB"],
- "18": ["PETS-AKPSDRS", "PETS"],
- "16": ["ETSS-AKPSDRS", "ETSS"],
- }
- },
- "553": {
- "825": {
- "12": ["PSS-AKPS", "PSS"],
- "109": ["RTM-AKPS", "RTMA"],
- "17": ["ESSP-AKPS", "ESSP"],
- }
- },
- }
- },
- "20.191999": {
- "238.445999": {
- "1377": {
- "2145": {
- "183": ["NDF-USLCDRES", "NDF"],
- "109": ["RTM-USLCDRES", "RTMA"],
- "118": ["URMA-USLCDR1", "URMA"],
- "83": ["HRR-USLCDRES", "HRR"],
- "14": ["ESSA-USLCDRS", "ESSA"],
- "17": ["ESSP-USLCDRS", "ESSP"],
- "12": ["PSS-USLCDRES", "PSS"],
- "114": ["NAEF-USLCDRS", "NAEF"],
- "222": ["WPC-USLCDRES", "WPC"],
- "223": ["WPC-USLCDRES", "WPC"],
- "96": ["GFS-USLCDRS1", "GFS"],
- "16": ["ETSS-USLCDRS", "ETSS"],
- "18": ["PETS-USLCDRS", "PETS"],
- "0": ["NDF-USLCDRES", "NDFD"],
- }
- },
- "689": {
- "1073": {"109": ["RTM-USLCAWI4", "RTMA"], "12": ["PSS-USLCAWI4", "PSS"]}
- },
- "1597": {
- "2145": {"180": ["NCE-USLCDRS", "NCEP"], "221": ["WPC-USLCPD10", "WPC"]}
- },
- "5505": {
- "8577": {"16": ["ETSS-USLC", "ETSS"], "18": ["PETS-PD10LC", "PETS"]}
- },
- }
- },
- "40.53": {
- "181.429": {
- "553": {"825": {"89": ["NMMB-AKPS", "NMMB"]}},
- "1105": {
- "1649": {
- "89": ["NMMB-AKPSDRS", "NMMB"],
- "0": ["MOS-AKPSDRES", "MOS"],
- "104": ["NBM-AKPSDRS", "NBM"],
- "118": ["URMA-AKPSDRS", "URMA"],
- }
- },
- }
- },
- "-40.0": {"130.0": {"80": {"120": {"85": ["RTOF-APME", "RTOF"]}}}},
- "40.0": {
- "140.0": {"150": {"350": {"85": ["RTOF-ARCTME", "RTOF"]}}},
- "251.0": {"152": {"328": {"85": ["RTOF-NATLME", "RTOF"]}}},
- "195.0": {"45": {"84": {"85": ["RTOF-NEPME", "RTOF"]}}},
- "155.0": {"340": {"700": {"85": ["RTOF-NPME", "RTOF"]}}},
- },
- "-30.0": {"170.0": {"375": {"560": {"85": ["RTOF-SCPME", "RTOF"]}}}},
- "10.0": {
- "260.0": {"435": {"575": {"85": ["RTOF-USECME", "RTOF"]}}},
- "210.0": {"625": {"625": {"85": ["RTOF-USWCME", "RTOF"]}}},
- "190.0": {"101": {"126": {"113": ["SREF-USME", "SREF"]}}},
- },
- "37.979684": {"234.042704": {"795": {"709": {"118": ["URMA-USLCDR2", "URMA"]}}}},
- "44.8": {
- "185.5": {
- "603": {
- "825": {
- "116": ["WRFE-AKPS", "WRFE"],
- "112": ["WRFN-AKPS", "WRFN"],
- "132": ["HREF-AKPS", "HREF"],
- "84": ["NAM-AKPS", "NAM"],
- }
- }
- }
- },
- "16.4": {
- "197.65": {
- "170": {
- "223": {
- "116": ["WRFE-HIME", "WRFE"],
- "112": ["WRFN-HIME", "WRFN"],
- "132": ["HREF-HIME", "HREF"],
- "84": ["NAM-HIME", "NAM"],
- }
- }
- }
- },
- "11.7": {
- "141.0": {
- "170": {
- "223": {
- "116": ["WRFE-WPME", "WRFE"],
- "112": ["WRFN-WPME", "WRFN"],
- "84": ["NAM-WPME", "NAM"],
- }
- }
- }
- },
- "13.5": {
- "283.41": {
- "208": {
- "340": {
- "116": ["WRFE-PRME", "WRFE"],
- "112": ["WRFN-PRME", "WRFN"],
- "132": ["HREF-PRME", "HREF"],
- "84": ["NAM-PRME", "NAM"],
- }
- }
- }
- },
- "22.1": {
- "250.2": {
- "614": {
- "884": {"116": ["WRFE-USLCSE", "WRFE"], "112": ["WRFN-USLCSE", "WRFN"]}
- }
- }
- },
- "24.5": {
- "230.8": {
- "614": {
- "884": {"116": ["WRFE-USLCSW", "WRFE"], "112": ["WRFN-USLCSW", "WRFN"]}
- }
- }
- },
- "50.75": {
- "271.75": {
- "205": {
- "275": {"96": ["GFS-PRMEP25D", "GFS"], "81": ["GFS-PRMEP25D", "GFS"]}
- },
- "102": {"137": {"96": ["GFS-WAME", "GFS"], "81": ["GFS-WAME", "GFS"]}},
- "103": {"137": {"96": ["GFS-PRMEP5D", "GFS"]}},
- }
- },
- "-0.268": {"220.525": {"110": {"147": {"84": ["NAM-USPS", "NAM"]}}}},
- "41.530708": {
- "267.364016": {
- "361": {"581": {"131": ["GLW-USLC", "GLWM"], "133": ["GLSW-USLC", "GLSW"]}}
- }
- },
- "49.099998": {"267.799988": {"235": {"327": {"131": ["GLW-MWUSME", "GLWM"]}}}},
- "60.0": {"160.0": {"250": {"950": {"85": ["RTOF-SARCME", "RTOF"]}}}},
- "35.0": {
- "170.0": {
- "225": {"277": {"96": ["GFS-NPPS", "GFS"], "81": ["GFS-NPPS", "GFS"]}}
- },
- "234.2": {
- "123": {"116": {"15": ["NWPS-CAMEMR1", "NWPS"]}},
- "109": {"103": {"15": ["NWPS-CAMEMR2", "NWPS"]}},
- "98": {"93": {"15": ["NWPS-CAMEMR3", "NWPS"]}},
- "196": {"185": {"15": ["NWPS-CAMEMR4", "NWPS"]}},
- "140": {"132": {"15": ["NWPS-CAMEMR5", "NWPS"]}},
- },
- },
- "54.995": {
- "230.005": {"3500": {"7000": {"97": ["MRMS-USME1", "OAR"]}}},
- "230.005992": {"3500": {"7000": {"97": ["MRMS-USME4", "OAR"]}}},
- "230.005004": {"3500": {"7000": {"97": ["MRMS-USME1", "OAR"]}}},
- },
- "54.9975": {"230.0025": {"7000": {"14000": {"97": ["MRMS-USME2", "OAR"]}}}},
- "54.95": {"230.05": {"350": {"700": {"97": ["MRMS-USME3", "OAR"]}}}},
- "35.8": {
- "282.7": {
- "138": {
- "92": {"15": ["NWPS-VAMEWF2", "NWPS"]},
- "91": {"15": ["NWPS-DMVAME4", "NWPS"]},
- },
- "229": {"153": {"15": ["NWPS-VAMEWF1", "NWPS"]}},
- }
- },
- "25.45": {
- "275.2": {
- "263": {"185": {"15": ["NWPS-FLMETB2", "NWPS"]}},
- "119": {"83": {"15": ["NWPS-FLMETB3", "NWPS"]}},
- }
- },
- "24.1": {
- "276.46": {
- "134": {"174": {"15": ["NWPS-FLMESF1", "NWPS"]}},
- "201": {"261": {"15": ["NWPS-FLMESF2", "NWPS"]}},
- }
- },
- "33.85": {
- "282.0": {
- "170": {"167": {"15": ["NWPS-NCMEMC1", "NWPS"]}},
- "62": {"61": {"15": ["NWPS-NCMEMC2", "NWPS"]}},
- }
- },
- "36.0": {
- "284.32": {"112": {"90": {"15": ["NWPS-NCMEKDH", "NWPS"]}}},
- "284.2": {"67": {"54": {"15": ["NWPS-NCMENOB", "NWPS"]}}},
- },
- "39.75": {
- "285.762": {"62": {"91": {"15": ["NWPS-NYMENY1", "NWPS"]}}},
- "285.55": {
- "102": {"162": {"15": ["NWPS-NYMENY2", "NWPS"]}},
- "62": {"97": {"15": ["NWPS-NYMENY1", "NWPS"]}},
- },
- },
- "40.83": {"287.25": {"61": {"90": {"15": ["NWPS-NYMELI1", "NWPS"]}}}},
- "40.4": {
- "285.7": {
- "37": {"41": {"15": ["NWPS-LIME4", "NWPS"]}},
- "74": {"82": {"15": ["NWPS-DEMEDB2", "NWPS"]}},
- }
- },
- "40.54": {"286.22": {"29": {"85": {"15": ["NWPS-NYMELI2", "NWPS"]}}}},
- "27.41": {
- "264.97": {
- "74": {"114": {"15": ["NWPS-LATXME1", "NWPS"]}},
- "179": {"253": {"15": ["NWPS-LAMELC2", "NWPS"]}},
- "81": {"114": {"15": ["NWPS-LAMELC1", "NWPS"]}},
- }
- },
- "25.26": {
- "262.359": {"84": {"57": {"15": ["NWPS-TXME3", "NWPS"]}}},
- "262.0": {
- "186": {"145": {"15": ["NWPS-TXMEBR2", "NWPS"]}},
- "84": {"66": {"15": ["NWPS-TXMEBR1", "NWPS"]}},
- },
- },
- "26.0": {
- "262.345": {"90": {"67": {"15": ["NWPS-TXME1", "NWPS"]}}},
- "261.5": {
- "217": {"195": {"15": ["NWPS-TXMECC2", "NWPS"]}},
- "98": {"88": {"15": ["NWPS-TXMECC1", "NWPS"]}},
- },
- "279.86": {"178": {"70": {"15": ["NWPS-FLMEFTL", "NWPS"]}}},
- },
- "26.13": {"278.1": {"75": {"84": {"15": ["NWPS-SWFLME1", "NWPS"]}}}},
- "26.18": {"279.89": {"65": {"59": {"15": ["NWPS-SEFLME1", "NWPS"]}}}},
- "26.89": {"279.93": {"47": {"42": {"15": ["NWPS-SEFLME2", "NWPS"]}}}},
- "25.85": {"279.87": {"167": {"101": {"15": ["NWPS-SEFLME3", "NWPS"]}}}},
- "28.5": {
- "270.9": {
- "61": {"79": {"15": ["NWPS-MSALME1", "NWPS"]}},
- "155": {"174": {"15": ["NWPS-ALMEMP2", "NWPS"]}},
- "70": {"79": {"15": ["NWPS-ALMEMP1", "NWPS"]}},
- }
- },
- "40.45": {
- "287.5": {
- "104": {"103": {"15": ["NWPS-MAMEBT1", "NWPS"]}},
- "156": {"155": {"15": ["NWPS-MAMEBT2", "NWPS"]}},
- }
- },
- "41.9": {"288.8": {"289": {"149": {"15": ["NWPS-MAMEBT3", "NWPS"]}}}},
- "43.47": {"288.5": {"289": {"251": {"15": ["NWPS-NHMELKW", "NWPS"]}}}},
- "43.76": {"289.36": {"189": {"145": {"15": ["NWPS-MEMELKS", "NWPS"]}}}},
- "42.4": {
- "289.021": {"67": {"74": {"15": ["NWPS-MEME7", "NWPS"]}}},
- "288.25": {
- "103": {"129": {"15": ["NWPS-MEMEGP2", "NWPS"]}},
- "74": {"92": {"15": ["NWPS-MEMEGP1", "NWPS"]}},
- },
- },
- "27.0": {
- "262.6": {
- "217": {"222": {"15": ["NWPS-TXMEHG2", "NWPS"]}},
- "84": {"100": {"15": ["NWPS-TXMEHG4", "NWPS"]}},
- "98": {"100": {"15": ["NWPS-TXMEHG1", "NWPS"]}},
- }
- },
- "30.93": {"278.53": {"81": {"42": {"15": ["NWPS-GAMESAS", "NWPS"]}}}},
- "30.62": {"278.51": {"89": {"111": {"15": ["NWPS-GAMEKBY", "NWPS"]}}}},
- "38.5": {"284.3": {"151": {"92": {"15": ["NWPS-DEMEDB1", "NWPS"]}}}},
- "24.3": {"276.8": {"123": {"315": {"15": ["NWPS-FLMEFKY", "NWPS"]}}}},
- "23.0": {
- "276.5": {
- "84": {"116": {"15": ["NWPS-FLMEKW1", "NWPS"]}},
- "96": {"132": {"15": ["NWPS-FLMEKW2", "NWPS"]}},
- }
- },
- "27.48": {"276.8": {"182": {"232": {"15": ["NWPS-FLMETB1", "NWPS"]}}}},
- "38.2": {
- "284.424": {"89": {"65": {"15": ["NWPS-PAMEMH1", "NWPS"]}}},
- "284.25": {
- "149": {"117": {"15": ["NWPS-PAMEMH2", "NWPS"]}},
- "89": {"70": {"15": ["NWPS-PAMEMH1", "NWPS"]}},
- },
- },
- "28.2": {"279.2": {"134": {"99": {"15": ["NWPS-FLMECPC", "NWPS"]}}}},
- "26.5": {
- "278.681": {"98": {"70": {"15": ["NWPS-NFLAME1", "NWPS"]}}},
- "278.6": {
- "217": {"160": {"15": ["NWPS-FLMEMB2", "NWPS"]}},
- "98": {"72": {"15": ["NWPS-FLMEMB1", "NWPS"]}},
- },
- },
- "29.87": {"278.68": {"34": {"26": {"15": ["NWPS-FLMESTA", "NWPS"]}}}},
- "30.38": {"278.57": {"28": {"34": {"15": ["NWPS-FLMEMPT", "NWPS"]}}}},
- "17.0": {
- "292.0": {
- "93": {"142": {"15": ["NWPS-PRMESJ1", "NWPS"]}},
- "155": {"237": {"15": ["NWPS-PRMESJ2", "NWPS"]}},
- }
- },
- "17.55": {"292.6": {"140": {"319": {"15": ["NWPS-PRMESJ3", "NWPS"]}}}},
- "18.33": {"292.7": {"212": {"212": {"15": ["NWPS-PRMERIN", "NWPS"]}}}},
- "29.0": {
- "264.82": {"178": {"141": {"15": ["NWPS-TXMEHG3", "NWPS"]}}},
- "276.45": {"89": {"104": {"15": ["NWPS-FLMECKY", "NWPS"]}}},
- },
- "28.35": {
- "272.65": {
- "62": {
- "116": {"15": ["NWPS-FLMETL3", "NWPS"]},
- "115": {"15": ["NWPS-FLMETL4", "NWPS"]},
- },
- "140": {"259": {"15": ["NWPS-FLMETL1", "NWPS"]}},
- "63": {"117": {"15": ["NWPS-FLMETL2", "NWPS"]}},
- }
- },
- "30.15": {
- "271.55": {"104": {"103": {"15": ["NWPS-ALMEMBY", "NWPS"]}}},
- "273.2": {"60": {"94": {"15": ["NWPS-FLMECBY", "NWPS"]}}},
- },
- "30.1": {"272.68": {"82": {"70": {"15": ["NWPS-FLMEESP", "NWPS"]}}}},
- "30.08": {"272.32": {"70": {"62": {"15": ["NWPS-FLMEPBY", "NWPS"]}}}},
- "30.67": {
- "278.3": {
- "180": {"192": {"15": ["NWPS-SCMECH1", "NWPS"]}},
- "81": {"87": {"15": ["NWPS-SCMECH2", "NWPS"]}},
- },
- "278.384": {"81": {"85": {"15": ["NWPS-SCARME2", "NWPS"]}}},
- },
- "36.75": {
- "282.2": {
- "164": {"107": {"15": ["NWPS-MDMEBW1", "NWPS"]}},
- "110": {"72": {"15": ["NWPS-MDMEBW2", "NWPS"]}},
- }
- },
- "43.3": {
- "290.8": {
- "109": {"131": {"15": ["NWPS-MEMECB1", "NWPS"]}},
- "49": {"59": {"15": ["NWPS-MEMECB2", "NWPS"]}},
- }
- },
- "44.16": {"291.58": {"156": {"160": {"15": ["NWPS-MEMEBHR", "NWPS"]}}}},
- "44.27": {"291.83": {"167": {"168": {"15": ["NWPS-MEMEWHR", "NWPS"]}}}},
- "32.5": {
- "279.6": {
- "103": {"147": {"15": ["NWPS-NCMEWL1", "NWPS"]}},
- "85": {"122": {"15": ["NWPS-NCMEWL2", "NWPS"]}},
- "86": {"122": {"15": ["NWPS-NCMEWL2", "NWPS"]}},
- }
- },
- "34.4": {"282.73": {"83": {"68": {"15": ["NWPS-NCMESOB", "NWPS"]}}}},
- "34.6": {"282.85": {"99": {"153": {"15": ["NWPS-NCMEEMI", "NWPS"]}}}},
- "29.85": {"275.5": {"56": {"137": {"15": ["NWPS-FLMESTM", "NWPS"]}}}},
- "29.6": {"274.54": {"65": {"99": {"15": ["NWPS-FLMECSB", "NWPS"]}}}},
- "29.95": {"274.13": {"78": {"81": {"15": ["NWPS-FLMEPCY", "NWPS"]}}}},
- "29.3": {"269.4": {"98": {"140": {"15": ["NWPS-LAMELPN", "NWPS"]}}}},
- "27.5": {
- "268.2": {
- "87": {"109": {"15": ["NWPS-LAMENB1", "NWPS"]}},
- "192": {"242": {"15": ["NWPS-LAMENB2", "NWPS"]}},
- }
- },
- "16.201": {"285.72": {"94": {"108": {"105": ["RAP-PRLC16KM", "RAP"]}}}},
- "20.19": {"238.45": {"689": {"1073": {"0": ["SPC-USLCAWI4", "SPC"]}}}},
- "32.6": {
- "236.64": {
- "98": {"142": {"15": ["NWPS-CAMELA1", "NWPS"]}},
- "112": {"163": {"15": ["NWPS-CAMELA2", "NWPS"]}},
- "78": {"114": {"15": ["NWPS-CAMELA3", "NWPS"]}},
- }
- },
- "32.08": {
- "241.0": {
- "97": {"118": {"15": ["NWPS-CAMESD1", "NWPS"]}},
- "65": {"79": {"15": ["NWPS-CAMESD2", "NWPS"]}},
- }
- },
- "33.52": {"241.59": {"56": {"112": {"15": ["NWPS-CAMEPLB", "NWPS"]}}}},
- "31.95": {"240.8": {"37": {"44": {"15": ["NWPS-CAMESD3", "NWPS"]}}}},
- "32.68": {"242.66": {"195": {"79": {"15": ["NWPS-CAMECLJ", "NWPS"]}}}},
- "37.3": {"236.7": {"123": {"89": {"15": ["NWPS-CAMESFB", "NWPS"]}}}},
- "36.2": {"237.8": {"178": {"90": {"15": ["NWPS-CAMEMRB", "NWPS"]}}}},
- "53.9": {
- "215.5": {
- "177": {"237": {"15": ["NWPS-AKMEJN1", "NWPS"]}},
- "59": {"79": {"15": ["NWPS-AKMEJN2", "NWPS"]}},
- }
- },
- "57.0": {"222.6": {"300": {"297": {"15": ["NWPS-AKMEGBY", "NWPS"]}}}},
- "54.5": {"224.0": {"306": {"387": {"15": ["NWPS-AKMEWLS", "NWPS"]}}}},
- "53.15": {
- "183.0": {
- "82": {"136": {"15": ["NWPS-AKMEAI1", "NWPS"]}},
- "132": {"218": {"15": ["NWPS-AKMEAI2", "NWPS"]}},
- }
- },
- "58.8": {"205.6": {"311": {"317": {"15": ["NWPS-AKMEAN3", "NWPS"]}}}},
- "41.0": {
- "232.7": {
- "192": {"154": {"15": ["NWPS-ORMEMD1", "NWPS"]}},
- "77": {"62": {"15": ["NWPS-ORMEMD2", "NWPS"]}},
- "128": {"103": {"15": ["NWPS-ORMEMD3", "NWPS"]}},
- }
- },
- "32.61": {"279.99": {"49": {"75": {"15": ["NWPS-SCMECHB", "NWPS"]}}}},
- "31.88": {"278.87": {"112": {"131": {"15": ["NWPS-GAMESHB", "NWPS"]}}}},
- "39.4": {"285.4": {"112": {"69": {"15": ["NWPS-NJMELBI", "NWPS"]}}}},
- "46.1": {
- "233.0": {
- "93": {"99": {"15": ["NWPS-WAMESE1", "NWPS"]}},
- "123": {"131": {"15": ["NWPS-WAMESE2", "NWPS"]}},
- "74": {"79": {"15": ["NWPS-WAMESE3", "NWPS"]}},
- }
- },
- "38.4": {
- "233.73": {
- "106": {"64": {"15": ["NWPS-CAMEEU1", "NWPS"]}},
- "141": {"85": {"15": ["NWPS-CAMEEU2", "NWPS"]}},
- "85": {"51": {"15": ["NWPS-CAMEEU3", "NWPS"]}},
- }
- },
- "18.07": {
- "198.5": {
- "207": {"298": {"15": ["NWPS-HIME1", "NWPS"]}},
- "70": {"101": {"15": ["NWPS-HIME2", "NWPS"]}},
- }
- },
- "21.55": {"199.96": {"190": {"197": {"15": ["NWPS-HIMEKAN", "NWPS"]}}}},
- "19.4": {"204.8": {"112": {"126": {"15": ["NWPS-HIMEHIL", "NWPS"]}}}},
- "20.4": {"203.1": {"223": {"209": {"15": ["NWPS-HIMEMAU", "NWPS"]}}}},
- "21.2": {"201.65": {"136": {"156": {"15": ["NWPS-HIMEOAH", "NWPS"]}}}},
- "12.34": {
- "143.67": {
- "125": {"126": {"15": ["NWPS-GUME1", "NWPS"]}},
- "67": {"67": {"15": ["NWPS-GUME2", "NWPS"]}},
- }
- },
- "14.72": {"145.32": {"74": {"72": {"15": ["NWPS-GUMENMI", "NWPS"]}}}},
- "14.02": {"145.02": {"29": {"39": {"15": ["NWPS-GUMEROT", "NWPS"]}}}},
- "13.12": {"144.52": {"74": {"61": {"15": ["NWPS-GUMEISL", "NWPS"]}}}},
- "23.018": {"275.667": {"574": {"480": {"84": ["NAM-FLLC", "NWPS"]}}}},
- "23.097": {"240.964": {"881": {"1121": {"190": ["AWC-UNKNPS", "AWC"]}}}},
- "19.229": {
- "233.7234": {
- "1597": {
- "2345": {
- "104": ["NBM-USLCDRS3", "NBM"],
- "113": ["SREF-USLCDRS", "SREF"],
- "108": ["LMP-USLCDRS2", "LMP"],
- }
- }
- }
- },
- "65.92": {"193.82": {"143": {"155": {"15": ["NWPS-AKMEKBS", "NWPS"]}}}},
- "62.24": {"193.33": {"176": {"180": {"15": ["NWPS-AKMENTS", "NWPS"]}}}},
- "61.0": {"183.5": {"178": {"266": {"15": ["NWPS-AKMEFBK", "NWPS"]}}}},
- "59.65": {"211.23": {"189": {"193": {"15": ["NWPS-AKMEPWS", "NWPS"]}}}},
- "43.5": {
- "233.72": {
- "136": {"81": {"15": ["NWPS-ORMEPT2", "NWPS"]}},
- "82": {"49": {"15": ["NWPS-ORMEPT3", "NWPS"]}},
- "203": {"121": {"15": ["NWPS-ORMEPT1", "NWPS"]}},
- }
- },
- "46.84": {"235.74": {"38": {"25": {"15": ["NWPS-WAME4", "NWPS"]}}}},
- "43.6": {"286": {"91": {"31": {"133": ["GLSW", "GLSW"]}}}},
- "16.293825": {"233.8932": {"337": {"451": {"191": ["ICN-USLC13KM", "ICNG"]}}}},
- "14.3515": {"195.0305": {"561": {"625": {"104": ["NBM-TRUEME1", "NBM"]}}}},
- "16.828685": {
- "291.804687": {
- "129": {
- "177": {"89": ["NMMB-TRUEME1", "NMMB"], "109": ["RTM-TRUEME3", "RTMA"]}
- }
- }
- },
- "12.349884": {
- "143.686538": {
- "193": {
- "193": {"109": ["RTM-TRUEME4", "RTMA"], "104": ["NBM-TRUEME4", "NBM"]}
- }
- }
- },
- "18.072699": {
- "198.474999": {
- "225": {
- "321": {
- "109": ["RTM-TRUEME1", "RTMA"],
- "17": ["ESSP-TRUEME", "ESSP"],
- "89": ["NMMB-TRUEME2", "NMMB"],
- }
- }
- }
- },
- "-30.4192": {"129.9058": {"1817": {"2517": {"104": ["NBM-TRUEME2", "NBM"]}}}},
- "-45.0": {
- "110.0": {
- "725": {"837": {"96": ["GFS-TRUEME", "GFS"], "81": ["GFS-TRUEME", "GFS"]}}
- }
- },
- "16.9775": {"291.9722": {"225": {"339": {"104": ["NBM-TRUEME3", "NBM"]}}}},
- "30.000002": {"130.0": {"301": {"511": {"11": ["MGWM-EPME", "MGWM"]}}}},
- "46.13": {"235.8": {"56": {"39": {"15": ["NWPS-ORMECRB", "NWPS"]}}}},
- "43.15": {"235.3": {"167": {"122": {"15": ["NWPS-ORMECWB", "NWPS"]}}}},
- "89.277": {"0.0": {"190": {"384": {"98": ["CFS-UNKNGAUS", "CFS"]}}}},
- "16.977485": {"291.972167": {"225": {"339": {"14": ["ESSA-TRUEME", "ESSA"]}}}},
- "45.37": {"235.8": {"89": {"47": {"15": ["NWPS-ORMETKB", "NWPS"]}}}},
- "52.870833": {"235.270833": {"3351": {"6935": {"25": ["SNOW-UNKNME", "SNOW"]}}}},
- "26.72": {"279.86": {"156": {"80": {"15": ["NWPS-FLMEPAB", "NWPS"]}}}},
- "28.7": {
- "278.3": {
- "105": {"67": {"15": ["NWPS-FLMEJK2", "NWPS"]}},
- "204": {"131": {"15": ["NWPS-FLMEJK1", "NWPS"]}},
- }
- },
- "26.67": {"278.87": {"125": {"112": {"15": ["NWPS-FLMELKO", "NWPS"]}}}},
- "25.72": {"279.8": {"156": {"86": {"15": ["NWPS-FLMEMIB", "NWPS"]}}}},
- "42.22": {"234.9": {"181": {"140": {"15": ["NWPS-ORMECPB", "NWPS"]}}}},
- "42.0": {"235.55": {"56": {"75": {"15": ["NWPS-ORMEBGS", "NWPS"]}}}},
- "16.8287": {
- "291.8047": {
- "257": {
- "353": {
- "109": ["RTM-TRUEME2", "RTMA"],
- "89": ["NMMB-TRUEME3", "NMMB"],
- "118": ["URMA-TRUEME1", "URMA"],
- }
- }
- }
- },
- "47.0": {"261.0": {"481": {"586": {"11": ["MGWM-ECGMME", "MGWM"]}}}},
- "21.138": {
- "237.28": {
- "635": {
- "1079": {"84": ["NAM-USLCUNKN", "NAM"], "116": ["WRFE-USLCUNK", "WRFE"]}
- }
- }
- },
- "71.995": {"184.005": {"2200": {"5000": {"98": ["OAR-UNKNME1", "CFS"]}}}},
- "35.349744": {"315.0": {"1006": {"1006": {"11": ["MGWM-UNKNPS", "MGWM"]}}}},
- "25.998": {"196.001996": {"2200": {"2600": {"99": ["OAR-UNKNME2", "TEST"]}}}},
- "20.191924": {"238.445993": {"1377": {"2145": {"118": ["URMA-USLCDR1", "URMA"]}}}},
- "53.82": {"193.25": {"300": {"296": {"15": ["NWPS-AKMEUNK", "NWPS"]}}}},
- "57.8": {"200.8": {"296": {"296": {"15": ["NWPS-AKMEUN2", "NWPS"]}}}},
- "37.979669": {"234.042695": {"795": {"709": {"118": ["URMA-USLCDR2", "URMA"]}}}},
-}
-
-
-def xcd_lookup(
- first_lat: float, first_lon: float, rows: int, cols: int, gen_proc_id: int
-) -> tuple[str, str]:
- """Looks up xcd model names and ids based on grib message properties.
-
- Args:
- first_lat (float): The first latitude of the grib message.
- first_lon (float): The first longitude of the grib message.
- rows (int): The number of rows in the grib message.
- cols (int): The number of columns in the grib message.
- gen_proc_id (int): The generating process id of the grib message.
-
- Returns:
- tuple[str, str]: (xcd model name, xcd model id) or ("UNKWN", "UNKWN")
- if can't find model info based on given properties.
- """
- try:
- return tuple(
- XCD_MODELS[str(first_lat)][str(first_lon)][str(rows)][str(cols)][
- str(gen_proc_id)
- ]
- )
- except KeyError:
- return ("UNKWN", "UNKWN")
-
-
-# argparse types
-def non_negative_int(_arg: str) -> int:
- i = int(_arg)
- if i < 0:
- raise ValueError
- return i
-
-
-def positive_int(_arg: str) -> int:
- i = int(_arg)
- if i <= 0:
- raise ValueError
- return i
-
-
-def setup() -> argparse.Namespace:
- """Initialize the script so it's ready to run."""
- # Set up cmdline args
- parser = argparse.ArgumentParser(
- description="Set paramters for ingesting and publishing of grib data."
- )
- parser.add_argument(
- "grib_dir",
- help="Directory to watch for new grib files, use -- to read file paths from stdin.",
- )
- parser.add_argument(
- "-v",
- "--verbosity",
- type=non_negative_int,
- default=3,
- help="Specify verbosity to stderr from 0 (CRITICAL) to 5 (DEBUG).",
- )
- parser.add_argument(
- "--watch-debounce",
- type=non_negative_int,
- default=5000,
- help="maximum time in milliseconds to group grib file changes over before processing them.",
- )
- parser.add_argument(
- "--log-dir", default=LOG_DIR, help="Directory to put rotating file logs into."
- )
- parser.add_argument(
- "--testing",
- action="store_true",
- help='Prepends route key with "null.".',
- )
-
- ch_group = parser.add_argument_group(
- "cache arguments",
- "Specify how the cache will behave (the defaults work best for standard usage).",
- )
- ch_group.add_argument(
- "--no-cache",
- action="store_true",
- help="Turn the cache off. Could be useful if watched files are only written to once.",
- )
- ch_group.add_argument(
- "--clean-interval",
- metavar="SECONDS",
- type=positive_int,
- default=CACHE_CLEAN_INTERVAL,
- help="How often to clean the message cache.",
- )
- ch_group.add_argument(
- "--cache-ttl",
- metavar="SECONDS",
- type=positive_int,
- default=CACHE_TTL,
- help="How long paths stay in the cache for.",
- )
-
- mq_group = parser.add_argument_group(
- "RabbitMQ arguments", "arguments to specify how to publish the data."
- )
- mq_group.add_argument(
- "--servers",
- nargs="+",
- default=DEFAULT_AMQP_SERVERS,
- help="Servers to publish AMQP messages to. Defaults to %(default)s",
- )
- mq_group.add_argument(
- "-X",
- "--exchange",
- default=DEFAULT_AMQP_EXCHANGE,
- help='RabbitMQ exchange to publish the messages to. Defaults to "%(default)s"',
- )
- mq_group.add_argument(
- "-u",
- "--user",
- default=None,
- help="RabbitMQ user to login with. Can also specify using a .env file, see README#configuration.",
- )
- mq_group.add_argument(
- "-p",
- "--password",
- default=None,
- help="RabbitMQ password to login with. Can also specify using a .env file, see README#configuration.",
- )
-
- args = parser.parse_args()
-
- # Verify parsed arguments
- if not os.path.isdir(args.grib_dir):
- if args.grib_dir == "--":
- raise NotImplementedError(
- "Reading from stdin is not yet implemented ¯\_(ツ)_/¯"
- )
- else:
- parser.error("{0} is not a valid directory!".format(args.grib_dir))
-
- if args.no_cache:
- raise NotImplementedError(
- "Turning the cache off is not yet implemented ¯\_(ツ)_/¯"
- )
-
- if not os.path.isdir(args.log_dir):
- os.mkdir(args.log_dir)
-
- if args.testing:
- global ROUTE_KEY_FMT
- ROUTE_KEY_FMT = "null.{xcd_model}.{xcd_model_id}.realtime.reserved.2"
-
- # Set up logging
- log_formatter = logging.Formatter(
- fmt="[%(asctime)s][%(levelname)s]-%(message)s", datefmt="%Y-%m-%dT%H:%M:%S"
- )
-
- file_handler = logging.handlers.TimedRotatingFileHandler(
- os.path.join(args.log_dir, LOG_NAME), when="D", utc=True
- )
- file_handler.setFormatter(log_formatter)
- file_handler.setLevel(logging.DEBUG)
-
- out_handler = logging.StreamHandler()
- out_handler.setFormatter(log_formatter)
- out_handler.setLevel(LOG_LEVELS[min(args.verbosity, len(LOG_LEVELS) - 1)])
-
- LOG.addHandler(file_handler)
- LOG.addHandler(out_handler)
- LOG.setLevel(logging.DEBUG)
-
- return args
-
-
-def _cache_entry() -> tuple[float, int]:
- """default cache entry (current time, first grib message)."""
- return time.time(), 0
-
-
-def _cache_cleaner(
- cache: DefaultDict[str, tuple[float, int]], interval: float, ttl: float
-) -> None:
- """Called from another thread, continually cleans cache in order to
- prevent memory leaks.
- """
- while True:
- to_remove = []
- LOG.debug("Cleaning the message cache, size=%d!", len(cache))
- for path, (last_update_time, last_msg_num) in cache.items():
- time_in_cache = time.time() - last_update_time
- if time_in_cache > ttl:
- LOG.info(
- "Evicted %s from cache! (last message: %d, time since last update: %f).",
- path,
- last_msg_num - 1,
- time_in_cache,
- )
- to_remove.append(path)
- if not to_remove:
- LOG.debug("Nothing to remove from cache...")
- else:
- for rem_path in to_remove:
- cache.pop(rem_path, None)
- # LOG.debug('Cleaning grib2io caches')
- # grib2io._grib2io._msg_class_store.clear()
- # grib2io._grib2io._latlon_datastore.clear()
- time.sleep(interval)
-
-
-def watch_filter(change: Change, path: str) -> bool:
- """Make sure files we ingest are grib files."""
- if change == Change.deleted:
- return False
- if os.path.splitext(path)[1] in GRIB_ENDINGS:
- return True
- return False
-
-
-def amqp_grib(grib_path: str, start_message: int) -> Generator[dict, None, None]:
- """Generate AMQP payloads from a grib file, starting at a specific message.
-
- Args:
- grib_path (str): The path to the grib file to create payloads for.
- start_message (int): Which grib message to start yielding payloads for.
-
- Yields:
- JSON-able: The json formatted AMQP payload for each grib message from start_message to end of the file.
- """
- with grib2io.open(grib_path) as grib_file:
- for msg in grib_file[start_message:]:
- f_lat = getattr(msg, "latitudeFirstGridpoint", None)
- f_lon = getattr(msg, "longitudeFirstGridpoint", None)
- rows = msg.ny
- cols = msg.nx
- gen_proc = msg.generatingProcess
- xcd_info = xcd_lookup(f_lat, f_lon, rows, cols, gen_proc.value)
- yield {
- "__payload_gen_time__": amqp_utils.format_datetime(datetime.now()),
- "__injector_script__": amqp_utils.INJECTOR_SCRIPT,
- "path": grib_path,
- "directory": os.path.dirname(grib_path),
- "file_name": os.path.basename(grib_path),
- "server_ip": amqp_utils.SERVER_NAME,
- "server_type": "realtime",
- "first_lat": f_lat,
- "last_lat": getattr(msg, "latitudeLastGridpoint", None),
- "first_lon": f_lon,
- "last_long": getattr(msg, "longitudeLastGridpoint", None),
- "forecast_hour": int(msg.valueOfForecastTime),
- "run_hour": int(msg.hour) * 100 + int(msg.minute),
- "model_time": amqp_utils.format_datetime(msg.refDate),
- "start_time": amqp_utils.format_datetime(msg.refDate),
- "projection": msg.gridDefinitionTemplateNumber.definition,
- "center_id": int(msg.originatingCenter.value),
- "center_desc": msg.originatingCenter.definition,
- "level": msg.level,
- "parameter": msg.shortName,
- "param_long": msg.fullName,
- "param_units": msg.units,
- "grib_number": 2,
- "size": "%d x %d" % (rows, cols),
- "ensemble": getattr(msg, "pertubationNumber", None),
- "model_id": int(gen_proc.value),
- "model": gen_proc.definition,
- "xcd_model": xcd_info[1],
- "xcd_model_id": xcd_info[0],
- # "resolution": msg._xydivisor, THIS IS NOT CORRECT!
- "grib_record_number": msg._msgnum,
- "title": str(msg),
- }
-
-
-def gribs_from_dir(
- watch_dir: str, cache: DefaultDict[str, tuple[float, int]], watch_debounce: int
-) -> None:
- """Process and publish grib files by watching a directory.
-
- Args:
- watch_dir (str): The directory to watch for grib files.
- cache (DefaultDict[str, tuple[int, float]]): The cache to use to store grib messages in (if messages arrive in chunks).
- watch_debounce (int): debounce for watch.
- """
- LOG.info("Watching %s for gribs with debounce=%d", watch_dir, watch_debounce)
- for changes in watch(
- watch_dir,
- watch_filter=watch_filter,
- debounce=watch_debounce,
- ):
- for _, path in changes:
- next_msg = cache[path][1]
- LOG.debug("Got grib file %s (next msg to process: %d).", path, next_msg)
- msg_num = 0 # declare here to avoid unbound errors.
- try:
- for msg_num, amqp in enumerate(amqp_grib(path, next_msg)):
- route_key = ROUTE_KEY_FMT.format(
- xcd_model=amqp["xcd_model"], xcd_model_id=amqp["xcd_model_id"]
- )
- LOG.debug(
- "Publishing %s msg %d to %s",
- path,
- next_msg + msg_num,
- route_key,
- )
- mq.publish(amqp, route_key=route_key)
- except (ValueError, KeyError, OSError):
- LOG.exception("Error parsing grib file %s!", path)
- else:
- cache[path] = time.time(), next_msg + msg_num + 1
-
-
-def main() -> None:
- """Main method, setup and start processing."""
- try:
- args = setup()
-
- if not load_dotenv():
- LOG.warning("Couldn't find .env file")
-
- mq.connect(
- *args.servers,
- user=args.user or os.getenv("RMQ_USER", DEFAULT_AMQP_USER),
- password=args.password or os.getenv("RMQ_PASS", DEFAULT_AMQP_PASS),
- exchange=args.exchange,
- )
- LOG.info("Connected to %s", ", ".join(args.servers))
-
- # maybe add loading/storing cache to disk after exits?
- cache: DefaultDict[str, tuple[float, int]] = defaultdict(_cache_entry)
- LOG.debug("Starting cache cleaner")
- threading.Thread(
- name="grib_pipeline_cleaner",
- target=_cache_cleaner,
- args=(cache, args.clean_interval, args.cache_ttl),
- daemon=True,
- ).start()
- except KeyboardInterrupt:
- mq.disconnect()
- return
-
- try:
- gribs_from_dir(args.grib_dir, cache, args.watch_debounce)
- except KeyboardInterrupt:
- LOG.critical("Got interrupt, goodbye!")
- mq.disconnect()
- return
-
-
-if __name__ == "__main__":
- sys.exit(main())
diff --git a/grib_processor/__init__.py b/grib_processor/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..75b931eae9163ea5a49fe6a9c6899f10361c5d46
--- /dev/null
+++ b/grib_processor/__init__.py
@@ -0,0 +1,23 @@
+"""
+grib_processor
+~~~~~~~~~~~~~~
+
+Ingest grib files and publish metadata for files to RabbitMQ.
+"""
+
+from grib_processor.grib import GribPayload, itergrib
+from grib_processor.main import dump_message, publish_message
+from grib_processor.utils import grib_file_watch
+
+
+__author__ = "Max Drexler"
+__email__ = "mndrexler@wisc.edu"
+__version__ = "0.0.1"
+
+__all__ = [
+ "GribPayload",
+ "itergrib",
+ "dump_message",
+ "publish_message",
+ "grib_file_watch",
+]
diff --git a/grib_processor/__main__.py b/grib_processor/__main__.py
new file mode 100644
index 0000000000000000000000000000000000000000..117c77974b4829028437db09381a9b42f2622902
--- /dev/null
+++ b/grib_processor/__main__.py
@@ -0,0 +1,14 @@
+"""
+grib_processor.__main__
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Entrypoint to grib processing when using python -m grib_processor.
+"""
+
+import sys
+
+from grib_processor.main import main
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/grib_processor/data/__init__.py b/grib_processor/data/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/grib_processor/data/xcd_model_info.json b/grib_processor/data/xcd_model_info.json
new file mode 100644
index 0000000000000000000000000000000000000000..060f44c851fb9d1056b2f0b3bbabe60c44670be1
--- /dev/null
+++ b/grib_processor/data/xcd_model_info.json
@@ -0,0 +1,3242 @@
+{
+ "44.196": {
+ "174.759": {
+ "237": {
+ "377": {
+ "115": [
+ "DGEX-AKPS",
+ "DGEX"
+ ]
+ }
+ }
+ }
+ },
+ "19.943": {
+ "234.907": {
+ "303": {
+ "491": {
+ "115": [
+ "DGEX-USLC",
+ "DGEX"
+ ]
+ }
+ }
+ }
+ },
+ "0.0": {
+ "0.0": {
+ "91": {
+ "360": {
+ "96": [
+ "GFS-NHME",
+ "GFS"
+ ]
+ }
+ },
+ "181": {
+ "720": {
+ "30": [
+ "FGF-NHME",
+ "FGF"
+ ]
+ }
+ }
+ },
+ "180.0": {
+ "500": {
+ "625": {
+ "85": [
+ "RTOF-HIME",
+ "RTOF"
+ ]
+ }
+ }
+ },
+ "130.0": {
+ "375": {
+ "625": {
+ "85": [
+ "RTOF-WPME",
+ "RTOF"
+ ]
+ }
+ }
+ }
+ },
+ "90.0": {
+ "0.0": {
+ "73": {
+ "144": {
+ "81": [
+ "GFS-GLME2P5D",
+ "GFS"
+ ],
+ "96": [
+ "GFS-GLME2P5D",
+ "GFS"
+ ]
+ }
+ },
+ "145": {
+ "288": {
+ "81": [
+ "GFS-GLME1P25",
+ "GFS"
+ ],
+ "96": [
+ "GFS-GLME1P25",
+ "GFS"
+ ]
+ }
+ },
+ "181": {
+ "360": {
+ "81": [
+ "GFS-GLME1P0D",
+ "GFS"
+ ],
+ "96": [
+ "GFS-GLME1P0D",
+ "GFS"
+ ],
+ "107": [
+ "GEFS-GLME",
+ "GEFS"
+ ]
+ }
+ },
+ "336": {
+ "720": {
+ "11": [
+ "MGWM-GLME",
+ "MGWM"
+ ]
+ }
+ },
+ "361": {
+ "720": {
+ "81": [
+ "GFS-GLME0P5D",
+ "GFS"
+ ],
+ "96": [
+ "GFS-GLME0P5D",
+ "GFS"
+ ]
+ }
+ },
+ "721": {
+ "1440": {
+ "81": [
+ "GFS-GLMEP25D",
+ "GFS"
+ ],
+ "96": [
+ "GFS-GLMEP25D",
+ "GFS"
+ ]
+ }
+ }
+ }
+ },
+ "40.5301": {
+ "181.429": {
+ "1105": {
+ "1649": {
+ "0": [
+ "MOS-AKPSDRES",
+ "MOS"
+ ],
+ "81": [
+ "GFS-AKPSDRES",
+ "GFS"
+ ],
+ "96": [
+ "GFS-AKPSDRES",
+ "GFS"
+ ],
+ "114": [
+ "NAEF-AKPSDRS",
+ "NAEF"
+ ]
+ }
+ }
+ }
+ },
+ "19.132": {
+ "174.163": {
+ "156": {
+ "180": {
+ "81": [
+ "GFS-AKPSHRES",
+ "GFS"
+ ],
+ "96": [
+ "GFS-AKPSHRES",
+ "GFS"
+ ]
+ }
+ }
+ }
+ },
+ "-20.826": {
+ "210.0": {
+ "65": {
+ "65": {
+ "81": [
+ "GFS-NHPS",
+ "GFS"
+ ],
+ "96": [
+ "GFS-NHPS",
+ "GFS"
+ ]
+ }
+ }
+ }
+ },
+ "12.19": {
+ "226.541": {
+ "129": {
+ "185": {
+ "81": [
+ "GFS-USLC2",
+ "GFS"
+ ],
+ "84": [
+ "NAM-USLC2",
+ "NAM"
+ ],
+ "89": [
+ "NMMB-USLCA40",
+ "NMMB"
+ ],
+ "96": [
+ "GFS-USLC2",
+ "GFS"
+ ],
+ "111": [
+ "NAM-USLC2",
+ "NAM"
+ ],
+ "112": [
+ "WRFN-USLC40K",
+ "WRFN"
+ ],
+ "113": [
+ "SREF-USLC2",
+ "SREF"
+ ],
+ "116": [
+ "WRFE-USLC40K",
+ "WRFE"
+ ]
+ }
+ },
+ "257": {
+ "369": {
+ "81": [
+ "GFS-USLCAW20",
+ "GFS"
+ ],
+ "96": [
+ "GFS-USLCAW20",
+ "GFS"
+ ]
+ }
+ },
+ "428": {
+ "614": {
+ "84": [
+ "NAM-USLCAW12",
+ "NAM"
+ ]
+ }
+ },
+ "1025": {
+ "1473": {
+ "132": [
+ "HREF-USLCEST",
+ "HREF"
+ ]
+ }
+ }
+ }
+ },
+ "20.192": {
+ "238.446": {
+ "689": {
+ "1073": {
+ "96": [
+ "GFS-USLCAWI4",
+ "GFS"
+ ],
+ "108": [
+ "LMP-USLCAWI4",
+ "LMP"
+ ]
+ }
+ },
+ "1377": {
+ "2145": {
+ "0": [
+ "MOS-USLCDRES",
+ "MOS"
+ ],
+ "89": [
+ "NMMB-USLCDRS",
+ "NMMB"
+ ],
+ "96": [
+ "GFS-USLCDRS1",
+ "GFS"
+ ],
+ "108": [
+ "LMP-USLCDRES",
+ "LMP"
+ ],
+ "183": [
+ "NDF-USLCDRES",
+ "NDF"
+ ]
+ }
+ },
+ "1597": {
+ "2145": {
+ "96": [
+ "GFS-USLCDRS2",
+ "GFS"
+ ],
+ "104": [
+ "NBM-USLCDRS2",
+ "NBM"
+ ]
+ }
+ }
+ },
+ "238.45": {
+ "689": {
+ "1073": {
+ "89": [
+ "NMMB-USLCAW4",
+ "NMMB"
+ ]
+ }
+ }
+ }
+ },
+ "7.838": {
+ "218.972": {
+ "85": {
+ "129": {
+ "81": [
+ "GFS-USPS",
+ "GFS"
+ ],
+ "96": [
+ "GFS-USPS",
+ "GFS"
+ ]
+ }
+ }
+ }
+ },
+ "49.1": {
+ "267.799984": {
+ "235": {
+ "327": {
+ "131": [
+ "GLWM",
+ "GLWM"
+ ]
+ }
+ }
+ }
+ },
+ "89.958": {
+ "0.042": {
+ "2160": {
+ "4320": {
+ "44": [
+ "SST-GLME",
+ "SST"
+ ],
+ "120": [
+ "ICA-GLME",
+ "ICA"
+ ]
+ }
+ }
+ }
+ },
+ "16.281": {
+ "233.862": {
+ "113": {
+ "151": {
+ "105": [
+ "RAP-USLC40KM",
+ "RAP"
+ ]
+ }
+ },
+ "225": {
+ "301": {
+ "105": [
+ "RAP-USLC20KM",
+ "RAP"
+ ]
+ }
+ },
+ "337": {
+ "451": {
+ "105": [
+ "RAP-USLC13KM",
+ "RAP"
+ ],
+ "191": [
+ "ICN-USLC13KM",
+ "ICNG"
+ ],
+ "193": [
+ "ICI-USLC13KM",
+ "ICI"
+ ]
+ }
+ }
+ },
+ "233.861999": {
+ "337": {
+ "451": {
+ "105": [
+ "RAP-USLC13KM",
+ "RAP"
+ ]
+ }
+ }
+ }
+ },
+ "74.0": {
+ "165.0": {
+ "391": {
+ "548": {
+ "11": [
+ "MGWM-AKME",
+ "MGWM"
+ ]
+ }
+ }
+ }
+ },
+ "30.0": {
+ "130.0": {
+ "301": {
+ "511": {
+ "11": [
+ "MGWM-EPME",
+ "MGWM"
+ ]
+ }
+ }
+ },
+ "187.0": {
+ "107": {
+ "139": {
+ "84": [
+ "NAM-AKPS45KM",
+ "NAM"
+ ],
+ "113": [
+ "SREF-AKPS",
+ "SREF"
+ ]
+ }
+ },
+ "213": {
+ "277": {
+ "81": [
+ "GFS-AKPSAWI2",
+ "GFS"
+ ],
+ "96": [
+ "GFS-AKPSAWI2",
+ "GFS"
+ ]
+ }
+ },
+ "425": {
+ "553": {
+ "84": [
+ "NAM-AKPSAWI4",
+ "NAM"
+ ]
+ }
+ }
+ }
+ },
+ "75.0": {
+ "140.0": {
+ "187": {
+ "401": {
+ "11": [
+ "MGWM-NPME",
+ "MGWM"
+ ]
+ }
+ }
+ }
+ },
+ "55.0": {
+ "260.0": {
+ "331": {
+ "301": {
+ "11": [
+ "MGWM-WAME",
+ "MGWM"
+ ]
+ }
+ }
+ },
+ "202.0": {
+ "62": {
+ "81": {
+ "15": [
+ "NWPS-AKMEAN1",
+ "NWPS"
+ ]
+ }
+ },
+ "147": {
+ "193": {
+ "15": [
+ "NWPS-AKMEAN2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "50.0": {
+ "195.0": {
+ "526": {
+ "736": {
+ "11": [
+ "MGWM-WCHIME",
+ "MGWM"
+ ]
+ }
+ }
+ },
+ "210.0": {
+ "151": {
+ "241": {
+ "11": [
+ "MGWM-WCME",
+ "MGWM"
+ ]
+ }
+ }
+ }
+ },
+ "1.0": {
+ "214.5": {
+ "277": {
+ "349": {
+ "111": [
+ "NAM-USLC32KM",
+ "NAM"
+ ],
+ "112": [
+ "WRFN-USLC32K",
+ "WRFN"
+ ],
+ "116": [
+ "WRFE-USLC32K",
+ "WRFE"
+ ]
+ }
+ }
+ }
+ },
+ "20.190001": {
+ "238.449996": {
+ "689": {
+ "1073": {
+ "180": [
+ "NCE-USLCAWI4",
+ "NCE"
+ ],
+ "221": [
+ "WPC-USLCAWI4",
+ "WPC"
+ ],
+ "224": [
+ "WPC-USLCAWI4",
+ "WPC"
+ ]
+ }
+ }
+ }
+ },
+ "40.530094": {
+ "181.429031": {
+ "553": {
+ "825": {
+ "183": [
+ "NDF-AKPS2",
+ "NDF"
+ ]
+ }
+ }
+ }
+ },
+ "40.530096": {
+ "181.429024": {
+ "553": {
+ "825": {
+ "183": [
+ "NDF-AKPS2",
+ "NDF"
+ ]
+ }
+ }
+ }
+ },
+ "40.529998": {
+ "181.429993": {
+ "553": {
+ "825": {
+ "183": [
+ "NDF-AKPS",
+ "NDF"
+ ]
+ }
+ }
+ }
+ },
+ "40.530101": {
+ "181.429": {
+ "553": {
+ "825": {
+ "12": [
+ "PSS-AKPS",
+ "PSS"
+ ],
+ "17": [
+ "ESSP-AKPS",
+ "ESSP"
+ ],
+ "109": [
+ "RTM-AKPS",
+ "RTMA"
+ ]
+ }
+ },
+ "1105": {
+ "1649": {
+ "0": [
+ "NDF-AKPSDRS",
+ "NDF"
+ ],
+ "16": [
+ "ETSS-AKPSDRS",
+ "ETSS"
+ ],
+ "18": [
+ "PETS-AKPSDRS",
+ "PETS"
+ ],
+ "89": [
+ "NMMB-AKPSDRS",
+ "NMMB"
+ ],
+ "109": [
+ "RTM-AKPSDRES",
+ "RTMA"
+ ]
+ }
+ }
+ }
+ },
+ "20.191999": {
+ "238.445999": {
+ "689": {
+ "1073": {
+ "12": [
+ "PSS-USLCAWI4",
+ "PSS"
+ ],
+ "109": [
+ "RTM-USLCAWI4",
+ "RTMA"
+ ]
+ }
+ },
+ "1377": {
+ "2145": {
+ "0": [
+ "NDF-USLCDRES",
+ "NDFD"
+ ],
+ "12": [
+ "PSS-USLCDRES",
+ "PSS"
+ ],
+ "14": [
+ "ESSA-USLCDRS",
+ "ESSA"
+ ],
+ "16": [
+ "ETSS-USLCDRS",
+ "ETSS"
+ ],
+ "17": [
+ "ESSP-USLCDRS",
+ "ESSP"
+ ],
+ "18": [
+ "PETS-USLCDRS",
+ "PETS"
+ ],
+ "83": [
+ "HRR-USLCDRES",
+ "HRR"
+ ],
+ "96": [
+ "GFS-USLCDRS1",
+ "GFS"
+ ],
+ "109": [
+ "RTM-USLCDRES",
+ "RTMA"
+ ],
+ "114": [
+ "NAEF-USLCDRS",
+ "NAEF"
+ ],
+ "118": [
+ "URMA-USLCDR1",
+ "URMA"
+ ],
+ "183": [
+ "NDF-USLCDRES",
+ "NDF"
+ ],
+ "222": [
+ "WPC-USLCDRES",
+ "WPC"
+ ],
+ "223": [
+ "WPC-USLCDRES",
+ "WPC"
+ ]
+ }
+ },
+ "1597": {
+ "2145": {
+ "180": [
+ "NCE-USLCDRS",
+ "NCEP"
+ ],
+ "221": [
+ "WPC-USLCPD10",
+ "WPC"
+ ]
+ }
+ },
+ "5505": {
+ "8577": {
+ "16": [
+ "ETSS-USLC",
+ "ETSS"
+ ],
+ "18": [
+ "PETS-PD10LC",
+ "PETS"
+ ]
+ }
+ }
+ }
+ },
+ "40.53": {
+ "181.429": {
+ "553": {
+ "825": {
+ "89": [
+ "NMMB-AKPS",
+ "NMMB"
+ ]
+ }
+ },
+ "1105": {
+ "1649": {
+ "0": [
+ "MOS-AKPSDRES",
+ "MOS"
+ ],
+ "89": [
+ "NMMB-AKPSDRS",
+ "NMMB"
+ ],
+ "104": [
+ "NBM-AKPSDRS",
+ "NBM"
+ ],
+ "118": [
+ "URMA-AKPSDRS",
+ "URMA"
+ ]
+ }
+ }
+ }
+ },
+ "-40.0": {
+ "130.0": {
+ "80": {
+ "120": {
+ "85": [
+ "RTOF-APME",
+ "RTOF"
+ ]
+ }
+ }
+ }
+ },
+ "40.0": {
+ "140.0": {
+ "150": {
+ "350": {
+ "85": [
+ "RTOF-ARCTME",
+ "RTOF"
+ ]
+ }
+ }
+ },
+ "251.0": {
+ "152": {
+ "328": {
+ "85": [
+ "RTOF-NATLME",
+ "RTOF"
+ ]
+ }
+ }
+ },
+ "195.0": {
+ "45": {
+ "84": {
+ "85": [
+ "RTOF-NEPME",
+ "RTOF"
+ ]
+ }
+ }
+ },
+ "155.0": {
+ "340": {
+ "700": {
+ "85": [
+ "RTOF-NPME",
+ "RTOF"
+ ]
+ }
+ }
+ }
+ },
+ "-30.0": {
+ "170.0": {
+ "375": {
+ "560": {
+ "85": [
+ "RTOF-SCPME",
+ "RTOF"
+ ]
+ }
+ }
+ }
+ },
+ "10.0": {
+ "260.0": {
+ "435": {
+ "575": {
+ "85": [
+ "RTOF-USECME",
+ "RTOF"
+ ]
+ }
+ }
+ },
+ "210.0": {
+ "625": {
+ "625": {
+ "85": [
+ "RTOF-USWCME",
+ "RTOF"
+ ]
+ }
+ }
+ },
+ "190.0": {
+ "101": {
+ "126": {
+ "113": [
+ "SREF-USME",
+ "SREF"
+ ]
+ }
+ }
+ }
+ },
+ "37.979684": {
+ "234.042704": {
+ "795": {
+ "709": {
+ "118": [
+ "URMA-USLCDR2",
+ "URMA"
+ ]
+ }
+ }
+ }
+ },
+ "44.8": {
+ "185.5": {
+ "603": {
+ "825": {
+ "84": [
+ "NAM-AKPS",
+ "NAM"
+ ],
+ "112": [
+ "WRFN-AKPS",
+ "WRFN"
+ ],
+ "116": [
+ "WRFE-AKPS",
+ "WRFE"
+ ],
+ "132": [
+ "HREF-AKPS",
+ "HREF"
+ ]
+ }
+ }
+ }
+ },
+ "16.4": {
+ "197.65": {
+ "170": {
+ "223": {
+ "84": [
+ "NAM-HIME",
+ "NAM"
+ ],
+ "112": [
+ "WRFN-HIME",
+ "WRFN"
+ ],
+ "116": [
+ "WRFE-HIME",
+ "WRFE"
+ ],
+ "132": [
+ "HREF-HIME",
+ "HREF"
+ ]
+ }
+ }
+ }
+ },
+ "11.7": {
+ "141.0": {
+ "170": {
+ "223": {
+ "84": [
+ "NAM-WPME",
+ "NAM"
+ ],
+ "112": [
+ "WRFN-WPME",
+ "WRFN"
+ ],
+ "116": [
+ "WRFE-WPME",
+ "WRFE"
+ ]
+ }
+ }
+ }
+ },
+ "13.5": {
+ "283.41": {
+ "208": {
+ "340": {
+ "84": [
+ "NAM-PRME",
+ "NAM"
+ ],
+ "112": [
+ "WRFN-PRME",
+ "WRFN"
+ ],
+ "116": [
+ "WRFE-PRME",
+ "WRFE"
+ ],
+ "132": [
+ "HREF-PRME",
+ "HREF"
+ ]
+ }
+ }
+ }
+ },
+ "22.1": {
+ "250.2": {
+ "614": {
+ "884": {
+ "112": [
+ "WRFN-USLCSE",
+ "WRFN"
+ ],
+ "116": [
+ "WRFE-USLCSE",
+ "WRFE"
+ ]
+ }
+ }
+ }
+ },
+ "24.5": {
+ "230.8": {
+ "614": {
+ "884": {
+ "112": [
+ "WRFN-USLCSW",
+ "WRFN"
+ ],
+ "116": [
+ "WRFE-USLCSW",
+ "WRFE"
+ ]
+ }
+ }
+ }
+ },
+ "50.75": {
+ "271.75": {
+ "102": {
+ "137": {
+ "81": [
+ "GFS-WAME",
+ "GFS"
+ ],
+ "96": [
+ "GFS-WAME",
+ "GFS"
+ ]
+ }
+ },
+ "103": {
+ "137": {
+ "96": [
+ "GFS-PRMEP5D",
+ "GFS"
+ ]
+ }
+ },
+ "205": {
+ "275": {
+ "81": [
+ "GFS-PRMEP25D",
+ "GFS"
+ ],
+ "96": [
+ "GFS-PRMEP25D",
+ "GFS"
+ ]
+ }
+ }
+ }
+ },
+ "-0.268": {
+ "220.525": {
+ "110": {
+ "147": {
+ "84": [
+ "NAM-USPS",
+ "NAM"
+ ]
+ }
+ }
+ }
+ },
+ "41.530708": {
+ "267.364016": {
+ "361": {
+ "581": {
+ "131": [
+ "GLW-USLC",
+ "GLWM"
+ ],
+ "133": [
+ "GLSW-USLC",
+ "GLSW"
+ ]
+ }
+ }
+ }
+ },
+ "49.099998": {
+ "267.799988": {
+ "235": {
+ "327": {
+ "131": [
+ "GLW-MWUSME",
+ "GLWM"
+ ]
+ }
+ }
+ }
+ },
+ "60.0": {
+ "160.0": {
+ "250": {
+ "950": {
+ "85": [
+ "RTOF-SARCME",
+ "RTOF"
+ ]
+ }
+ }
+ }
+ },
+ "35.0": {
+ "170.0": {
+ "225": {
+ "277": {
+ "81": [
+ "GFS-NPPS",
+ "GFS"
+ ],
+ "96": [
+ "GFS-NPPS",
+ "GFS"
+ ]
+ }
+ }
+ },
+ "234.2": {
+ "98": {
+ "93": {
+ "15": [
+ "NWPS-CAMEMR3",
+ "NWPS"
+ ]
+ }
+ },
+ "109": {
+ "103": {
+ "15": [
+ "NWPS-CAMEMR2",
+ "NWPS"
+ ]
+ }
+ },
+ "123": {
+ "116": {
+ "15": [
+ "NWPS-CAMEMR1",
+ "NWPS"
+ ]
+ }
+ },
+ "140": {
+ "132": {
+ "15": [
+ "NWPS-CAMEMR5",
+ "NWPS"
+ ]
+ }
+ },
+ "196": {
+ "185": {
+ "15": [
+ "NWPS-CAMEMR4",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "54.995": {
+ "230.005": {
+ "3500": {
+ "7000": {
+ "97": [
+ "MRMS-USME1",
+ "OAR"
+ ]
+ }
+ }
+ },
+ "230.005992": {
+ "3500": {
+ "7000": {
+ "97": [
+ "MRMS-USME4",
+ "OAR"
+ ]
+ }
+ }
+ },
+ "230.005004": {
+ "3500": {
+ "7000": {
+ "97": [
+ "MRMS-USME1",
+ "OAR"
+ ]
+ }
+ }
+ }
+ },
+ "54.9975": {
+ "230.0025": {
+ "7000": {
+ "14000": {
+ "97": [
+ "MRMS-USME2",
+ "OAR"
+ ]
+ }
+ }
+ }
+ },
+ "54.95": {
+ "230.05": {
+ "350": {
+ "700": {
+ "97": [
+ "MRMS-USME3",
+ "OAR"
+ ]
+ }
+ }
+ }
+ },
+ "35.8": {
+ "282.7": {
+ "138": {
+ "91": {
+ "15": [
+ "NWPS-DMVAME4",
+ "NWPS"
+ ]
+ },
+ "92": {
+ "15": [
+ "NWPS-VAMEWF2",
+ "NWPS"
+ ]
+ }
+ },
+ "229": {
+ "153": {
+ "15": [
+ "NWPS-VAMEWF1",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "25.45": {
+ "275.2": {
+ "119": {
+ "83": {
+ "15": [
+ "NWPS-FLMETB3",
+ "NWPS"
+ ]
+ }
+ },
+ "263": {
+ "185": {
+ "15": [
+ "NWPS-FLMETB2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "24.1": {
+ "276.46": {
+ "134": {
+ "174": {
+ "15": [
+ "NWPS-FLMESF1",
+ "NWPS"
+ ]
+ }
+ },
+ "201": {
+ "261": {
+ "15": [
+ "NWPS-FLMESF2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "33.85": {
+ "282.0": {
+ "62": {
+ "61": {
+ "15": [
+ "NWPS-NCMEMC2",
+ "NWPS"
+ ]
+ }
+ },
+ "170": {
+ "167": {
+ "15": [
+ "NWPS-NCMEMC1",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "36.0": {
+ "284.32": {
+ "112": {
+ "90": {
+ "15": [
+ "NWPS-NCMEKDH",
+ "NWPS"
+ ]
+ }
+ }
+ },
+ "284.2": {
+ "67": {
+ "54": {
+ "15": [
+ "NWPS-NCMENOB",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "39.75": {
+ "285.762": {
+ "62": {
+ "91": {
+ "15": [
+ "NWPS-NYMENY1",
+ "NWPS"
+ ]
+ }
+ }
+ },
+ "285.55": {
+ "62": {
+ "97": {
+ "15": [
+ "NWPS-NYMENY1",
+ "NWPS"
+ ]
+ }
+ },
+ "102": {
+ "162": {
+ "15": [
+ "NWPS-NYMENY2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "40.83": {
+ "287.25": {
+ "61": {
+ "90": {
+ "15": [
+ "NWPS-NYMELI1",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "40.4": {
+ "285.7": {
+ "37": {
+ "41": {
+ "15": [
+ "NWPS-LIME4",
+ "NWPS"
+ ]
+ }
+ },
+ "74": {
+ "82": {
+ "15": [
+ "NWPS-DEMEDB2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "40.54": {
+ "286.22": {
+ "29": {
+ "85": {
+ "15": [
+ "NWPS-NYMELI2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "27.41": {
+ "264.97": {
+ "74": {
+ "114": {
+ "15": [
+ "NWPS-LATXME1",
+ "NWPS"
+ ]
+ }
+ },
+ "81": {
+ "114": {
+ "15": [
+ "NWPS-LAMELC1",
+ "NWPS"
+ ]
+ }
+ },
+ "179": {
+ "253": {
+ "15": [
+ "NWPS-LAMELC2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "25.26": {
+ "262.359": {
+ "84": {
+ "57": {
+ "15": [
+ "NWPS-TXME3",
+ "NWPS"
+ ]
+ }
+ }
+ },
+ "262.0": {
+ "84": {
+ "66": {
+ "15": [
+ "NWPS-TXMEBR1",
+ "NWPS"
+ ]
+ }
+ },
+ "186": {
+ "145": {
+ "15": [
+ "NWPS-TXMEBR2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "26.0": {
+ "262.345": {
+ "90": {
+ "67": {
+ "15": [
+ "NWPS-TXME1",
+ "NWPS"
+ ]
+ }
+ }
+ },
+ "261.5": {
+ "98": {
+ "88": {
+ "15": [
+ "NWPS-TXMECC1",
+ "NWPS"
+ ]
+ }
+ },
+ "217": {
+ "195": {
+ "15": [
+ "NWPS-TXMECC2",
+ "NWPS"
+ ]
+ }
+ }
+ },
+ "279.86": {
+ "178": {
+ "70": {
+ "15": [
+ "NWPS-FLMEFTL",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "26.13": {
+ "278.1": {
+ "75": {
+ "84": {
+ "15": [
+ "NWPS-SWFLME1",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "26.18": {
+ "279.89": {
+ "65": {
+ "59": {
+ "15": [
+ "NWPS-SEFLME1",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "26.89": {
+ "279.93": {
+ "47": {
+ "42": {
+ "15": [
+ "NWPS-SEFLME2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "25.85": {
+ "279.87": {
+ "167": {
+ "101": {
+ "15": [
+ "NWPS-SEFLME3",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "28.5": {
+ "270.9": {
+ "61": {
+ "79": {
+ "15": [
+ "NWPS-MSALME1",
+ "NWPS"
+ ]
+ }
+ },
+ "70": {
+ "79": {
+ "15": [
+ "NWPS-ALMEMP1",
+ "NWPS"
+ ]
+ }
+ },
+ "155": {
+ "174": {
+ "15": [
+ "NWPS-ALMEMP2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "40.45": {
+ "287.5": {
+ "104": {
+ "103": {
+ "15": [
+ "NWPS-MAMEBT1",
+ "NWPS"
+ ]
+ }
+ },
+ "156": {
+ "155": {
+ "15": [
+ "NWPS-MAMEBT2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "41.9": {
+ "288.8": {
+ "289": {
+ "149": {
+ "15": [
+ "NWPS-MAMEBT3",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "43.47": {
+ "288.5": {
+ "289": {
+ "251": {
+ "15": [
+ "NWPS-NHMELKW",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "43.76": {
+ "289.36": {
+ "189": {
+ "145": {
+ "15": [
+ "NWPS-MEMELKS",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "42.4": {
+ "289.021": {
+ "67": {
+ "74": {
+ "15": [
+ "NWPS-MEME7",
+ "NWPS"
+ ]
+ }
+ }
+ },
+ "288.25": {
+ "74": {
+ "92": {
+ "15": [
+ "NWPS-MEMEGP1",
+ "NWPS"
+ ]
+ }
+ },
+ "103": {
+ "129": {
+ "15": [
+ "NWPS-MEMEGP2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "27.0": {
+ "262.6": {
+ "84": {
+ "100": {
+ "15": [
+ "NWPS-TXMEHG4",
+ "NWPS"
+ ]
+ }
+ },
+ "98": {
+ "100": {
+ "15": [
+ "NWPS-TXMEHG1",
+ "NWPS"
+ ]
+ }
+ },
+ "217": {
+ "222": {
+ "15": [
+ "NWPS-TXMEHG2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "30.93": {
+ "278.53": {
+ "81": {
+ "42": {
+ "15": [
+ "NWPS-GAMESAS",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "30.62": {
+ "278.51": {
+ "89": {
+ "111": {
+ "15": [
+ "NWPS-GAMEKBY",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "38.5": {
+ "284.3": {
+ "151": {
+ "92": {
+ "15": [
+ "NWPS-DEMEDB1",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "24.3": {
+ "276.8": {
+ "123": {
+ "315": {
+ "15": [
+ "NWPS-FLMEFKY",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "23.0": {
+ "276.5": {
+ "84": {
+ "116": {
+ "15": [
+ "NWPS-FLMEKW1",
+ "NWPS"
+ ]
+ }
+ },
+ "96": {
+ "132": {
+ "15": [
+ "NWPS-FLMEKW2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "27.48": {
+ "276.8": {
+ "182": {
+ "232": {
+ "15": [
+ "NWPS-FLMETB1",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "38.2": {
+ "284.424": {
+ "89": {
+ "65": {
+ "15": [
+ "NWPS-PAMEMH1",
+ "NWPS"
+ ]
+ }
+ }
+ },
+ "284.25": {
+ "89": {
+ "70": {
+ "15": [
+ "NWPS-PAMEMH1",
+ "NWPS"
+ ]
+ }
+ },
+ "149": {
+ "117": {
+ "15": [
+ "NWPS-PAMEMH2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "28.2": {
+ "279.2": {
+ "134": {
+ "99": {
+ "15": [
+ "NWPS-FLMECPC",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "26.5": {
+ "278.681": {
+ "98": {
+ "70": {
+ "15": [
+ "NWPS-NFLAME1",
+ "NWPS"
+ ]
+ }
+ }
+ },
+ "278.6": {
+ "98": {
+ "72": {
+ "15": [
+ "NWPS-FLMEMB1",
+ "NWPS"
+ ]
+ }
+ },
+ "217": {
+ "160": {
+ "15": [
+ "NWPS-FLMEMB2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "29.87": {
+ "278.68": {
+ "34": {
+ "26": {
+ "15": [
+ "NWPS-FLMESTA",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "30.38": {
+ "278.57": {
+ "28": {
+ "34": {
+ "15": [
+ "NWPS-FLMEMPT",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "17.0": {
+ "292.0": {
+ "93": {
+ "142": {
+ "15": [
+ "NWPS-PRMESJ1",
+ "NWPS"
+ ]
+ }
+ },
+ "155": {
+ "237": {
+ "15": [
+ "NWPS-PRMESJ2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "17.55": {
+ "292.6": {
+ "140": {
+ "319": {
+ "15": [
+ "NWPS-PRMESJ3",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "18.33": {
+ "292.7": {
+ "212": {
+ "212": {
+ "15": [
+ "NWPS-PRMERIN",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "29.0": {
+ "264.82": {
+ "178": {
+ "141": {
+ "15": [
+ "NWPS-TXMEHG3",
+ "NWPS"
+ ]
+ }
+ }
+ },
+ "276.45": {
+ "89": {
+ "104": {
+ "15": [
+ "NWPS-FLMECKY",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "28.35": {
+ "272.65": {
+ "62": {
+ "115": {
+ "15": [
+ "NWPS-FLMETL4",
+ "NWPS"
+ ]
+ },
+ "116": {
+ "15": [
+ "NWPS-FLMETL3",
+ "NWPS"
+ ]
+ }
+ },
+ "63": {
+ "117": {
+ "15": [
+ "NWPS-FLMETL2",
+ "NWPS"
+ ]
+ }
+ },
+ "140": {
+ "259": {
+ "15": [
+ "NWPS-FLMETL1",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "30.15": {
+ "271.55": {
+ "104": {
+ "103": {
+ "15": [
+ "NWPS-ALMEMBY",
+ "NWPS"
+ ]
+ }
+ }
+ },
+ "273.2": {
+ "60": {
+ "94": {
+ "15": [
+ "NWPS-FLMECBY",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "30.1": {
+ "272.68": {
+ "82": {
+ "70": {
+ "15": [
+ "NWPS-FLMEESP",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "30.08": {
+ "272.32": {
+ "70": {
+ "62": {
+ "15": [
+ "NWPS-FLMEPBY",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "30.67": {
+ "278.3": {
+ "81": {
+ "87": {
+ "15": [
+ "NWPS-SCMECH2",
+ "NWPS"
+ ]
+ }
+ },
+ "180": {
+ "192": {
+ "15": [
+ "NWPS-SCMECH1",
+ "NWPS"
+ ]
+ }
+ }
+ },
+ "278.384": {
+ "81": {
+ "85": {
+ "15": [
+ "NWPS-SCARME2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "36.75": {
+ "282.2": {
+ "110": {
+ "72": {
+ "15": [
+ "NWPS-MDMEBW2",
+ "NWPS"
+ ]
+ }
+ },
+ "164": {
+ "107": {
+ "15": [
+ "NWPS-MDMEBW1",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "43.3": {
+ "290.8": {
+ "49": {
+ "59": {
+ "15": [
+ "NWPS-MEMECB2",
+ "NWPS"
+ ]
+ }
+ },
+ "109": {
+ "131": {
+ "15": [
+ "NWPS-MEMECB1",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "44.16": {
+ "291.58": {
+ "156": {
+ "160": {
+ "15": [
+ "NWPS-MEMEBHR",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "44.27": {
+ "291.83": {
+ "167": {
+ "168": {
+ "15": [
+ "NWPS-MEMEWHR",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "32.5": {
+ "279.6": {
+ "85": {
+ "122": {
+ "15": [
+ "NWPS-NCMEWL2",
+ "NWPS"
+ ]
+ }
+ },
+ "86": {
+ "122": {
+ "15": [
+ "NWPS-NCMEWL2",
+ "NWPS"
+ ]
+ }
+ },
+ "103": {
+ "147": {
+ "15": [
+ "NWPS-NCMEWL1",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "34.4": {
+ "282.73": {
+ "83": {
+ "68": {
+ "15": [
+ "NWPS-NCMESOB",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "34.6": {
+ "282.85": {
+ "99": {
+ "153": {
+ "15": [
+ "NWPS-NCMEEMI",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "29.85": {
+ "275.5": {
+ "56": {
+ "137": {
+ "15": [
+ "NWPS-FLMESTM",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "29.6": {
+ "274.54": {
+ "65": {
+ "99": {
+ "15": [
+ "NWPS-FLMECSB",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "29.95": {
+ "274.13": {
+ "78": {
+ "81": {
+ "15": [
+ "NWPS-FLMEPCY",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "29.3": {
+ "269.4": {
+ "98": {
+ "140": {
+ "15": [
+ "NWPS-LAMELPN",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "27.5": {
+ "268.2": {
+ "87": {
+ "109": {
+ "15": [
+ "NWPS-LAMENB1",
+ "NWPS"
+ ]
+ }
+ },
+ "192": {
+ "242": {
+ "15": [
+ "NWPS-LAMENB2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "16.201": {
+ "285.72": {
+ "94": {
+ "108": {
+ "105": [
+ "RAP-PRLC16KM",
+ "RAP"
+ ]
+ }
+ }
+ }
+ },
+ "20.19": {
+ "238.45": {
+ "689": {
+ "1073": {
+ "0": [
+ "SPC-USLCAWI4",
+ "SPC"
+ ]
+ }
+ }
+ }
+ },
+ "32.6": {
+ "236.64": {
+ "78": {
+ "114": {
+ "15": [
+ "NWPS-CAMELA3",
+ "NWPS"
+ ]
+ }
+ },
+ "98": {
+ "142": {
+ "15": [
+ "NWPS-CAMELA1",
+ "NWPS"
+ ]
+ }
+ },
+ "112": {
+ "163": {
+ "15": [
+ "NWPS-CAMELA2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "32.08": {
+ "241.0": {
+ "65": {
+ "79": {
+ "15": [
+ "NWPS-CAMESD2",
+ "NWPS"
+ ]
+ }
+ },
+ "97": {
+ "118": {
+ "15": [
+ "NWPS-CAMESD1",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "33.52": {
+ "241.59": {
+ "56": {
+ "112": {
+ "15": [
+ "NWPS-CAMEPLB",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "31.95": {
+ "240.8": {
+ "37": {
+ "44": {
+ "15": [
+ "NWPS-CAMESD3",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "32.68": {
+ "242.66": {
+ "195": {
+ "79": {
+ "15": [
+ "NWPS-CAMECLJ",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "37.3": {
+ "236.7": {
+ "123": {
+ "89": {
+ "15": [
+ "NWPS-CAMESFB",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "36.2": {
+ "237.8": {
+ "178": {
+ "90": {
+ "15": [
+ "NWPS-CAMEMRB",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "53.9": {
+ "215.5": {
+ "59": {
+ "79": {
+ "15": [
+ "NWPS-AKMEJN2",
+ "NWPS"
+ ]
+ }
+ },
+ "177": {
+ "237": {
+ "15": [
+ "NWPS-AKMEJN1",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "57.0": {
+ "222.6": {
+ "300": {
+ "297": {
+ "15": [
+ "NWPS-AKMEGBY",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "54.5": {
+ "224.0": {
+ "306": {
+ "387": {
+ "15": [
+ "NWPS-AKMEWLS",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "53.15": {
+ "183.0": {
+ "82": {
+ "136": {
+ "15": [
+ "NWPS-AKMEAI1",
+ "NWPS"
+ ]
+ }
+ },
+ "132": {
+ "218": {
+ "15": [
+ "NWPS-AKMEAI2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "58.8": {
+ "205.6": {
+ "311": {
+ "317": {
+ "15": [
+ "NWPS-AKMEAN3",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "41.0": {
+ "232.7": {
+ "77": {
+ "62": {
+ "15": [
+ "NWPS-ORMEMD2",
+ "NWPS"
+ ]
+ }
+ },
+ "128": {
+ "103": {
+ "15": [
+ "NWPS-ORMEMD3",
+ "NWPS"
+ ]
+ }
+ },
+ "192": {
+ "154": {
+ "15": [
+ "NWPS-ORMEMD1",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "32.61": {
+ "279.99": {
+ "49": {
+ "75": {
+ "15": [
+ "NWPS-SCMECHB",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "31.88": {
+ "278.87": {
+ "112": {
+ "131": {
+ "15": [
+ "NWPS-GAMESHB",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "39.4": {
+ "285.4": {
+ "112": {
+ "69": {
+ "15": [
+ "NWPS-NJMELBI",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "46.1": {
+ "233.0": {
+ "74": {
+ "79": {
+ "15": [
+ "NWPS-WAMESE3",
+ "NWPS"
+ ]
+ }
+ },
+ "93": {
+ "99": {
+ "15": [
+ "NWPS-WAMESE1",
+ "NWPS"
+ ]
+ }
+ },
+ "123": {
+ "131": {
+ "15": [
+ "NWPS-WAMESE2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "38.4": {
+ "233.73": {
+ "85": {
+ "51": {
+ "15": [
+ "NWPS-CAMEEU3",
+ "NWPS"
+ ]
+ }
+ },
+ "106": {
+ "64": {
+ "15": [
+ "NWPS-CAMEEU1",
+ "NWPS"
+ ]
+ }
+ },
+ "141": {
+ "85": {
+ "15": [
+ "NWPS-CAMEEU2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "18.07": {
+ "198.5": {
+ "70": {
+ "101": {
+ "15": [
+ "NWPS-HIME2",
+ "NWPS"
+ ]
+ }
+ },
+ "207": {
+ "298": {
+ "15": [
+ "NWPS-HIME1",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "21.55": {
+ "199.96": {
+ "190": {
+ "197": {
+ "15": [
+ "NWPS-HIMEKAN",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "19.4": {
+ "204.8": {
+ "112": {
+ "126": {
+ "15": [
+ "NWPS-HIMEHIL",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "20.4": {
+ "203.1": {
+ "223": {
+ "209": {
+ "15": [
+ "NWPS-HIMEMAU",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "21.2": {
+ "201.65": {
+ "136": {
+ "156": {
+ "15": [
+ "NWPS-HIMEOAH",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "12.34": {
+ "143.67": {
+ "67": {
+ "67": {
+ "15": [
+ "NWPS-GUME2",
+ "NWPS"
+ ]
+ }
+ },
+ "125": {
+ "126": {
+ "15": [
+ "NWPS-GUME1",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "14.72": {
+ "145.32": {
+ "74": {
+ "72": {
+ "15": [
+ "NWPS-GUMENMI",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "14.02": {
+ "145.02": {
+ "29": {
+ "39": {
+ "15": [
+ "NWPS-GUMEROT",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "13.12": {
+ "144.52": {
+ "74": {
+ "61": {
+ "15": [
+ "NWPS-GUMEISL",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "23.018": {
+ "275.667": {
+ "574": {
+ "480": {
+ "84": [
+ "NAM-FLLC",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "23.097": {
+ "240.964": {
+ "881": {
+ "1121": {
+ "190": [
+ "AWC-UNKNPS",
+ "AWC"
+ ]
+ }
+ }
+ }
+ },
+ "19.229": {
+ "233.7234": {
+ "1597": {
+ "2345": {
+ "104": [
+ "NBM-USLCDRS3",
+ "NBM"
+ ],
+ "108": [
+ "LMP-USLCDRS2",
+ "LMP"
+ ],
+ "113": [
+ "SREF-USLCDRS",
+ "SREF"
+ ]
+ }
+ }
+ }
+ },
+ "65.92": {
+ "193.82": {
+ "143": {
+ "155": {
+ "15": [
+ "NWPS-AKMEKBS",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "62.24": {
+ "193.33": {
+ "176": {
+ "180": {
+ "15": [
+ "NWPS-AKMENTS",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "61.0": {
+ "183.5": {
+ "178": {
+ "266": {
+ "15": [
+ "NWPS-AKMEFBK",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "59.65": {
+ "211.23": {
+ "189": {
+ "193": {
+ "15": [
+ "NWPS-AKMEPWS",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "43.5": {
+ "233.72": {
+ "82": {
+ "49": {
+ "15": [
+ "NWPS-ORMEPT3",
+ "NWPS"
+ ]
+ }
+ },
+ "136": {
+ "81": {
+ "15": [
+ "NWPS-ORMEPT2",
+ "NWPS"
+ ]
+ }
+ },
+ "203": {
+ "121": {
+ "15": [
+ "NWPS-ORMEPT1",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "46.84": {
+ "235.74": {
+ "38": {
+ "25": {
+ "15": [
+ "NWPS-WAME4",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "43.6": {
+ "286": {
+ "91": {
+ "31": {
+ "133": [
+ "GLSW",
+ "GLSW"
+ ]
+ }
+ }
+ }
+ },
+ "16.293825": {
+ "233.8932": {
+ "337": {
+ "451": {
+ "191": [
+ "ICN-USLC13KM",
+ "ICNG"
+ ]
+ }
+ }
+ }
+ },
+ "14.3515": {
+ "195.0305": {
+ "561": {
+ "625": {
+ "104": [
+ "NBM-TRUEME1",
+ "NBM"
+ ]
+ }
+ }
+ }
+ },
+ "16.828685": {
+ "291.804687": {
+ "129": {
+ "177": {
+ "89": [
+ "NMMB-TRUEME1",
+ "NMMB"
+ ],
+ "109": [
+ "RTM-TRUEME3",
+ "RTMA"
+ ]
+ }
+ }
+ }
+ },
+ "12.349884": {
+ "143.686538": {
+ "193": {
+ "193": {
+ "104": [
+ "NBM-TRUEME4",
+ "NBM"
+ ],
+ "109": [
+ "RTM-TRUEME4",
+ "RTMA"
+ ]
+ }
+ }
+ }
+ },
+ "18.072699": {
+ "198.474999": {
+ "225": {
+ "321": {
+ "17": [
+ "ESSP-TRUEME",
+ "ESSP"
+ ],
+ "89": [
+ "NMMB-TRUEME2",
+ "NMMB"
+ ],
+ "109": [
+ "RTM-TRUEME1",
+ "RTMA"
+ ]
+ }
+ }
+ }
+ },
+ "-30.4192": {
+ "129.9058": {
+ "1817": {
+ "2517": {
+ "104": [
+ "NBM-TRUEME2",
+ "NBM"
+ ]
+ }
+ }
+ }
+ },
+ "-45.0": {
+ "110.0": {
+ "725": {
+ "837": {
+ "81": [
+ "GFS-TRUEME",
+ "GFS"
+ ],
+ "96": [
+ "GFS-TRUEME",
+ "GFS"
+ ]
+ }
+ }
+ }
+ },
+ "16.9775": {
+ "291.9722": {
+ "225": {
+ "339": {
+ "104": [
+ "NBM-TRUEME3",
+ "NBM"
+ ]
+ }
+ }
+ }
+ },
+ "30.000002": {
+ "130.0": {
+ "301": {
+ "511": {
+ "11": [
+ "MGWM-EPME",
+ "MGWM"
+ ]
+ }
+ }
+ }
+ },
+ "46.13": {
+ "235.8": {
+ "56": {
+ "39": {
+ "15": [
+ "NWPS-ORMECRB",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "43.15": {
+ "235.3": {
+ "167": {
+ "122": {
+ "15": [
+ "NWPS-ORMECWB",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "89.277": {
+ "0.0": {
+ "190": {
+ "384": {
+ "98": [
+ "CFS-UNKNGAUS",
+ "CFS"
+ ]
+ }
+ }
+ }
+ },
+ "16.977485": {
+ "291.972167": {
+ "225": {
+ "339": {
+ "14": [
+ "ESSA-TRUEME",
+ "ESSA"
+ ]
+ }
+ }
+ }
+ },
+ "45.37": {
+ "235.8": {
+ "89": {
+ "47": {
+ "15": [
+ "NWPS-ORMETKB",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "52.870833": {
+ "235.270833": {
+ "3351": {
+ "6935": {
+ "25": [
+ "SNOW-UNKNME",
+ "SNOW"
+ ]
+ }
+ }
+ }
+ },
+ "26.72": {
+ "279.86": {
+ "156": {
+ "80": {
+ "15": [
+ "NWPS-FLMEPAB",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "28.7": {
+ "278.3": {
+ "105": {
+ "67": {
+ "15": [
+ "NWPS-FLMEJK2",
+ "NWPS"
+ ]
+ }
+ },
+ "204": {
+ "131": {
+ "15": [
+ "NWPS-FLMEJK1",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "26.67": {
+ "278.87": {
+ "125": {
+ "112": {
+ "15": [
+ "NWPS-FLMELKO",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "25.72": {
+ "279.8": {
+ "156": {
+ "86": {
+ "15": [
+ "NWPS-FLMEMIB",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "42.22": {
+ "234.9": {
+ "181": {
+ "140": {
+ "15": [
+ "NWPS-ORMECPB",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "42.0": {
+ "235.55": {
+ "56": {
+ "75": {
+ "15": [
+ "NWPS-ORMEBGS",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "16.8287": {
+ "291.8047": {
+ "257": {
+ "353": {
+ "89": [
+ "NMMB-TRUEME3",
+ "NMMB"
+ ],
+ "109": [
+ "RTM-TRUEME2",
+ "RTMA"
+ ],
+ "118": [
+ "URMA-TRUEME1",
+ "URMA"
+ ]
+ }
+ }
+ }
+ },
+ "47.0": {
+ "261.0": {
+ "481": {
+ "586": {
+ "11": [
+ "MGWM-ECGMME",
+ "MGWM"
+ ]
+ }
+ }
+ }
+ },
+ "21.138": {
+ "237.28": {
+ "635": {
+ "1079": {
+ "84": [
+ "NAM-USLCUNKN",
+ "NAM"
+ ],
+ "116": [
+ "WRFE-USLCUNK",
+ "WRFE"
+ ]
+ }
+ }
+ }
+ },
+ "71.995": {
+ "184.005": {
+ "2200": {
+ "5000": {
+ "98": [
+ "OAR-UNKNME1",
+ "CFS"
+ ]
+ }
+ }
+ }
+ },
+ "35.349744": {
+ "315.0": {
+ "1006": {
+ "1006": {
+ "11": [
+ "MGWM-UNKNPS",
+ "MGWM"
+ ]
+ }
+ }
+ }
+ },
+ "25.998": {
+ "196.001996": {
+ "2200": {
+ "2600": {
+ "99": [
+ "OAR-UNKNME2",
+ "TEST"
+ ]
+ }
+ }
+ }
+ },
+ "20.191924": {
+ "238.445993": {
+ "1377": {
+ "2145": {
+ "118": [
+ "URMA-USLCDR1",
+ "URMA"
+ ]
+ }
+ }
+ }
+ },
+ "53.82": {
+ "193.25": {
+ "300": {
+ "296": {
+ "15": [
+ "NWPS-AKMEUNK",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "57.8": {
+ "200.8": {
+ "296": {
+ "296": {
+ "15": [
+ "NWPS-AKMEUN2",
+ "NWPS"
+ ]
+ }
+ }
+ }
+ },
+ "37.979669": {
+ "234.042695": {
+ "795": {
+ "709": {
+ "118": [
+ "URMA-USLCDR2",
+ "URMA"
+ ]
+ }
+ }
+ }
+ }
+ }
diff --git a/grib_processor/grib.py b/grib_processor/grib.py
new file mode 100644
index 0000000000000000000000000000000000000000..7abd23561a6485ca21cb2a5b7fa8290d138ef352
--- /dev/null
+++ b/grib_processor/grib.py
@@ -0,0 +1,205 @@
+"""
+grib_processor.amqp
+~~~~~~~~~~~~~~~~~~~
+
+How the processor generates grib messages.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+import json
+import os
+import sys
+from typing import Generator
+
+import grib2io
+from ssec_amqp import utils as amqp_utils
+from typing_extensions import Literal, TypedDict
+
+from grib_processor import data
+
+
+if sys.version_info < (3, 9):
+ import importlib_resources as resources
+else:
+ import importlib.resources as resources
+
+# Contains a serializable mapping of first_lat, first_lon, rows, cols,
+# and generating_process_ids to xcd model names and ids.
+#
+# format:
+
+# dict[first_lat,
+# dict[first_lon,
+# dict[rows,
+# dict[cols,
+# dict[generating_process_id, list[str]]
+# ]
+# ]
+# ]
+# ]
+# Loaded using load_xcd_models()
+_XCD_MODELS = None
+
+# default model_name, model_id
+_XCD_MISSING = ("UNKWN", "UNKWN")
+
+
+class GribMetadata(TypedDict):
+ """Metadata extracted from each grib message."""
+
+ first_lat: float | None
+ first_lon: float | None
+ last_lat: float | None
+ last_lon: float | None
+ forecast_hour: int
+ run_hour: int
+ model_time: datetime
+ start_time: datetime
+ projection: str
+ center_id: int
+ center_desc: str
+ level: str
+ parameter: str
+ param_long: str
+ param_units: str
+ grib_number: Literal[1, 2]
+ size: str
+ ensemble: int | None
+ model_id: int
+ model: str
+ xcd_model: str
+ xcd_model_id: str
+ grib_record_number: int
+ title: str
+
+
+class GribPayload(GribMetadata):
+ """Amqp payload for each grib message.
+
+ Includes metadata about the grib message and metadata about
+ the file and server itself.
+ """
+
+ __payload_gen_time__: datetime
+ __injector_script__: str
+ path: str
+ directory: str
+ file_name: str
+ server_ip: str
+ server_type: Literal["realtime", "archive"]
+
+
+def load_xcd_models(
+ *addtnl_models: dict[str, dict[str, dict[str, dict[str, dict[str, list[str]]]]]],
+) -> None:
+ """Load the xcd models from the package."""
+ global _XCD_MODELS
+ if _XCD_MODELS is None:
+ # This MUST match name of xcd file in grib_processor.data
+ data_path = resources.files(data) / "xcd_model_info.json"
+ with data_path.open("r") as xcd_data:
+ _XCD_MODELS = json.load(xcd_data)
+ for addtnl in addtnl_models:
+ _XCD_MODELS.update(addtnl)
+
+
+def xcd_lookup(
+ first_lat: float, first_lon: float, rows: int, cols: int, gen_proc_id: int
+) -> tuple[str, str]:
+ """Looks up xcd model names and ids based on grib message properties.
+
+ Args:
+ first_lat (float): The first latitude of the grib message.
+ first_lon (float): The first longitude of the grib message.
+ rows (int): The number of rows in the grib message.
+ cols (int): The number of columns in the grib message.
+ gen_proc_id (int): The generating process id of the grib message.
+
+ Returns:
+ tuple[str, str]: (xcd model name, xcd model id) or ("UNKWN", "UNKWN")
+ if can't find model info based on given properties.
+ """
+ if _XCD_MODELS is None:
+ return _XCD_MISSING
+ try:
+ return tuple(
+ _XCD_MODELS[str(first_lat)][str(first_lon)][str(rows)][str(cols)][
+ str(gen_proc_id)
+ ]
+ )
+ except KeyError:
+ return _XCD_MISSING
+
+
+def extract_metadata(msg: grib2io.Grib2Message) -> GribMetadata:
+ """Extract metadata about a grib2 message created using grib2io.
+
+ Returns:
+ dictionary with keys defined by GribMetadata.
+ """
+ f_lat = getattr(msg, "latitudeFirstGridpoint", None)
+ f_lon = getattr(msg, "longitudeFirstGridpoint", None)
+ rows = msg.ny
+ cols = msg.nx
+ gen_proc = msg.generatingProcess
+ if f_lat is None or f_lon is None:
+ xcd_info = _XCD_MISSING
+ else:
+ xcd_info = xcd_lookup(f_lat, f_lon, rows, cols, gen_proc.value)
+ return GribMetadata(
+ first_lat=f_lat,
+ last_lat=getattr(msg, "latitudeLastGridpoint", None),
+ first_lon=f_lon,
+ last_lon=getattr(msg, "longitudeLastGridpoint", None),
+ forecast_hour=int(msg.valueOfForecastTime),
+ run_hour=int(msg.hour) * 100 + int(msg.minute),
+ model_time=amqp_utils.format_datetime(msg.refDate),
+ start_time=amqp_utils.format_datetime(msg.refDate),
+ projection=msg.gridDefinitionTemplateNumber.definition,
+ center_id=int(msg.originatingCenter.value),
+ center_desc=msg.originatingCenter.definition,
+ level=msg.level,
+ parameter=msg.shortName,
+ param_long=msg.fullName,
+ param_units=msg.units,
+ grib_number=2,
+ size="%d x %d" % (rows, cols),
+ ensemble=getattr(msg, "pertubationNumber", None),
+ model_id=int(gen_proc.value),
+ model=gen_proc.definition,
+ xcd_model=xcd_info[1],
+ xcd_model_id=xcd_info[0],
+ # resolution=msg._xydivisor, THIS IS NOT CORRECT!
+ grib_record_number=msg._msgnum,
+ title=str(msg),
+ )
+
+
+def itergrib(
+ grib_path: str, start_message: int | None = None
+) -> Generator[GribPayload, None, None]:
+ """Generate AMQP payloads from a grib file, starting at a specific message.
+
+ Args:
+ grib_path (str): The path to the grib file to create payloads for.
+ start_message (int): Which grib message to start yielding payloads for.
+
+ Yields:
+ JSON-able: The json formatted AMQP payload for each grib message from start_message to end of the file.
+ """
+ start_message = start_message or 0
+ with grib2io.open(grib_path) as grib_file:
+ for msg in grib_file[start_message:]:
+ meta = extract_metadata(msg)
+ yield GribPayload(
+ __payload_gen_time__=amqp_utils.format_datetime(datetime.now()),
+ __injector_script__=amqp_utils.INJECTOR_SCRIPT,
+ path=grib_path,
+ directory=os.path.dirname(grib_path),
+ file_name=os.path.basename(grib_path),
+ server_ip=amqp_utils.SERVER_NAME,
+ server_type="realtime",
+ **meta,
+ )
diff --git a/grib_processor/main.py b/grib_processor/main.py
new file mode 100644
index 0000000000000000000000000000000000000000..ea7dfad8fd2472b189ad9feb2c6b27b8ef2c28af
--- /dev/null
+++ b/grib_processor/main.py
@@ -0,0 +1,238 @@
+"""
+grib_processor.main
+~~~~~~~~~~~~~~~~~~~
+
+Unified entrypoint to the grib processor.
+"""
+
+from __future__ import annotations
+
+
+import argparse
+from itertools import chain
+import logging
+import logging.handlers
+import os
+import signal
+import sys
+from typing import Callable, Iterable
+import warnings
+
+from dotenv import load_dotenv
+import ssec_amqp.api as mq
+
+from grib_processor.grib import GribPayload, itergrib, load_xcd_models
+from grib_processor.utils import (
+ dump_message,
+ grib_file_watch,
+ publish_message,
+ realtime,
+ DEFAULT_WATCH_DEBOUNCE,
+ REALTIME_WATCH_DEBOUNCE,
+ signalcontext,
+)
+
+
+# Logging stuff
+LOG = logging.getLogger("grib_ingest")
+LOG_LEVELS = [
+ logging.CRITICAL,
+ logging.ERROR,
+ logging.WARNING,
+ logging.INFO,
+ logging.DEBUG,
+]
+LOG_NAME = "grib_processor.log"
+
+# Where we're publishing data to
+DEFAULT_AMQP_SERVERS = ["mq1.ssec.wisc.edu", "mq2.ssec.wisc.edu", "mq3.ssec.wisc.edu"]
+
+# Where on the servers we're putting the data.
+DEFAULT_AMQP_EXCHANGE = "model"
+
+
+def initialize_logging(verbosity: int, rotating_dir: str | None = None) -> None:
+ """Set up logging for the package. Don't use basicConfig so we don't
+ override other's logs.
+
+ Optionally, can specify an existing directory to put rotating log files into.
+ """
+ # Set up logging
+ log_formatter = logging.Formatter(
+ fmt="[%(asctime)s][%(levelname)s]-%(message)s", datefmt="%Y-%m-%dT%H:%M:%S"
+ )
+
+ if rotating_dir is not None:
+ file_handler = logging.handlers.TimedRotatingFileHandler(
+ os.path.join(rotating_dir, LOG_NAME), when="D", utc=True
+ )
+ file_handler.setFormatter(log_formatter)
+ file_handler.setLevel(logging.DEBUG)
+ LOG.addHandler(file_handler)
+
+ out_handler = logging.StreamHandler()
+ out_handler.setFormatter(log_formatter)
+ out_handler.setLevel(LOG_LEVELS[min(verbosity, len(LOG_LEVELS) - 1)])
+ LOG.addHandler(out_handler)
+
+ LOG.setLevel(logging.DEBUG)
+
+
+def setup() -> tuple[Iterable[GribPayload], Callable[[GribPayload], None]]:
+ """Initialize the script so it's ready to run."""
+ # Set up cmdline args
+ parser = argparse.ArgumentParser(
+ description="Set paramters for ingesting and publishing of grib data."
+ )
+ parser.add_argument(
+ "grib_src",
+ nargs="?",
+ default=None,
+ metavar="grib_source",
+ help="Where to get grib files from. Either stdin (default), a directory to watch, or a grib file itself.",
+ )
+ parser.add_argument(
+ "-v",
+ "--verbosity",
+ type=int,
+ default=3,
+ help="Specify verbosity to stderr from 0 (CRITICAL) to 5 (DEBUG).",
+ )
+ parser.add_argument(
+ "--log-dir",
+ default=None,
+ help="Set up a rotating file logger in this directory.",
+ )
+
+ parser.add_argument(
+ "-R",
+ "--realtime",
+ action="store_true",
+ help="Turns on parameters for more efficient realtime processing like caching and debounce. Also adds more robust error handling.",
+ )
+ parser.add_argument(
+ "-r",
+ "--recursive",
+ action="store_true",
+ help="When using a directory as the source, process all files in the directory recursively instead of setting up a watch.",
+ )
+
+ parser.add_argument(
+ "-o",
+ "--output",
+ choices=["amqp", "json"],
+ default="amqp",
+ help="Where to output processed grib messages to. Default is %(default)s.",
+ )
+
+ mq_group = parser.add_argument_group(
+ "RabbitMQ arguments",
+ "Options for how to output grib messages when using '-o=amqp'.",
+ )
+ mq_group.add_argument(
+ "--servers",
+ nargs="+",
+ default=DEFAULT_AMQP_SERVERS,
+ help="Servers to publish AMQP messages to. Defaults to %(default)s",
+ )
+ mq_group.add_argument(
+ "-X",
+ "--exchange",
+ default=DEFAULT_AMQP_EXCHANGE,
+ help='RabbitMQ exchange to publish the messages to. Defaults to "%(default)s"',
+ )
+ mq_group.add_argument(
+ "-u",
+ "--user",
+ default=None,
+ help="RabbitMQ user to login with. Can also specify using a .env file, see README#configuration.",
+ )
+ mq_group.add_argument(
+ "-p",
+ "--password",
+ default=None,
+ help="RabbitMQ password to login with. Can also specify using a .env file, see README#configuration.",
+ )
+
+ args = parser.parse_args()
+
+ if args.verbosity < 0:
+ parser.error("--verbosity cannot be negative!")
+ initialize_logging(args.verbosity, args.log_dir)
+ load_xcd_models()
+
+ # Get an iterator over grib files to process
+ file_iter: Iterable[str]
+ if args.grib_src is None:
+ LOG.info("Sourcing grib files from stdin, realtime=%s", args.realtime)
+ file_iter = map(str.strip, iter(sys.stdin.readline, ""))
+ elif os.path.isdir(args.grib_src):
+ LOG.info(
+ "Sourcing grib files from directory watch @ %s realtime=%s",
+ args.grib_src,
+ args.realtime,
+ )
+ if not args.realtime:
+ warnings.warn("Processing files from a directory without using --realtime!")
+ debounce = DEFAULT_WATCH_DEBOUNCE
+ else:
+ debounce = REALTIME_WATCH_DEBOUNCE
+
+ file_iter = grib_file_watch(
+ args.grib_src, debounce=debounce, recursive=args.recursive
+ )
+ elif os.path.isfile(args.grib_src):
+ LOG.info("Grib source is a file, parsing directly, %s", args.grib_src)
+ file_iter = (str(args.grib_src),)
+ else:
+ parser.error("{0} is not a valid source!".format(args.grib_src))
+
+ # Get an iterator of grib messages to emit
+ source: Iterable[GribPayload]
+ if args.realtime:
+ source = realtime(file_iter)
+ else:
+ source = chain.from_iterable(map(itergrib, file_iter))
+
+ # Possible to source connection parameters from the environment
+ if not load_dotenv():
+ LOG.warning("Couldn't find .env file")
+
+ if args.output == "amqp":
+ LOG.info("Emitting messages to amqp")
+ # will automatically disconnect on exit
+ mq.connect(
+ *args.servers,
+ user=args.user or os.getenv("RMQ_USER", None),
+ password=args.password or os.getenv("RMQ_PASS", None),
+ exchange=args.exchange,
+ )
+ LOG.info("Connected to %s", ", ".join(args.servers))
+ emit = publish_message
+ else:
+ LOG.info("Emitting messages to stdout")
+ emit = dump_message
+
+ return source, emit
+
+
+def __raise_interrupt(_sig, _frame): # noqa: ARG001
+ """Signal handler that raises KeybardInterrupt."""
+ raise KeyboardInterrupt
+
+
+def main() -> None:
+ """Main method, setup and start processing."""
+ try:
+ source, emit = setup()
+ except KeyboardInterrupt:
+ return
+
+ with signalcontext(signal.SIGTERM, __raise_interrupt):
+ try:
+ for grib_msg in source:
+ LOG.debug("Got %s from source", grib_msg)
+ emit(grib_msg)
+ except KeyboardInterrupt:
+ LOG.critical("Got interrupt, goodbye!")
+ return
diff --git a/grib_processor/utils.py b/grib_processor/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..7650600db16f4e42d5b7b3dda98682b1a399ea53
--- /dev/null
+++ b/grib_processor/utils.py
@@ -0,0 +1,247 @@
+"""
+grib_processor.sources
+~~~~~~~~~~~~~~~~~~~~~~
+
+The different ways to source grib files for processing.
+"""
+
+from __future__ import annotations
+
+import atexit
+from collections import defaultdict
+from contextlib import contextmanager
+import json
+import logging
+import os
+import signal
+import time
+from typing import Callable, DefaultDict, Generator, Iterable, Tuple
+
+from watchfiles import watch, Change
+from typing_extensions import TypeAlias
+import ssec_amqp.api as mq
+
+from grib_processor.grib import GribPayload, itergrib
+
+
+# Our logger
+LOG = logging.getLogger("grib_ingest")
+
+# How long grib files stay in the message cache.
+CACHE_TTL = 43200 # (12 hours)
+
+# How often to clean the cache of previous messages.
+CACHE_CLEAN_INTERVAL = 5400 # (1.5 hours)
+
+# What file extensions to parse as grib files
+GRIB_ENDINGS = [".grb", ".grib", ".grb2", ".grib2"]
+
+# Debounce is how long to group grib file changes over before processing them
+
+# watchfiles default debounce
+DEFAULT_WATCH_DEBOUNCE = 1600
+
+# debounce when doing realtime processing
+REALTIME_WATCH_DEBOUNCE = 4500
+
+# Format of the route key to messages publish with
+GRIB_ROUTE_KEY_FMT = "{xcd_model}.{xcd_model_id}.realtime.reserved.reserved.2"
+
+# Cache path
+REALTIME_CACHE_PATH = os.path.join(
+ os.path.expanduser("~/.cache/"), "grib_processor_cache.json"
+)
+
+# Types used for realtime caching
+_CEntry: TypeAlias = Tuple[float, int]
+_Cache: TypeAlias = DefaultDict[str, _CEntry]
+
+
+@contextmanager
+def signalcontext(signum: int, handler: Callable):
+ """Context manager that changes a signal handler on entry and resets it
+ on exit.
+
+ Args:
+ signum (int): Signal to change.
+ handler (tp.Callable): New signal handler to use.
+ """
+ try:
+ orig_h = signal.signal(signum, handler)
+ yield
+ finally:
+ signal.signal(signum, orig_h)
+
+
+def _default_cache_entry() -> _CEntry:
+ """default cache entry (current time, first grib message)."""
+ return time.time(), 0
+
+
+def _initialize_cache() -> _Cache:
+ """Sets up the realtime cache to use. Potentially loading previously saved
+ caches.
+
+ Returns:
+ _Cache: Cache of file names to last grib message processed.
+ """
+ LOG.info("Loading cache from disk")
+ try:
+ with open(REALTIME_CACHE_PATH, "r", encoding="utf-8") as cache_file:
+ cache_data = json.load(cache_file)
+ LOG.info("realtime - cache loaded from disk at %s", REALTIME_CACHE_PATH)
+ except (OSError, json.JSONDecodeError):
+ LOG.exception("Error loading previous cache data from %s", REALTIME_CACHE_PATH)
+ cache_data = {}
+
+ return defaultdict(_default_cache_entry, cache_data)
+
+
+def _save_cache(cache: _Cache) -> None:
+ """Saves a cache to disk at the path REALTIME_CACHE_PATH.
+
+ Args:
+ cache (_Cache): The cache to save to disk.
+ """
+ LOG.info("realtime - saving cache to disk")
+ if not cache:
+ LOG.debug("realtime - cache is empty")
+ return
+ try:
+ with open(REALTIME_CACHE_PATH, "w", encoding="utf-8") as cache_file:
+ json.dump(cache, cache_file)
+ LOG.info("realtime - cache saved to disk at %s", REALTIME_CACHE_PATH)
+ except (OSError, ValueError):
+ LOG.exception("Couldn't save cache to %s", REALTIME_CACHE_PATH)
+
+
+def _clean_caches(realtime_cache: _Cache) -> None:
+ """Removes expired items from the cache. Also, cleans caches
+ used by 3rd party libraries (grib2io) that could cause memory leaks
+ in long-running processing.
+
+ Args:
+ cache (_Cache): The realtime cache to clean.
+ """
+
+ to_remove = []
+ LOG.debug("realtime - Cleaning the cache, size=%d!", len(realtime_cache))
+ for path, (last_update_time, last_msg_num) in realtime_cache.items():
+ time_in_cache = time.time() - last_update_time
+ if time_in_cache > CACHE_TTL:
+ LOG.info(
+ "realtime - Evicted %s from cache! (last message: %d, time since last update: %f).",
+ path,
+ last_msg_num - 1,
+ time_in_cache,
+ )
+ to_remove.append(path)
+ if not to_remove:
+ LOG.debug("realtime - Nothing to remove from cache...")
+ else:
+ for rem_path in to_remove:
+ realtime_cache.pop(rem_path, None)
+ # LOG.debug('Cleaning grib2io caches')
+ # grib2io._grib2io._msg_class_store.clear()
+ # grib2io._grib2io._latlon_datastore.clear()
+
+
+def realtime(file_iter: Iterable[str]) -> Generator[GribPayload, None, None]:
+ """Add message caching and exception handling to a grib file iterator. This
+ is useful if the iterator returns files that aren't fully written to or
+ duplicates.
+
+ Args:
+ file_iter (Iterator[str]): An iterator returning paths to grib files.
+
+ Yields:
+ Generator[GribMessage, None, None]: A GribMessage generator.
+ """
+ cache = _initialize_cache()
+ _clean_caches(cache)
+ last_clean_time = time.time()
+ atexit.register(_save_cache, cache)
+
+ for file in file_iter:
+ LOG.info("realtime - got file %s", file)
+ next_msg = cache.get(file, (None, None))[1]
+ if next_msg is None:
+ LOG.debug("realtime - cache miss %s", file)
+ cache[file] = _default_cache_entry()
+ next_msg = 0
+ else:
+ LOG.debug("realtime - cache hit %s @ %d", file, next_msg)
+ msg_num = None
+ try:
+ for msg_num, msg in enumerate(
+ itergrib(file, start_message=next_msg), next_msg
+ ):
+ LOG.debug("realtime - got msg %s:%d", file, msg_num)
+ yield msg
+ except (KeyError, OSError):
+ LOG.exception(
+ "realtime - Error processing grib file %s @ msg %d", file, msg_num or 0
+ )
+
+ if msg_num is not None: # msg_num is None when exception or end of grib file
+ cache[file] = time.time(), msg_num + 1
+
+ if time.time() - last_clean_time > CACHE_CLEAN_INTERVAL:
+ LOG.info("realtime - time to clean the cache")
+ _clean_caches(cache)
+
+
+def grib_filter(change: Change, path: str) -> bool:
+ """Make sure files we ingest are grib files."""
+ if change == Change.deleted:
+ # Can't process a deleted file
+ return False
+ if os.path.splitext(path)[1] in GRIB_ENDINGS:
+ return True
+ return False
+
+
+def grib_file_watch(
+ watch_dir: str, debounce: int | None = None, recursive=False
+) -> Generator[str, None, None]:
+ """Get grib files by watching a directory for changes.
+
+ Args:
+ watch_dir (str): The directory to watch for grib files.
+ debounce (int | None): How long to wait to group file changes together before yielding.
+ recursive (bool): Watch directories under watch_dir recursively?
+
+ Returns:
+ Generator[str, None, None]: A grib file path generator.
+ """
+ debounce = debounce or DEFAULT_WATCH_DEBOUNCE
+ LOG.info(
+ "Watching directory %s for gribs with debounce=%d, recursive=%s",
+ watch_dir,
+ debounce,
+ recursive,
+ )
+ for changes in watch(
+ watch_dir,
+ watch_filter=grib_filter,
+ debounce=debounce,
+ recursive=recursive,
+ ):
+ for _, path in changes:
+ yield path
+
+
+def publish_message(msg: GribPayload) -> None:
+ """Publish a message to RabbitMQ servers using the ssec_amqp package.
+ This requires previous setup using ssec_amqp.api.connect().
+ """
+ route_key = GRIB_ROUTE_KEY_FMT.format(
+ xcd_model=msg.get("xcd_model"), xcd_model_id=msg.get("xcd_model_id")
+ )
+ status = mq.publish(msg, route_key=route_key)
+ LOG.info("Published to %s. status: %s", route_key, status)
+
+
+def dump_message(msg: GribPayload) -> None:
+ """Print a message to stdout."""
+ print(json.dumps(msg), flush=True)
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000000000000000000000000000000000000..da5a0699e36906116044a70530f7fbd64e19d1ab
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,37 @@
+[build-system]
+requires = ["setuptools >= 61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "grib_processor"
+dependencies = [
+ "grib2io >= 2.2.0",
+ "watchfiles >= 0.20.0",
+ "python-dotenv >= 1.0.0",
+ "quickmq >= 1.1.0",
+ "importlib-resources; python_version < '3.9'",
+ "typing-extensions",
+]
+requires-python = ">=3.8"
+authors = [
+ {name = "Max Drexler", email="mndrexler@wisc.edu"}
+]
+description = "Ingest and publish GRIB metadata to RabbitMQ servers. Made for the SSEC."
+readme = "README.md"
+license = { file = "LICENSE" }
+keywords = ["GRIB", "inventory", "SDS", "RabbitMQ", "SSEC"]
+dynamic = ['version']
+
+[project.urls]
+Homepage = "https://gitlab.ssec.wisc.edu/mdrexler/grib_rmq_stream"
+Repository = "https://gitlab.ssec.wisc.edu/mdrexler/grib_rmq_stream.git"
+Documentation = "https://gitlab.ssec.wisc.edu/mdrexler/grib_rmq_stream/-/blob/main/README.md?ref_type=heads"
+
+[project.scripts]
+grib_processor = "grib_processor:main.main"
+
+[tool.setuptools]
+packages = ["grib_processor"]
+
+[tool.setuptools.dynamic]
+version = {attr = "grib_processor.__version__"}
diff --git a/requirements.txt b/requirements.txt
index b9bf1777d94489b5a6d09f51d2cad962aa88d84f..2553c2a9b61382cd5c01a233e1edefd8ea1cc634 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,3 +2,5 @@ grib2io >= 2.2.0
watchfiles >= 0.20.0
python-dotenv >= 1.0.0
quickmq >= 1.1.0
+typing-extensions
+importlib_resoureces; python_version < '3.9'
diff --git a/requirements_dev.txt b/requirements_dev.txt
new file mode 100644
index 0000000000000000000000000000000000000000..9b775fe2d65f8dd5b60bf3c574311d81557c5ff2
--- /dev/null
+++ b/requirements_dev.txt
@@ -0,0 +1,5 @@
+ruff
+pre-commit
+mypy
+pytest
+coverage
diff --git a/tests/test_grib.py b/tests/test_grib.py
new file mode 100644
index 0000000000000000000000000000000000000000..351289ccf7e15c2a65347000184504556880d489
--- /dev/null
+++ b/tests/test_grib.py
@@ -0,0 +1,20 @@
+from grib_processor.grib import xcd_lookup, load_xcd_models
+
+
+def test_custom_loading():
+ """Test that loading a custom xcd model works with xcd_lookup.
+
+ Note: This is a leaky test, the custom model will be in following tests.
+ """
+ f_lat = 123
+ f_lon = 456
+ rows = 1
+ cols = 5
+ g_id = 6
+ m_name = "custom_name"
+ m_id = "custom_id"
+ c_model = {
+ str(f_lat): {str(f_lon): {str(rows): {str(cols): {str(g_id): [m_name, m_id]}}}}
+ }
+ load_xcd_models(c_model)
+ assert xcd_lookup(f_lat, f_lon, rows, cols, g_id) == (m_name, m_id)
diff --git a/todo b/todo
new file mode 100644
index 0000000000000000000000000000000000000000..5535e8968356c328deae26848e0e96e8e267caef
--- /dev/null
+++ b/todo
@@ -0,0 +1,8 @@
+Not in any particular order:
+
+* TESTING!!!
+* packaging
+ * Automatically bundle with grib2io
+* Add support for GRIB1 processing
+* DID I MENTION TESTING!!!
+* Documentation for the package.