Merge branch 'bugfix-pandas-masking' into 'master'

Fix bug in NetCDF creation masking all values

See merge request metobs/AossTower!6

.gitignore

+*.ascii
 *.csv
 *.swp
 *.nc

aosstower/level_b1/nc.py

@@ -37,7 +37,8 @@ def _get_data(input_files):
 
 def get_data(input_files):
     frame = pd.DataFrame(_get_data(input_files))
-    frame = frame.set_index('stamp').mask(frame == -99999.).fillna(value=np.nan)
+    frame = frame.set_index('stamp')
+    frame = frame.mask(frame == -99999.).fillna(value=np.nan)
     return frame
 
 

aosstower/tests/level_b1/test_nc.py

+#!/usr/bin/env python
+# encoding: utf8
+"""Test basic NetCDF generation."""
+
+import os
+from datetime import datetime
+import numpy as np
+
+
+def get_nc_schema_database(fields=None):
+    """Get a version of the NetCDF schema that mimics the nc script."""
+    from aosstower import schema
+    if fields is None:
+        fields = schema.met_vars
+    mini_database = {k: schema.database_dict[k] for k in fields}
+    return mini_database
+
+
+def test_nc_basic1(tmpdir):
+    """Test basic usage of the NetCDF generation."""
+    from aosstower.level_b1.nc import create_giant_netcdf
+    from aosstower.tests.utils import get_cached_level_00
+    from netCDF4 import Dataset
+
+    input_files = list(get_cached_level_00(num_files=2))
+    nc_out = tmpdir.join('test.nc')
+    create_giant_netcdf(
+        input_files, str(nc_out), True, None,
+        start=datetime(2020, 1, 2, 0, 0, 0),
+        interval_width='1T',
+        database=get_nc_schema_database(),
+    )
+    assert os.path.isfile(nc_out)
+    with Dataset(nc_out, 'r') as nc:
+        sflux = nc['solar_flux'][:]
+        assert np.count_nonzero(sflux.mask) == 2

aosstower/tests/utils.py

+#!/usr/bin/env python
+# encoding: utf8
+"""Utilities for running tests."""
+
+import os
+import urllib.request
+
+CACHE_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'test_data')
+
+
+def get_cached_level_00(num_files=2):
+    """Get a couple real ASCII files to test against."""
+    file_urls = (
+        "https://metobs-test.ssec.wisc.edu/pub/cache/aoss/tower/level_00/version_00/2020/01/01/aoss_tower.2020-01-01.ascii",
+        "https://metobs-test.ssec.wisc.edu/pub/cache/aoss/tower/level_00/version_00/2020/01/02/aoss_tower.2020-01-02.ascii",
+    )
+    if num_files > len(file_urls):
+        raise ValueError(f"Maximum of {len(file_urls)} can be loaded.")
+    file_urls = file_urls[:num_files]
+
+    for u in file_urls:
+        fn = os.path.join(CACHE_DIR, os.path.basename(u))
+        if not os.path.isfile(fn):
+            urllib.request.urlretrieve(u, fn)
+        yield fn