diff --git a/README.md b/README.md
index 8259c4079a5ec45d6598b0418dd4de5b0f9d7851..d8789e11305ad30ee839ddde9ae4ef617882b4fb 100644
--- a/README.md
+++ b/README.md
@@ -22,3 +22,11 @@
file_list_example: an example file list file needed by clavrxorb
+## Testing
+
+Requres py.test
+
+`test.sh` simply runs `py.test` in the `test/` directory.
+
+Sample output from select tests are saved for each release at `/ships19/cloud/archive/clavrx_test_data/version_granules`
+
diff --git a/test/save_record.py b/test/save_record.py
new file mode 100644
index 0000000000000000000000000000000000000000..b04c987720ac544d5dcd98210bba19e118431bd6
--- /dev/null
+++ b/test/save_record.py
@@ -0,0 +1,24 @@
+from pathlib import Path
+import test
+
+RECORD_DIR = Path('/ships19/cloud/archive/clavrx_test_data/version_granules')
+
+def main(version):
+ version_dir = RECORD_DIR / version
+ for func in test.save_funcs:
+ name = func.__name__
+ output_dir = version_dir / name
+ if output_dir.exists() and len(list(output_dir.glob('*')))>0:
+ print(output_dir, 'already exists')
+ else:
+ output_dir.mkdir(exist_ok=True,parents=True)
+ print(output_dir)
+ func(out_dir=output_dir)
+
+if __name__ == '__main__':
+ import argparse
+ parser = argparse.ArgumentParser()
+ parser.add_argument('version')
+ args = parser.parse_args()
+ main(args.version)
+
diff --git a/test/test.py b/test/test.py
index ddc1a3dc804ae9c68127d33969405667f07a8b56..4633df7127da449b8079892a6c5f8994dfc6630c 100644
--- a/test/test.py
+++ b/test/test.py
@@ -1,3 +1,16 @@
+"""
+test.py
+
+Test cases can be added by creating a function with "test" in the name
+
+A helper function "_run_it()" makes it easy to add a test.
+
+@save decorator marks tests that we'll permanently save the output from (see save_record.py)
+
+@extra decorator marks tests that won't be run unless we set TEST_EXTRA=True below)
+
+"""
+
from clavrx import run_clavrx, get_config, build_options_file, build_file_list
from pathlib import Path
from tempfile import mkdtemp
@@ -6,25 +19,54 @@ import subprocess
import pytest
import signal
import os
+import re
CLAVRX = Path('../clavrx_bin/clavrxorb').absolute()
assert CLAVRX.exists(), str(CLAVRX)+' does not exist'
HERE = Path(__file__).absolute().parent
+def get_all_l2_variables():
+ l2_variables = set()
+ pattern = re.compile('.*case\([\'"](\w+)[\'"]\).*')
+ with open(HERE / '../main_src/level2_mod.f90') as fp:
+ for line in fp:
+ if 'case("' in line:
+ match = pattern.match(line)
+ if match is not None:
+ l2_variables.add(match.group(1))
+ return sorted(l2_variables)
+
-L2_LIST_CONTENT = "testing\nlatitude\nlongitude\ncloud_probability\n"
+L2_LIST_CONTENT = '\n'.join(['testing',*get_all_l2_variables()])
TEST_EXTRA = False
extra = pytest.mark.skipif(not TEST_EXTRA, reason="extra test")
-def _run_it(main_l1b_file, aux_l1b_files=(), config_override=None):
+save_funcs = []
+def save(func):
+ save_funcs.append(func)
+ return func
+
+
+def _run_it(main_l1b_file, aux_l1b_files=(), config_override=None, out_dir=None):
+ """
+ Run clavrx case
+
+ main_l1b_file :: the primary l1b granule
+ aux_l1b_files :: list of files that will get linked next to the primary file in a temp dir
+ config_override :: dict of clavrx options to override the defaults (see options/default_options.yml)
+ out_dir :: directory to persist output the l2 file (default is a temporary directory which is removed)
+ """
main_l1b_file = Path(main_l1b_file)
tmpdir = Path(mkdtemp(dir=HERE))
try:
- out_dir = tmpdir / 'out'
- out_dir.mkdir()
+ if out_dir is None:
+ out_dir = tmpdir / 'out'
+ else:
+ out_dir = Path(out_dir)
+ out_dir.mkdir(exist_ok=True, parents=True)
l1b_links = []
for f in aux_l1b_files:
f = Path(f)
@@ -53,7 +95,7 @@ def _run_it(main_l1b_file, aux_l1b_files=(), config_override=None):
config[k][k2] = v2
else:
config[k] = v
- config['temp_dir'] = str(out_dir / 'temp_dir')
+ config['temp_dir'] = str(tmpdir) #str(out_dir / 'temp_dir')
options_file_content = build_options_file(config)
file_list_content = build_file_list(out_dir, [main_l1b_link])
level2_list_content = L2_LIST_CONTENT
@@ -97,16 +139,30 @@ def test_pgroup_bug():
rmtree(tmpdir)
-def test_viirs():
- ROOT = Path('/apollo/cloud/archive/Satellite_Input/VIIRS-N20/global/2018/098/')
- VIIRS_L1_FNAME = 'GMTCO_j01_d20180407_t2358242_e2359487_b01995_c20190226192728879159_noac_ops.h5'
- VIIRS_L1 = ROOT / VIIRS_L1_FNAME
- _run_it(VIIRS_L1)
+@save
+def test_noaa_viirs(out_dir=None):
+ #ROOT = Path('/apollo/cloud/archive/Satellite_Input/VIIRS-N20/global/2018/098/')
+ #VIIRS_L1_FNAME = 'GMTCO_j01_d20180407_t2358242_e2359487_b01995_c20190226192728879159_noac_ops.h5'
+ ROOT = Path('/ships19/cloud/archive/clavrx_test_data/viirs/noaa')
+ VIIRS_L1 = ROOT / 'GMTCO_npp_d20220221_t2356569_e0002373_b53481_c20220222005748538034_oebc_ops.h5'
+ aux = set(ROOT.glob('*npp_d20220221_t2356569_e0002373*'))
+ aux.remove(VIIRS_L1)
+ _run_it(VIIRS_L1, aux, out_dir=out_dir)
+
+@save
+def test_nasa_viirs(out_dir=None):
+ ROOT = Path('/ships19/cloud/archive/clavrx_test_data/viirs/nasa')
+ vnp03 = ROOT / 'VNP03MOD.A2019003.1700.002.2021102031552.nc'
+ extra = set(ROOT.glob('*A2019003.1700*'))
+ extra.remove(vnp03)
+ _run_it(vnp03, extra, out_dir=out_dir)
+
-def test_avhrr():
+@save
+def test_avhrr(out_dir=None):
AVHRR = Path('/arcdata/polar/noaa/noaa18/2020/2020_01_01_001/avhrr/NSS.GHRR.NN.D20001.S0000.E0143.B7532324.WI')
- _run_it(AVHRR)
+ _run_it(AVHRR, out_dir=out_dir)
def test_avhrr_get_goes_header_bug():
# This file is empty
@@ -118,14 +174,16 @@ def test_avhrr_get_goes_header_bug():
assert e.returncode == 4
-def test_fusion():
+@save
+def test_fusion(out_dir=None):
FUSION = Path('/ships19/cloud/archive/Satellite_Input/HIRS-FUSION/NN/2020/001/NSS.GHRR.NN.D20001.S0000.E0143.B7532324.WI.fusion.nc')
AVHRR = Path('/arcdata/polar/noaa/noaa18/2020/2020_01_01_001/avhrr/NSS.GHRR.NN.D20001.S0000.E0143.B7532324.WI')
override = {'lut':'ecm2_lut_avhrr123_hirs_common_chs.nc'}
- _run_it(FUSION, [AVHRR], config_override=override)
+ _run_it(FUSION, [AVHRR], config_override=override, out_dir=out_dir)
-def test_g16_fd():
+@save
+def test_g16_fd(out_dir=None):
ROOT = Path('/arcdata/goes/grb/goes16/2020/2020_05_02_123/abi/L1b/RadF/')
files = [next(ROOT.glob(f'OR_ABI-L1b-RadF-M6C{i:02d}_G16_s202012322201*.nc')) for i in range(1,17)]
override = {'bounds':{
@@ -141,15 +199,16 @@ def test_g16_fd():
'name': 'SUB'
}}
- _run_it(files[0], files[1:], config_override=override)
+ _run_it(files[0], files[1:], config_override=override, out_dir=out_dir)
-def test_g16_conus():
+@save
+def test_g16_conus(out_dir=None):
ROOT = Path('/arcdata/goes/grb/goes16/2020/2020_05_02_123/abi/L1b/RadC/')
files = [next(ROOT.glob(f'OR_ABI-L1b-RadC-M6C{i:02d}_G16_s2020123000111*.nc')) for i in range(1,17)]
override = {'bounds':{
'enable':True,
- 'lat_north': 20.0,
- 'lat_south': 10.0,
+ 'lat_north': 50.0,
+ 'lat_south': 40.0,
'lon_west': -90.,
'lon_east': -80.,
'zen_min': 0.0,
@@ -159,7 +218,7 @@ def test_g16_conus():
'name': 'SUB'
}}
- _run_it(files[0], files[1:], config_override=override)
+ _run_it(files[0], files[1:], config_override=override, out_dir=out_dir)
@extra
@@ -170,7 +229,8 @@ def test_process_cloud_off():
_run_it(AVHRR, config_override=override)
-def test_g17_fd():
+@save
+def test_g17_fd(out_dir=None):
ROOT = Path('/arcdata/goes/grb/goes17/2020/2020_05_02_123/abi/L1b/RadF/')
files = [next(ROOT.glob(f'OR_ABI-L1b-RadF-M6C{i:02d}_G17_s2020123222031*.nc')) for i in range(1,17)]
override = {'bounds':{
@@ -186,16 +246,17 @@ def test_g17_fd():
'name': 'SUB'
}}
- _run_it(files[0], files[1:], config_override=override)
+ _run_it(files[0], files[1:], config_override=override, out_dir=out_dir)
-def test_g17_conus():
+@save
+def test_g17_conus(out_dir=None):
ROOT = Path('/arcdata/goes/grb/goes17/2020/2020_05_02_123/abi/L1b/RadC/')
files = [next(ROOT.glob(f'OR_ABI-L1b-RadC-M6C{i:02d}_G17_s2020123222117*.nc')) for i in range(1,17)]
override = {'bounds':{
'enable':True,
- 'lat_north': 20.0,
- 'lat_south': 10.0,
+ 'lat_north': 40.0,
+ 'lat_south': 30.0,
'lon_west': -130.,
'lon_east': -120.,
'zen_min': 0.0,
@@ -205,10 +266,11 @@ def test_g17_conus():
'name': 'SUB'
}}
- _run_it(files[0], files[1:], config_override=override)
+ _run_it(files[0], files[1:], config_override=override, out_dir=out_dir)
-def test_h8_fd_dat():
+@save
+def test_h8_fd_dat(out_dir=None):
ROOT = Path('/arcdata/nongoes/japan/himawari08/2021_01/2021_01_01_001/2300/')
files = sorted(ROOT.glob('HS_H08_20210101_2300_B*_FLDK*'))
override = {'bounds':{
@@ -224,6 +286,6 @@ def test_h8_fd_dat():
'name': 'SUB'
}}
- _run_it(files[0], files[1:], config_override=override)
+ _run_it(files[0], files[1:], config_override=override, out_dir=out_dir)