From 00385bd7cc651ee36215c93563cfd816ee4b72b9 Mon Sep 17 00:00:00 2001
From: Paolo Veglio <paolo.veglio@ssec.wisc.edu>
Date: Thu, 2 Feb 2023 16:21:41 +0000
Subject: [PATCH] repository clean up
---
.gitignore | 1 +
dev/conf_xr.py | 186 ++++
dev/hires_module.py | 228 +++++
dev/main_hires.py | 125 +++
dev/ocean_day_tests.py | 338 +++++++
dev/plot_data.py | 161 ++++
dev/plot_mask.py | 295 +++++++
dev/sort_inputs.py | 97 ++
dev/tests.py | 535 ++++++++++++
dev/write_temp_data.py | 24 +
mvcm/__init__.py | 1 +
mvcm/ancillary.pyx | 205 +++++
mvcm/c_tools/__init__.py | 0
mvcm/c_tools/assign_geos_vals.c | 268 ++++++
mvcm/c_tools/bilinearInterpSST.c | 112 +++
mvcm/c_tools/check_reg_uniformity.c | 141 +++
mvcm/c_tools/cithr.c | 230 +++++
mvcm/c_tools/get_GEOS.c | 206 +++++
mvcm/c_tools/get_NDVI_background.c | 184 ++++
mvcm/c_tools/get_Olson_eco.c | 141 +++
mvcm/c_tools/get_Reynolds_SST.c | 162 ++++
mvcm/c_tools/get_b1_thresholds.c | 191 ++++
mvcm/c_tools/get_geos_times.c | 81 ++
mvcm/c_tools/get_granule_times.c | 76 ++
mvcm/c_tools/get_ti_vars.c | 91 ++
mvcm/c_tools/get_ti_vars_geos.c | 91 ++
mvcm/c_tools/get_ti_weights.c | 69 ++
mvcm/c_tools/get_ti_weights_geos.c | 69 ++
mvcm/c_tools/getcoord.c | 650 ++++++++++++++
mvcm/c_tools/include/ancillary.h | 41 +
.../include/mask_processing_constants.h | 39 +
mvcm/c_tools/include/sst.h | 26 +
mvcm/c_tools/read_GEOS.c | 219 +++++
mvcm/c_tools/read_GEOS_constants.c | 200 +++++
mvcm/c_tools/read_GEOS_lndocn.c | 187 ++++
mvcm/c_tools/snow_mask.c | 226 +++++
mvcm/c_tools/swap_bytes.c | 126 +++
mvcm/conf.py | 274 ++++++
mvcm/main.py | 532 +++++++++++
mvcm/preprocess_thresholds.py | 567 ++++++++++++
mvcm/read_data.py | 581 ++++++++++++
mvcm/restoral.py | 127 +++
mvcm/scene.py | 446 ++++++++++
mvcm/spectral_tests.py | 826 ++++++++++++++++++
mvcm/utils.py | 329 +++++++
pyproject.toml | 56 +-
setup.py | 46 +-
tests/test_read_data.py | 16 +-
thresholds/thresholds.mvcm.snpp.v0.0.1.yaml | 706 +++++++++++++++
unit_tests.py | 97 --
50 files changed, 10484 insertions(+), 141 deletions(-)
create mode 100644 dev/conf_xr.py
create mode 100644 dev/hires_module.py
create mode 100644 dev/main_hires.py
create mode 100644 dev/ocean_day_tests.py
create mode 100644 dev/plot_data.py
create mode 100644 dev/plot_mask.py
create mode 100644 dev/sort_inputs.py
create mode 100644 dev/tests.py
create mode 100644 dev/write_temp_data.py
create mode 100644 mvcm/__init__.py
create mode 100644 mvcm/ancillary.pyx
create mode 100644 mvcm/c_tools/__init__.py
create mode 100644 mvcm/c_tools/assign_geos_vals.c
create mode 100644 mvcm/c_tools/bilinearInterpSST.c
create mode 100644 mvcm/c_tools/check_reg_uniformity.c
create mode 100644 mvcm/c_tools/cithr.c
create mode 100644 mvcm/c_tools/get_GEOS.c
create mode 100644 mvcm/c_tools/get_NDVI_background.c
create mode 100644 mvcm/c_tools/get_Olson_eco.c
create mode 100644 mvcm/c_tools/get_Reynolds_SST.c
create mode 100644 mvcm/c_tools/get_b1_thresholds.c
create mode 100644 mvcm/c_tools/get_geos_times.c
create mode 100644 mvcm/c_tools/get_granule_times.c
create mode 100644 mvcm/c_tools/get_ti_vars.c
create mode 100644 mvcm/c_tools/get_ti_vars_geos.c
create mode 100644 mvcm/c_tools/get_ti_weights.c
create mode 100644 mvcm/c_tools/get_ti_weights_geos.c
create mode 100644 mvcm/c_tools/getcoord.c
create mode 100644 mvcm/c_tools/include/ancillary.h
create mode 100644 mvcm/c_tools/include/mask_processing_constants.h
create mode 100644 mvcm/c_tools/include/sst.h
create mode 100644 mvcm/c_tools/read_GEOS.c
create mode 100644 mvcm/c_tools/read_GEOS_constants.c
create mode 100644 mvcm/c_tools/read_GEOS_lndocn.c
create mode 100644 mvcm/c_tools/snow_mask.c
create mode 100644 mvcm/c_tools/swap_bytes.c
create mode 100644 mvcm/conf.py
create mode 100644 mvcm/main.py
create mode 100644 mvcm/preprocess_thresholds.py
create mode 100644 mvcm/read_data.py
create mode 100644 mvcm/restoral.py
create mode 100644 mvcm/scene.py
create mode 100644 mvcm/spectral_tests.py
create mode 100644 mvcm/utils.py
create mode 100644 thresholds/thresholds.mvcm.snpp.v0.0.1.yaml
delete mode 100644 unit_tests.py
diff --git a/.gitignore b/.gitignore
index bc2e71f..f1b919c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,6 +7,7 @@
*.vscode
+ancillary.c
viz.py
build
diff --git a/dev/conf_xr.py b/dev/conf_xr.py
new file mode 100644
index 0000000..655f687
--- /dev/null
+++ b/dev/conf_xr.py
@@ -0,0 +1,186 @@
+import numpy as np
+import xarray as xr
+
+
+def test(flipped=False):
+ bt = np.arange(265, 275)
+ if flipped is False:
+ thr = np.array([267, 270, 273, 1, 1])
+ else:
+ thr = np.array([273, 270, 267, 1, 1])
+ c = conf_test(bt, thr)
+ print(c)
+
+
+def test_dble(flipped=False):
+ bt = np.arange(260, 282)
+ if flipped is False:
+ thr = np.array([264, 267, 270, 273, 276, 279, 1, 1])
+ else:
+ thr = np.array([279, 276, 273, 270, 267, 264, 1, 1])
+ c = conf_test_dble(bt, thr)
+ print(c)
+
+
+def conf_test(data, band):
+ '''
+ Assuming a linear function between min and max confidence level, the plot below shows
+ how the confidence (y axis) is computed as function of radiance (x axis).
+ This case illustrates alpha < gamma, obviously in case alpha > gamma, the plot would be
+ flipped.
+ gamma
+ c 1 ________
+ o | /
+ n | /
+ f | /
+ i | beta /
+ d 1/2 |....../
+ e | /
+ n | /
+ c | /
+ e 0________/
+ | alpha
+ --------- radiance ---------->
+ '''
+
+ hicut = data.threshold[:, :, 2]
+ beta = data.threshold[:, :, 1]
+ locut = data.threshold[:, :, 0]
+ power = data.threshold[:, :, 3]
+ coeff = np.power(2, (power - 1))
+
+ gamma = data.threshold.where(hicut > locut, data.threshold[:, :, 0])[:, :, 2]
+ alpha = data.threshold.where(hicut > locut, data.threshold[:, :, 2])[:, :, 0]
+ flipped = xr.zeros_like(data[band]).where(hicut > locut, 1)
+
+ # Rad between alpha and beta
+ range_ = 2. * (beta - alpha)
+ s1 = (data[band].values - alpha)/range_
+ conf_tmp1 = (coeff * np.power(s1, power)).where((data[band] <= beta) & (flipped == 0))
+ conf_tmp2 = (1.0 - coeff * np.power(s1, power)).where((data[band] <= beta) & (flipped == 1))
+ conf_tmp12 = conf_tmp1.where(flipped == 0, conf_tmp2)
+
+ # Rad between beta and gamma
+ range_ = 2. * (beta - gamma)
+ s1 = (data[band].values - gamma)/range_
+ conf_tmp3 = (1.0 - coeff * np.power(s1, power)).where((data[band] <= beta) & (flipped == 0))
+ conf_tmp4 = (coeff * np.power(s1, power)).where((data[band] <= beta) & (flipped == 1))
+ conf_tmp34 = conf_tmp3.where(flipped == 0, conf_tmp4)
+
+ confidence = conf_tmp12.where(data[band] <= beta, conf_tmp34)
+
+ confidence = confidence.where(confidence > 0, 0)
+ confidence = confidence.where(confidence < 1, 1)
+
+ return confidence
+
+
+def conf_test_dble(rad, coeffs):
+ # '''
+ # gamma1 gamma2
+ # c 1_______ ________
+ # o | \ /
+ # n | \ /
+ # f | \ /
+ # i | \ beta1 beta2 /
+ # d 1/2 \....| |...../
+ # e | \ /
+ # n | \ /
+ # c | \ /
+ # e 0 \_____________/
+ # | alpha1 alpha2
+ # --------------------- radiance ------------------------->
+ # '''
+
+ coeffs = np.array(coeffs)
+ radshape = rad.shape
+ rad = rad.reshape(np.prod(radshape))
+ confidence = np.zeros(rad.shape)
+
+ alpha1, gamma1 = np.empty(rad.shape), np.empty(rad.shape)
+ alpha2, gamma2 = np.empty(rad.shape), np.empty(rad.shape)
+
+ if coeffs.ndim == 1:
+ coeffs = np.full((rad.shape[0], 7), coeffs[:7]).T
+
+ gamma1 = coeffs[0, :]
+ beta1 = coeffs[1, :]
+ alpha1 = coeffs[2, :]
+ alpha2 = coeffs[3, :]
+ beta2 = coeffs[4, :]
+ gamma2 = coeffs[5, :]
+ power = coeffs[6, :]
+
+ coeff = np.power(2, (power - 1))
+ # radshape = rad.shape
+ # rad = rad.reshape((rad.shape[0]*rad.shape[1]))
+
+ # ## Find if interval between inner cutoffs passes or fails test
+
+ # Inner region fails test
+
+ # Value is within range of lower set of limits
+ range_ = 2. * (beta1 - alpha1)
+ s1 = (rad - alpha1) / range_
+ idx = np.nonzero((rad <= alpha1) & (rad >= beta1) & (alpha1 - gamma1 > 0))
+ confidence[idx] = coeff[idx] * np.power(s1[idx], power[idx])
+
+ range_ = 2. * (beta1 - gamma1)
+ s1 = (rad - gamma1) / range_
+ idx = np.nonzero((rad >= gamma1) & (rad < beta1) & (alpha1 - gamma1 > 0))
+ confidence[idx] = 1.0 - coeff[idx] * np.power(s1[idx], power[idx])
+
+ # Value is within range of upper set of limits
+ range_ = 2. * (beta2 - alpha2)
+ s1 = (rad - alpha2) / range_
+ idx = np.nonzero((rad > alpha1) & (rad <= beta2) & (alpha1 - gamma1 > 0))
+ confidence[idx] = coeff[idx] * np.power(s1[idx], power[idx])
+
+ range_ = 2. * (beta2 - gamma2)
+ s1 = (rad - gamma2) / range_
+ idx = np.nonzero((rad > alpha1) & (rad > beta2) & (alpha1 - gamma1 > 0))
+ confidence[idx] = 1.0 - coeff[idx] * np.power(s1[idx], power[idx])
+
+ # Check for value beyond function range
+ confidence[(alpha1 - gamma1 > 0) & (rad > alpha1) & (rad < alpha2)] = 0
+ confidence[(alpha1 - gamma1 > 0) & ((rad < gamma1) | (rad > gamma2))] = 1
+
+ ###
+
+ # Inner region passes test
+ print("I NEED TO REVIEW THIS TO WRITE IT MORE CLEARLY")
+ # FOR NOW ALPHA AND GAMMA ARE SWITCHED BECAUSE OF HOW THE ARRAYS ARE DEFINED.
+ # THINK ON HOW THIS COULD BE WRITTEN SO THAT IT'S EASIER TO UNDERSTAND (AND DEBUG)
+ # Value is within range of lower set of limits
+ range_ = 2 * (beta1 - alpha1)
+ s1 = (rad - alpha1) / range_
+ idx = np.nonzero((rad > alpha1) & (rad <= gamma1) & (rad <= beta1) & (alpha1 - gamma1 <= 0))
+ confidence[idx] = 1.0 - coeff[idx] * np.power(s1[idx], power[idx])
+
+ range_ = 2 * (beta1 - gamma1)
+ s1 = (rad - gamma1) / range_
+ idx = np.nonzero((rad > alpha1) & (rad <= gamma1) & (rad > beta1) & (alpha1 - gamma1 <= 0))
+ confidence[idx] = coeff[idx] * np.power(s1[idx], power[idx])
+
+ # Values is within range of upper set of limits
+ range_ = 2 * (beta2 - alpha2)
+ s1 = (rad - alpha2) / range_
+ idx = np.nonzero((rad > gamma2) & (rad < alpha2) & (rad >= beta2) & (alpha1 - gamma1 <= 0))
+ confidence[idx] = 1.0 - coeff[idx] * np.power(s1[idx], power[idx])
+
+ range_ = 2 * (beta2 - gamma2)
+ s1 = (rad - gamma2) / range_
+ idx = np.nonzero((rad > gamma2) & (rad < alpha2) & (rad < beta2) & (alpha1 - gamma1 <= 0))
+ confidence[idx] = coeff[idx] * np.power(s1[idx], power[idx])
+
+ confidence[(alpha1 - gamma1 <= 0) & ((rad > gamma1) | (rad < gamma2))] = 0
+ confidence[(alpha1 - gamma1 <= 0) & (rad <= alpha1) & (rad >= alpha2)] = 1
+
+ confidence[confidence > 1] = 1
+ confidence[confidence < 0] = 0
+
+ return confidence
+
+
+if __name__ == "__main__":
+ test_dble()
diff --git a/dev/hires_module.py b/dev/hires_module.py
new file mode 100644
index 0000000..bbca3fa
--- /dev/null
+++ b/dev/hires_module.py
@@ -0,0 +1,228 @@
+import numpy as np
+import xarray as xr
+import netCDF4 as nc
+import sys
+
+from glob import glob
+
+try:
+ from ruamel import yaml as yml
+except ImportError:
+ import ruamel_yaml as yml
+
+import spectral_tests as tst
+import restoral
+import read_data as rd
+import scene as scn
+
+import importlib
+
+importlib.reload(tst)
+
+# #################################################################### #
+# TEST CASE
+# data:
+_datapath = '/ships19/hercules/pveglio/mvcm_viirs_hires'
+_fname_mod02 = ''
+_fname_mod03 = ''
+_fname_img02 = ''
+_fname_img03 = ''
+
+# MVCM output
+_cld_mask = ''
+
+# thresholds:
+_threshold_file = '/home/pveglio/mvcm/thresholds.mvcm.snpp.v0.0.1.yaml'
+
+# ancillary files:
+_geos_atm_1 = 'GEOS.fpit.asm.inst3_2d_asm_Nx.GEOS5124.20220622_1200.V01.nc4'
+_geos_atm_2 = 'GEOS.fpit.asm.inst3_2d_asm_Nx.GEOS5124.20220622_1500.V01.nc4'
+_geos_land = 'GEOS.fpit.asm.tavg1_2d_lnd_Nx.GEOS5124.20220622_1430.V01.nc4'
+_geos_ocean = 'GEOS.fpit.asm.tavg1_2d_ocn_Nx.GEOS5124.20220622_1430.V01.nc4'
+_geos_constants = 'GEOS.fp.asm.const_2d_asm_Nx.00000000_0000.V01.nc4'
+_ndvi_file = 'NDVI.FM.c004.v2.0.WS.00-04.177.hdf'
+_sst_file = 'oisst.20220622'
+_eco_file = 'goge1_2_img.v1'
+
+# #################################################################### #
+
+
+def main(*,
+ data_path: str = _datapath,
+ mod02: str = _fname_mod02,
+ mod03: str = _fname_mod03,
+ img02: str = _fname_img02,
+ img03: str = _fname_img03,
+ threshold_file: str = _threshold_file,
+ geos_atm_1: str = _geos_atm_1,
+ geos_atm_2: str = _geos_atm_2,
+ geos_land: str = _geos_land,
+ geos_ocean: str = _geos_ocean,
+ geos_constants: str = _geos_constants,
+ ndvi_file: str = _ndvi_file,
+ sst_file: str = _sst_file,
+ out_fname: str = 'temp.nc') -> None:
+
+ check = nc.Dataset(mod02)
+ if 'M01' not in list(check['observation_data'].variables):
+ check.close()
+ return
+
+ if check.isopen():
+ check.close()
+
+ print(threshold_file)
+ with open(threshold_file) as f:
+ text = f.read()
+ thresholds = yml.safe_load(text)
+
+ file_names = {'MOD02': f'{data_path}/{mod02}',
+ 'MOD03': f'{data_path}/{mod03}',
+ 'IMG02': f'{data_path}/{img02}',
+ 'IMG03': f'{data_path}/{img03}',
+ 'GEOS_atm_1': f'{geos_atm_1}',
+ 'GEOS_atm_2': f'{geos_atm_2}',
+ 'GEOS_land': f'{geos_land}',
+ 'GEOS_ocean': f'{geos_ocean}',
+ 'GEOS_constants': f'{geos_constants}',
+ 'NDVI': f'{ndvi_file}',
+ 'SST': f'{sst_file}',
+ 'ANC_DIR': f'{data_path}/ancillary'
+ }
+
+ sunglint_angle = thresholds['Sun_Glint']['bounds'][3]
+
+ viirs_data = rd.get_data(file_names, sunglint_angle, hires=True)
+ if 'I01' not in viirs_data:
+ return
+
+# cm_flag = np.array((np.array(cm[0, :, :], np.byte) &
+# (pow(2, 1) + pow(2, 2)))/pow(2, 1))
+#
+# mvcm_conf = np.ones(cm_flag.shape)
+# # conf[cm_flag == 3] = 1
+# mvcm_conf[cm_flag == 2] = 0.95
+# mvcm_conf[cm_flag == 1] = 0.66
+# mvcm_conf[cm_flag == 0] = 0
+
+ viirs_lores = rd.read_data('viirs', mod02, data_path + '/' + mod03)
+
+ viirs_data['M02'] = (('number_of_lines_2', 'number_of_pixels_2'),
+ viirs_lores.M02.values.repeat(2, 0).repeat(2, 1))
+ # viirs_data['M07'] = (('number_of_lines_2', 'number_of_pixels_2'),
+ # viirs_lores.M07.values.repeat(2, 0).repeat(2, 1))
+ viirs_data['M01'] = (('number_of_lines_2', 'number_of_pixels_2'), np.zeros(viirs_data.I01.shape))
+ viirs_data['M07'] = viirs_data.I02
+ viirs_data['M10'] = viirs_data.I03
+ viirs_data['M12'] = viirs_data.I04
+ viirs_data['M14'] = (('number_of_lines_2', 'number_of_pixels_2'),
+ viirs_lores.M14.values.repeat(2, 0).repeat(2, 1))
+ viirs_data['M15'] = (('number_of_lines_2', 'number_of_pixels_2'),
+ viirs_lores.M15.values.repeat(2, 0).repeat(2, 1))
+ viirs_data['M16'] = (('number_of_lines_2', 'number_of_pixels_2'),
+ viirs_lores.M16.values.repeat(2, 0).repeat(2, 1))
+ viirs_data['M15-M16'] = (('number_of_lines_2', 'number_of_pixels_2'),
+ viirs_data.M15.values - viirs_data.M16.values)
+ viirs_data['M14-M15'] = (('number_of_lines_2', 'number_of_pixels_2'),
+ viirs_data.M14.values - viirs_data.M15.values)
+
+ cmin_G1 = np.ones(viirs_data.M15.shape)
+ cmin_G2 = np.ones(viirs_data.M15.shape)
+ cmin_G3 = np.ones(viirs_data.M15.shape)
+
+ MyScene = tst.CloudTests(viirs_data, 'Ocean_Day', thresholds)
+
+ cmin_G1, bit1 = MyScene.test_11um('I05', cmin_G1, test_name='11um_Test')
+ cmin_G1, bit2 = MyScene.sst_test('M15', 'M16', cmin_G1, test_name='SST_Test')
+
+ cmin_G2, bit4 = MyScene.test_11_12um_diff('M15-M16', cmin_G2, test_name='11-12um_Cirrus_Test')
+ cmin_G2, bit3 = MyScene.oceanic_stratus_11_4um_test('I05-I04', cmin_G2,
+ test_name='11-4um_Oceanic_Stratus_Test')
+ # cmin_G2, bit_x = MyScene.bt_diff_86_11um('M14-M15', cmin_G2, test_name='8.6-11um_Test')
+
+ cmin_G3, bit4_1 = MyScene.nir_reflectance_test('M07', cmin_G3, test_name='NIR_Reflectance_Test')
+ cmin_G3, bit5 = MyScene.vis_nir_ratio_test('I02-I01ratio', cmin_G3, test_name='Vis/NIR_Ratio_Test')
+ # visnir, bit5 = MyScene.vis_nir_ratio_test('I02-I01ratio', np.ones(viirs_data.M15.shape), test_name='Vis/NIR_Ratio_Test')
+ cmin_G3, bit6 = MyScene.nir_reflectance_test('M10', cmin_G3, test_name='1.6_2.1um_NIR_Reflectance_Test')
+
+ confidence = np.power(cmin_G1 * cmin_G2 * cmin_G3, 1/3)
+ temp_confidence = np.power(cmin_G1 * cmin_G2 * cmin_G3, 1/3)
+ total_bit = np.zeros(confidence.shape)
+ total_bit = bit1 + bit2 + bit4
+
+ sunglint_angle = thresholds['Sun_Glint']['bounds'][3]
+ scene_flags = scn.find_scene(viirs_data, sunglint_angle)
+
+ idx = np.nonzero((scene_flags['water'] == 1) & (scene_flags['ice'] == 0) &
+ (scene_flags['uniform'] == 1) & (confidence <= 0.99) & (confidence >= 0.05))
+ confidence[idx] = restoral.spatial(viirs_data, thresholds['Sun_Glint'], scene_flags, temp_confidence)[idx]
+
+ temp_confidence = confidence + 0
+ idx = np.nonzero((scene_flags['water'] == 1) & (scene_flags['sunglint'] == 1) &
+ (scene_flags['uniform'] == 1) & (confidence <= 0.95))
+ confidence[idx] = restoral.sunglint(viirs_data, thresholds['Sun_Glint'], total_bit, temp_confidence)[idx]
+
+ # total_confidence = np.sqrt(confidence * mvcm_conf.repeat(2, 0).repeat(2, 1))
+
+ date = mod02.split('.')[1]
+ time = mod02.split('.')[2]
+
+# mvcm_confidence = mvcm_conf.repeat(2, 0).repeat(2, 1)
+ out_xr = xr.Dataset(
+ data_vars=dict(
+ hires_confidence=(['number_of_lines', 'number_of_pixels'], confidence),
+ ocean_day_mask=(['number_of_lines', 'number_of_pixels'], viirs_data.Ocean_Day.values),
+ ),
+ coords=dict(
+ latitude=(['number_of_lines', 'number_of_pixels'], viirs_data.latitude.values),
+ longitude=(['number_of_lines', 'number_of_pixels'], viirs_data.longitude.values),
+ )
+ )
+ out_path = '/ships19/hercules/pveglio/mvcm_viirs_hires/outputs'
+ out_xr.to_netcdf(f'{out_path}/hires_MVCM.{date}.{time}.nc')
+ # out_xr.to_netcdf(out_fname)
+
+# # np.savez(f'{data_path}/pyMVCM_{date}.{time}',
+# np.savez(f'{data_path}/outputs/pyMVCM_{date}.{time}',
+# confidence=total_confidence, py_conf=confidence, mvcm_conf=mvcm_conf,
+# lat=viirs_data.latitude.values, lon=viirs_data.longitude.values,
+# ocean_day_mask=viirs_data.Ocean_Day.values)
+# # c1=cmin_G1, c2=cmin_G2, c3=cmin_G3)
+
+ # return confidence # , conf, total_bit
+
+
+def read_bits(byte, bits):
+ # orig_shape = byte.shape
+ # if byte.ndim > 1:
+ # byte = byte.reshape((byte.shape[0]*byte.shape[1],))
+
+ if type(bits) is int:
+ flag = np.array((np.array(byte, np.byte) & pow(2, bits))/pow(2, bits),
+ dtype='bool')
+ else:
+ flag = (np.array(byte, np.byte) & pow(2, bits[0]) +
+ pow(2, bits[1]))/pow(2, bits[0])
+
+ flag = np.array(np.trunc(1.0-flag/2.0), dtype='bool')
+ # flag = flag.reshape((orig_shape))
+ return flag
+
+
+if __name__ == "__main__":
+ main(data_path=sys.argv[1],
+ mod02=sys.argv[2],
+ mod03=sys.argv[3],
+ img02=sys.argv[4],
+ img03=sys.argv[5],
+ threshold_file=sys.argv[6],
+ geos_atm_1=sys.argv[7],
+ geos_atm_2=sys.argv[8],
+ geos_land=sys.argv[9],
+ geos_ocean=sys.argv[10],
+ geos_constants=sys.argv[11],
+ ndvi_file=sys.argv[12],
+ sst_file=sys.argv[13],
+ out_fname=sys.argv[14])
+
+
diff --git a/dev/main_hires.py b/dev/main_hires.py
new file mode 100644
index 0000000..e1f421f
--- /dev/null
+++ b/dev/main_hires.py
@@ -0,0 +1,125 @@
+import ruamel_yaml as yml
+import numpy as np
+# import xarray as xr
+
+from glob import glob
+
+import read_data as rd
+import scene as scn
+# import tests
+import ocean_day_tests as odt
+import restoral
+
+# #################################################################### #
+# TEST CASE
+# data:
+_datapath = '/ships19/hercules/pveglio/mvcm_viirs_hires'
+_fname_mod02 = glob(f'{_datapath}/VNP02MOD.A2022173.1454.001.*.uwssec_bowtie_restored.nc')[0]
+_fname_mod03 = glob(f'{_datapath}/VNP03MOD.A2022173.1454.001.*.uwssec.nc')[0]
+_fname_img02 = glob(f'{_datapath}/VNP02IMG.A2022173.1454.001.*.uwssec_bowtie_restored.nc')[0]
+_fname_img03 = glob(f'{_datapath}/VNP03IMG.A2022173.1454.001.*.uwssec.nc')[0]
+
+# thresholds:
+_threshold_file = '/home/pveglio/mvcm_leo/thresholds/new_thresholds.mvcm.snpp.v1.0.0.yaml'
+
+# ancillary files:
+_geos_atm_1 = 'GEOS.fpit.asm.inst3_2d_asm_Nx.GEOS5124.20220622_1200.V01.nc4'
+_geos_atm_2 = 'GEOS.fpit.asm.inst3_2d_asm_Nx.GEOS5124.20220622_1500.V01.nc4'
+_geos_land = 'GEOS.fpit.asm.tavg1_2d_lnd_Nx.GEOS5124.20220622_1430.V01.nc4'
+_geos_ocean = 'GEOS.fpit.asm.tavg1_2d_ocn_Nx.GEOS5124.20220622_1430.V01.nc4'
+_geos_constants = 'GEOS.fp.asm.const_2d_asm_Nx.00000000_0000.V01.nc4'
+_ndvi_file = 'NDVI.FM.c004.v2.0.WS.00-04.177.hdf'
+_sst_file = 'oisst.20220622'
+_eco_file = 'goge1_2_img.v1'
+
+mod_bands = ['M01', 'M02', 'M03', 'M04', 'M06', 'M08', 'M09', 'M11', 'M13', 'M14', 'M15', 'M16']
+
+# #################################################################### #
+
+
+def main(*, data_path=_datapath, mod02=_fname_mod02, mod03=_fname_mod03,
+ img02=_fname_img02, img03=_fname_img03, threshold_file=_threshold_file,
+ geos_atm_1=_geos_atm_1, geos_atm_2=_geos_atm_2, geos_land=_geos_land,
+ geos_ocean=_geos_ocean, geos_constants=_geos_constants, ndvi_file=_ndvi_file, sst_file=_sst_file):
+
+ # datapath = '/ships19/hercules/pveglio/neige_data/snpp_test_input'
+ # fname_l1b = 'VNP02MOD.A2014213.1548.001.2017301015346.uwssec.bowtie_restored_scaled.nc'
+ # fname_geo = 'VNP03MOD.A2014213.1548.001.2017301015705.uwssec.nc'
+ # thresh_file = '/home/pveglio/mvcm_leo/thresholds/new_thresholds.mvcm.snpp.v1.0.0.yaml'
+
+ ancillary_file_names = {'GEOS_atm_1': f'{geos_atm_1}',
+ 'GEOS_atm_2': f'{geos_atm_2}',
+ 'GEOS_land': f'{geos_land}',
+ 'GEOS_ocean': f'{geos_ocean}',
+ 'GEOS_constants': f'{geos_constants}',
+ 'NDVI': f'{ndvi_file}',
+ 'SST': f'{sst_file}',
+ 'ANC_DIR': f'{data_path}/ancillary'
+ }
+
+ viirs_data = rd.read_data('viirs', f'{mod02}', f'{mod03}')
+ viirs_hires = rd.read_data('viirs', f'{img02}', f'{img03}')
+
+ for bnd in mod_bands:
+ viirs_hires[bnd] = (['number_of_lines', 'number_of_pixels'], viirs_data[bnd].values.repeat(2, 0).repeat(2, 1))
+ viirs_hires = viirs_hires.rename({'I01': 'M05', 'I02': 'M07', 'I03': 'M10', 'I04': 'M12'})
+ print('VIIRS data read')
+
+ viirs_hires = rd.read_ancillary_data(ancillary_file_names, viirs_hires, resolution=2)
+ print('ancillary data read')
+
+ viirs_data = viirs_hires
+
+ with open(threshold_file) as f:
+ text = f.read()
+ thresholds = yml.safe_load(text)
+
+ sunglint_angle = thresholds['Sun_Glint']['bounds'][3]
+ scene_flags = scn.find_scene(viirs_data, sunglint_angle)
+
+ cmin1 = np.ones(viirs_data.M01.shape)
+ cmin2 = np.ones(viirs_data.M01.shape)
+ cmin3 = np.ones(viirs_data.M01.shape)
+ cmin4 = np.ones(viirs_data.M01.shape)
+
+ c = np.ones((9, viirs_data['M01'].shape[0], viirs_data['M01'].shape[1]))
+ cmin1, c[0, :, :], bit1 = odt.simple_test(viirs_data.M15.values, thresholds['Daytime_Ocean']['bt11'], cmin1)
+ # For this test I changed the sst to [7.000, 6.500, 6.000, 1.0, 1.0]
+ cmin1, c[1, :, :], bit2 = odt.sst_test(viirs_data.M15.values, viirs_data.M16.values,
+ viirs_data.sensor_zenith.values, viirs_data.geos_sfct.values,
+ thresholds['Daytime_Ocean']['sst'], cmin1)
+ cmin2, c[2, :, :], bit3 = odt.simple_test(viirs_data.M14.values-viirs_data.M15.values,
+ thresholds['Daytime_Ocean']['diff_11_86um'], cmin2)
+ cmin2, c[3, :, :], bit4 = odt.test_11_12_diff(viirs_data, thresholds['Daytime_Ocean']['diff11_12um'], cmin2)
+ cmin2, c[4, :, :] = odt.test_11_4_diff(viirs_data.I05.values, viirs_data.M12.values,
+ thresholds['Daytime_Ocean']['test11_4lo'], scene_flags, cmin2)
+ cmin3, c[5, :, :] = odt.nir_refl_test(viirs_data.M07.values, thresholds['Daytime_Ocean'],
+ thresholds['Sun_Glint'], viirs_data, cmin3)
+ cmin3, c[6, :, :] = odt.vis_nir_ratio_test(viirs_data.M05.values, viirs_data.M07.values,
+ thresholds, scene_flags, cmin3)
+ cmin3, c[7, :, :] = odt.nir_refl_test(viirs_data.M10.values, thresholds['Daytime_Ocean'],
+ thresholds['Sun_Glint'], viirs_data, cmin3)
+
+ total_bit = bit1 + bit2 + bit4
+ temp_confidence = cmin1 * cmin2 * cmin3 * cmin4
+ confidence = cmin1 * cmin2 * cmin3 * cmin4
+ # idx = np.nonzero((scene_flags['water'] == 1) & (scene_flags['ice'] == 0) & (scene_flags['uniform'] == 1) &
+ # (confidence <= 0.99) & (confidence >= 0.05))
+ # confidence[idx] = restoral.spatial(viirs_data, thresholds['Sun_Glint'], scene_flags, confidence)[idx]
+
+ idx = np.nonzero((scene_flags['water'] == 1) & (scene_flags['sunglint'] == 1) &
+ (scene_flags['uniform'] == 1) & (confidence <= 0.95))
+ confidence[idx] = restoral.sunglint(viirs_data, thresholds['Sun_Glint'], total_bit, temp_confidence)[idx]
+
+ temp = np.zeros((viirs_data.M01.shape[0], viirs_data.M01.shape[1]))
+ temp[idx] = 1
+ c[8, :, :] = temp
+
+ np.savez('test_confidence', confidence=confidence, conf_test=c,
+ lat=viirs_data.latitude.values, lon=viirs_data.longitude.values)
+
+ return confidence
+
+
+if __name__ == "__main__":
+ main()
diff --git a/dev/ocean_day_tests.py b/dev/ocean_day_tests.py
new file mode 100644
index 0000000..6bd8d2c
--- /dev/null
+++ b/dev/ocean_day_tests.py
@@ -0,0 +1,338 @@
+import numpy as np
+
+from numpy.lib.stride_tricks import sliding_window_view
+
+import utils
+import conf
+
+import ancillary_data as c_tools
+
+
+# ############## GROUP 1 TESTS ############## #
+
+# 11 micron brightness temperature threshold test
+def simple_test(rad, threshold, cmin):
+
+ radshape = rad.shape
+ rad = rad.reshape(np.prod(radshape))
+
+ thr = np.array(threshold)
+ confidence = np.ones(rad.shape)
+ bit = np.zeros(rad.shape)
+
+ if thr[4] == 1:
+ print("simple test running")
+ # the C code has the line below that I don't quite understand the purpose of.
+ # It seems to be setting the bit to 0 if the BT value is greater than the midpoint
+ #
+ # if (m31 >= dobt11[1]) (void) set_bit(13, pxout.testbits);
+
+ # confidence = utils.conf_test(rad, thr)
+ confidence = conf.conf_test(rad, thr)
+ bit[rad >= thr[1]] = 1
+
+ return np.minimum(cmin, confidence.reshape(radshape)), confidence.reshape(radshape), bit.reshape(radshape)
+
+
+def sst_test(rad1, rad2, vza, surf_temp, threshold, cmin):
+
+ a1 = 1.8860
+ a2 = 0.9380
+ a3 = 0.1280
+ a4 = 1.0940
+
+ radshape = rad1.shape
+ b31 = rad1.reshape(np.prod(radshape)) - 273.16
+ b32 = rad2.reshape(np.prod(radshape)) - 273.16
+
+ thr = np.array(threshold)
+ confidence = np.ones(b31.shape)
+ bit = np.zeros(b31.shape)
+
+ rad_diff = b31 - b32
+ sstc = surf_temp.reshape(np.prod(radshape)) - 273.16
+ mu = np.cos(vza.reshape(np.prod(radshape)) * np.pi/180.0)
+
+ modsst = 273.16 + a1 + a2*b31 + a3*rad_diff*sstc + a4*rad_diff*((1/mu)-1)
+ sfcdif = surf_temp.reshape(np.prod(radshape)) - modsst
+
+ if thr[4] == 1:
+ print('SST test running')
+ confidence = conf.conf_test(sfcdif, thr)
+ bit[sfcdif < thr[1]] = 1
+
+ return np.minimum(cmin, confidence.reshape(radshape)), confidence.reshape(radshape), bit.reshape(radshape)
+
+
+def test_11_12_diff(data, threshold, cmin):
+ radshape = data.M15.shape
+ b31 = data.M15.values.reshape(np.prod(radshape))
+ b32 = data.M15.values.reshape(np.prod(radshape))
+ vza = data.sensor_zenith.values.reshape(np.prod(radshape))
+ thr = np.array(threshold)
+
+ confidence = np.ones(b31.shape)
+ bit = np.zeros(b31.shape)
+ rad_diff = b31 - b32
+
+ # Get secant of viewing zenith angle
+ dtr = np.pi/180
+ cosvza = np.cos(vza * dtr)
+ schi = np.full(cosvza.shape, 99.0)
+ schi[cosvza > 0.0] = 1.0/cosvza[cosvza > 0.0]
+
+ # Need to define this in cython
+ btd_thr = c_tools.py_cithr(1, np.array(schi, dtype=np.float32), np.array(b31, dtype=np.float32))
+ idx = np.nonzero((btd_thr < 0.1) | (np.abs(schi-99.0) < 0.0001))
+ btd_thr[idx] = thr[0]
+ locut = btd_thr + 0.3*btd_thr
+ hicut = btd_thr - 1.25
+ corr_thr = np.array([locut, btd_thr, hicut, np.ones(locut.shape)], dtype=np.float)
+
+ if thr[1] == 1:
+ print('11-12um diff test running')
+ bit[rad_diff < thr[1]] = 1
+ confidence = conf.conf_test(rad_diff, corr_thr)
+
+ return np.minimum(cmin, confidence.reshape(radshape)), confidence.reshape(radshape), bit.reshape(radshape)
+
+
+def test_11_4_diff(rad1, rad2, threshold, scene_flags, cmin):
+ radshape = rad1.shape
+ b31 = rad1.reshape(np.prod(radshape))
+ b20 = rad2.reshape(np.prod(radshape))
+ thr = np.array(threshold)
+ sunglint = scene_flags['sunglint'].reshape(np.prod(radshape))
+
+ confidence = np.ones(b31.shape)
+
+ if thr[4] == 1:
+ print('11-4um diff test running')
+ confidence[sunglint == 0] = conf.conf_test((b31-b20)[sunglint == 0], thr)
+
+ return np.minimum(cmin, confidence.reshape(radshape)), confidence.reshape(radshape)
+
+
+def vis_nir_ratio_test(rad1, rad2, threshold, scene, cmin):
+ if threshold['Daytime_Ocean']['vis_nir_ratio'][6] == 1:
+ print("NIR-Visible ratio test running")
+
+ radshape = rad1.shape
+ rad1 = rad1.reshape(np.prod(radshape))
+ rad2 = rad2.reshape(np.prod(radshape))
+ sunglint = scene['sunglint'].reshape(np.prod(radshape))
+ vrat = rad2/rad1
+
+ confidence = np.ones(rad1.shape)
+ tmp = threshold['Daytime_Ocean']['vis_nir_ratio']
+ thr_no_sunglint = np.array([tmp[0], tmp[1], tmp[2], tmp[3], tmp[4], tmp[5], 1, 1])
+ tmp = threshold['Sun_Glint']['snglnt']
+ thr_sunglint = np.array([tmp[0], tmp[1], tmp[2], tmp[3], tmp[4], tmp[5], 1])
+ # thr_no_sunglint = np.array(threshold['Daytime_Ocean']['vis_nir_ratio'])
+ # thr_sunglint = np.array(threshold['Sun_Glint']['snglnt'])
+ # thr_sunglint = np.append(thr_sunglint, 1)
+ # temp value to avoid linter bitching at me
+ # eventually we would have the test run in two blocks as:
+ # confidence[sunglint == 1] = conf.conf_test_dble(vrat[sunglint == 1], sg_threshold['snglnt'])
+ # confidence[sunglint == 0] = conf.conf_test_dble(vrat[sunglint == 0], threshold['vis_nir_ratio'])
+ # sunglint needs to be defined somewhere
+ # thr = np.full((rad.shape[0], 4), thr[:4]).T
+
+ # thresh = np.full((rad1.shape[0], thr_no_sunglint.shape[0]), thr_no_sunglint)
+ # thresh[sunglint == 1, :6] = thr_sunglint
+
+ confidence[sunglint == 0] = conf.conf_test_dble(vrat, thr_no_sunglint)[sunglint == 0]
+ confidence[sunglint == 1] = conf.conf_test_dble(vrat, thr_sunglint)[sunglint == 1]
+ # confidence = conf.conf_test_dble(vrat, thresh.T)
+
+ return np.minimum(cmin, confidence.reshape(radshape)), confidence.reshape(radshape)
+
+
+def nir_refl_test(rad, threshold, sunglint_thresholds, viirs_data, cmin):
+
+ print("NIR reflectance test running")
+ sza = viirs_data.solar_zenith.values
+ refang = viirs_data.sunglint_angle.values
+ vza = viirs_data.sensor_zenith.values
+ dtr = np.pi/180
+ # Keep in mind that band_n uses MODIS band numbers (i.e. 2=0.86um and 7=2.1um)
+ # For VIIRS would be 2=M07 (0.865um) and 7=M11 (2.25um)
+ band_n = 2
+ vzcpow = 0.75 # THIS NEEDS TO BE READ FROM THE THRESHOLDS FILE
+
+ radshape = rad.shape
+ rad = rad.reshape(np.prod(radshape))
+ confidence = np.ones(rad.shape)
+ sza = sza.reshape(rad.shape)
+ vza = vza.reshape(rad.shape)
+ refang = refang.reshape(rad.shape)
+ sunglint_flag = utils.sunglint_scene(refang, sunglint_thresholds).reshape(rad.shape)
+
+ # ref2 [5]
+ # b2coeffs [4]
+ # b2mid [1]
+ # b2bias_adj [1]
+ # b2lo [1]
+ # vzcpow [3] (in different place)
+
+ cosvza = np.cos(vza*dtr)
+ coeffs = threshold['b2coeffs']
+ hicut0 = np.array(coeffs[0] + coeffs[1]*sza + coeffs[2]*np.power(sza, 2) + coeffs[3]*np.power(sza, 3))
+ hicut0 = (hicut0 * 0.01) + threshold['b2adj']
+ hicut0 = hicut0 * threshold['b2bias_adj']
+ midpt0 = hicut0 + (threshold['b2mid'] * threshold['b2bias_adj'])
+ locut0 = midpt0 + (threshold['b2lo'] * threshold['b2bias_adj'])
+ thr = np.array([locut0, midpt0, hicut0, threshold['ref2'][3]*np.ones(rad.shape)])
+ corr_thr = np.zeros((4, rad.shape[0]))
+
+ corr_thr[:3, sunglint_flag == 0] = thr[:3, sunglint_flag == 0] * (1./np.power(cosvza[sunglint_flag == 0], vzcpow))
+ corr_thr[3, sunglint_flag == 0] = thr[3, sunglint_flag == 0]
+ # corr_thr[:3, :] = thr[:3, :] * (1./np.power(cosvza[:], vzcpow))
+ # corr_thr[3, :] = thr[3, :]
+
+ for flag in range(1, 4):
+ if len(refang[sunglint_flag == flag]) > 0:
+ sunglint_thr = utils.get_sunglint_thresholds(refang, sunglint_thresholds, band_n, flag, thr)
+ corr_thr[:3, sunglint_flag == flag] = sunglint_thr[:3, sunglint_flag == flag] * (1./np.power(cosvza[sunglint_flag == flag], vzcpow))
+ corr_thr[3, sunglint_flag == flag] = sunglint_thr[3, sunglint_flag == flag]
+
+ confidence = conf.conf_test(rad, corr_thr)
+
+ return np.minimum(cmin, confidence.reshape(radshape)), confidence.reshape(radshape)
+
+
+def nir_high_cloud_test():
+ pass
+
+
+
+
+
+
+
+
+
+
+
+def test_11um_var(rad, threshold, var_threshold):
+
+ print("11um variability test running")
+ thr = np.array(threshold['11um_var'])
+
+ radshape = rad.shape
+ var = np.zeros((radshape[0], radshape[1], 9))
+
+ # chk_spatial2() need to figure out what this is
+ # np = rg_var.num_small_diffs * 1.0
+ test = sliding_window_view(np.pad(rad, [1, 1], mode='constant'), (3, 3)) - np.expand_dims(rad, (2, 3))
+
+ var[np.abs(test).reshape(radshape[0], radshape[1], 9) < var_threshold['dovar11']] = 1
+ var = var.sum(axis=2).reshape(np.prod(radshape))
+
+ rad = rad.reshape(np.prod(radshape))
+ confidence = np.zeros(rad.shape)
+
+ confidence[var == 9] = conf.conf_test(rad[var == 9], thr)
+
+ return confidence.reshape(radshape)
+
+
+def test_11_4diff(rad1, rad2, threshold, viirs_data, sg_thresh):
+
+ print("11um - 4um difference test running")
+ radshape = rad1.shape
+ raddiff = (rad1 - rad2).reshape(np.prod(radshape))
+
+ day = np.zeros(radshape)
+ day[viirs_data.solar_zenith <= 85] = 1
+ day = day.reshape(raddiff.shape)
+ sunglint = np.zeros(rad1.shape)
+ sunglint[viirs_data.sunglint_angle <= sg_thresh] = 1
+ sunglint = sunglint.reshape(raddiff.shape)
+ thr = np.array(threshold['test11_4lo'])
+ confidence = np.zeros(raddiff.shape)
+
+ # confidence[(day == 1) & (sunglint == 0)] = utils.conf_test(raddiff[(day == 1) & (sunglint == 0)], thr)
+ confidence[(day == 1) & (sunglint == 0)] = conf.conf_test(raddiff[(day == 1) & (sunglint == 0)], thr)
+
+ return confidence.reshape(radshape)
+
+
+def vir_refl_test(rad, threshold, viirs_data):
+
+ print('Visible reflectance test running')
+
+ thr = threshold['vis_refl_test']
+
+ radshape = rad.shape()
+ rad = rad.reshape(np.prod(radshape))
+ confidence = np.zeros(radshape)
+ vzcpow = 0.75 # THIS NEEDS TO BE READ FROM THE THRESHOLDS FILE
+
+ vza = viirs_data.sensor_zenith.values
+ dtr = np.pi/180
+ cosvza = np.cos(vza*dtr)
+
+ coeffs = utils.get_b1_thresholds()
+ coeffs[:, :3] = coeffs[:, :3] * threshold['b1_bias_adj']
+
+ # this quantity is the return of get_b1_thresholds() in the C code
+ # it's defined here to keep a consistent logic with the original source, for now
+ irtn = 0
+
+ if irtn != 0:
+ coeffs = thr
+
+ coeffs[:, :3] = coeffs[:, :3] * 1/np.power(cosvza, vzcpow)
+
+ confidence = conf.conf_test(rad, coeffs)
+
+ return confidence.reshape(radshape)
+
+
+
+class CloudMaskTests(object):
+
+ def __init__(self, scene, radiance, coefficients):
+ self.scene = scene
+ self.coefficients = coefficients
+
+ def select_coefficients(self):
+ pass
+
+ def test_G1(self):
+ pass
+
+ def test_G2(self):
+ pass
+
+ def test_G3(self):
+ pass
+
+ def test_G4(self):
+ pass
+
+ def overall_confidence(self):
+ pass
+
+
+def test():
+ rad = np.random.randint(50, size=[4, 8])
+ # coeffs = [5, 42, 20, 28, 15, 35, 1]
+ # coeffs = [20, 28, 5, 42, 15, 35, 1]
+ coeffs = [35, 15, 20, 1, 1]
+ # confidence = conf_test_dble(rad, coeffs)
+ confidence = test_11um(rad, coeffs)
+ print(rad)
+ print('\n')
+ print(confidence)
+
+
+if __name__ == "__main__":
+ test()
+
+
+
+
+
+
diff --git a/dev/plot_data.py b/dev/plot_data.py
new file mode 100644
index 0000000..31358cd
--- /dev/null
+++ b/dev/plot_data.py
@@ -0,0 +1,161 @@
+import matplotlib.pyplot as plt
+import matplotlib
+import cartopy.crs as ccrs
+import numpy as np
+import netCDF4 as nc
+
+import sys
+import os
+
+from scene import test_scene
+
+matplotlib.use('Agg')
+
+
+def main(scene_flag):
+ scn = test_scene()
+ lat = scn['lat']
+ lon = scn['lon']
+
+ plt.figure(figsize=[15, 10])
+ ax = plt.axes(projection=ccrs.PlateCarree())
+ ax.set_extent([np.min(lon)-15, np.max(lon)+15,
+ np.min(lat)-15, np.max(lat)+15], crs=ccrs.PlateCarree())
+
+ ax.coastlines(resolution='50m', color='black', linewidth=1)
+
+ plt.pcolormesh(lon, lat, scn[scene_flag],
+ transform=ccrs.PlateCarree())
+
+ plt.show()
+
+
+def plot_tests(numpy_file='test_confidence.npz'):
+ data = np.load(numpy_file)
+ lat = data['lat']
+ lon = data['lon']
+ # confidence = data['confidence']
+ conf_test = data['conf_test']
+ confidence = data['confidence']
+ cm = np.zeros(confidence.shape)
+ cm[confidence > 0.66] = 1
+ cm[confidence > 0.95] = 2
+ cm[confidence > 0.99] = 3
+
+ for i in range(9):
+ print(f'Making plot {i}\n')
+ plt.figure(figsize=[15, 10])
+ ax = plt.axes(projection=ccrs.PlateCarree())
+ ax.set_extent([np.min(lon)-5, np.max(lon)+5,
+ np.min(lat)-5, np.max(lat)+5], crs=ccrs.PlateCarree())
+
+ ax.coastlines(resolution='50m', color='black', linewidth=1)
+
+ plt.pcolormesh(lon, lat, conf_test[i, :, :], vmin=0, vmax=1,
+ transform=ccrs.PlateCarree())
+ plt.title(f'Confidence {i}')
+ plt.colorbar()
+
+ plt.savefig(f'figures/confplot_{i}.png')
+
+ plt.figure(figsize=[15, 10])
+ ax = plt.axes(projection=ccrs.PlateCarree())
+ ax.set_extent([np.min(lon)-5, np.max(lon)+5,
+ np.min(lat)-5, np.max(lat)+5], crs=ccrs.PlateCarree())
+
+ ax.coastlines(resolution='50m', color='black', linewidth=1)
+
+ plt.pcolormesh(lon, lat, cm, vmin=0, vmax=3,
+ transform=ccrs.PlateCarree())
+ plt.title('Cloud Mask')
+ # plt.colorbar()
+ cb = plt.colorbar(ticks=range(4))
+ cb.set_ticklabels(['confident cloudy', 'probably cloudy', 'probably clear', 'confident clear'])
+
+ plt.savefig('figures/total_cm.png')
+
+
+def plot_confidence(numpy_file='test_confidence.npz'):
+ data = np.load(numpy_file)
+ lat = data['lat']
+ lon = data['lon']
+ # confidence = data['confidence']
+ confidence = data['confidence']
+ cm = np.zeros(confidence.shape)
+ cm[confidence > 0.66] = 1
+ cm[confidence > 0.95] = 2
+ cm[confidence > 0.99] = 3
+
+ plt.figure(figsize=[15, 10])
+ ax = plt.axes(projection=ccrs.PlateCarree())
+ ax.set_extent([np.min(lon)-5, np.max(lon)+5,
+ np.min(lat)-5, np.max(lat)+5], crs=ccrs.PlateCarree())
+
+ ax.coastlines(resolution='50m', color='black', linewidth=1)
+
+ plt.pcolormesh(lon, lat, cm, vmin=0, vmax=3,
+ transform=ccrs.PlateCarree())
+ # plt.title('Confidence')
+ # plt.colorbar()
+
+ plt.title('Cloud Mask')
+ cb = plt.colorbar(ticks=range(4))
+ cb.set_ticklabels(['confident cloudy', 'probably cloudy', 'probably clear', 'confident clear'])
+ plt.savefig('figures/python_cloudmask.png', dpi=300)
+
+
+def plot_cloud_mask():
+ # f = nc.Dataset('/ships19/hercules/pveglio/mvcm_viirs_hires/CLDMSK_L2_VIIRS_SNPP.A2022173.1454.001.2022174035130.nc')
+ f = nc.Dataset('/ships19/hercules/pveglio/mvcm_viirs_hires/outputs/CLDMSK_L2_MODIS_SNPP.A2022173.1454.001_viirs_cris.nc')
+ lat = f['geolocation_data/latitude'][:]
+ lon = f['geolocation_data/longitude'][:]
+ # icm = f['geophysical_data/Integer_Cloud_Mask'][:]
+ cm = f['geophysical_data/Cloud_Mask'][:]
+ qa = f['geophysical_data/Quality_Assurance'][:]
+ sst = f['geolocation_data/SST'][:]
+ f.close()
+
+ # cm_flag = np.array((np.array(cm[0, :, :], np.byte) &
+ # (pow(2, 1) + pow(2, 2)))/pow(2, 1))
+ # cm_flag = read_bits(qa[:, :, 2], 2) & ~read_bits(cm[2, :, :], 2)
+ cm_flag = read_bits(qa[:, :, 2], 0) & ~read_bits(cm[2, :, :], 0)
+
+ plt.figure(figsize=[15, 10])
+ ax = plt.axes(projection=ccrs.PlateCarree())
+ ax.set_extent([np.min(lon)-5, np.max(lon)+5,
+ np.min(lat)-5, np.max(lat)+5], crs=ccrs.PlateCarree())
+
+ ax.coastlines(resolution='50m', color='black', linewidth=1)
+
+ plt.pcolormesh(lon, lat, cm_flag, vmin=0, vmax=1,
+ transform=ccrs.PlateCarree())
+ # cb = plt.colorbar(ticks=range(4))
+ # cb.set_ticklabels(['confident cloudy', 'probably cloudy', 'probably clear', 'confident clear'])
+ cb = plt.colorbar(ticks=range(2))
+ cb.set_ticklabels(['No', 'Yes'])
+ # plt.savefig('figures/ref_cloudmask.png', dpi=300)
+ plt.savefig('figures/1_38um_high_cloud_test.png', dpi=300)
+
+
+def read_bits(byte, bits):
+ # orig_shape = byte.shape
+ # if byte.ndim > 1:
+ # byte = byte.reshape((byte.shape[0]*byte.shape[1],))
+
+ if type(bits) is int:
+ flag = np.array((np.array(byte, np.byte) & pow(2, bits))/pow(2, bits),
+ dtype='bool')
+ else:
+ flag = (np.array(byte, np.byte) & pow(2, bits[0]) +
+ pow(2, bits[1]))/pow(2, bits[0])
+
+ flag = np.array(np.trunc(1.0-flag/2.0), dtype='bool')
+ # flag = flag.reshape((orig_shape))
+ return flag
+
+
+if __name__ == "__main__":
+ # main(sys.argv[1])
+ # plot_tests(sys.argv[1])
+ plot_confidence(sys.argv[1])
+ # plot_cloud_mask()
diff --git a/dev/plot_mask.py b/dev/plot_mask.py
new file mode 100644
index 0000000..ff56476
--- /dev/null
+++ b/dev/plot_mask.py
@@ -0,0 +1,295 @@
+import sys
+
+import numpy as np
+import matplotlib.pyplot as plt
+import matplotlib.image as mpimg
+
+#import cartopy.crs as ccrs
+
+from matplotlib import colors
+from netCDF4 import Dataset
+from numpy import less, greater
+
+import argparse
+
+_map_names = ['bit0', 'cloudy', 'cloud_mask_value', 'day', 'sunglint', 'snow_ice',
+ 'land', 'thin_cirrus_solar', 'snow_covered_from_anc', 'thin_cirrus_IR',
+ 'cloudy_plus_1_pixel_adj', 'ocean_BT_threshold', 'high_cloud_flag_138',
+ 'high_cloud_flag_39_12', 'high_cloud_flag_11_12',
+ 'cloud_flag_39_11_BTD_test', 'cloud_flag_VisNIR_test',
+ 'cloud_flag_VisNIR_ratio', 'shallow_water_clear_sky_restoral',
+ 'cloud_flag_16_21_test', 'high_cloud_flag_86_11_BTD_test',
+ 'ocean_clear_sky_restoral_test', 'land_sunglint_polar_clearsky_restoral',
+ 'cloud_flag_sfc_temp_test', 'cloud_flag_11um_var_test',
+ 'low_cloud_flag_39_11BTD_test', 'simple_IR_test',
+ 'high_cloud_CO2_test', 'high_cloud_067um_test',
+ 'cloud_vis_reflectance_test', 'cloud_near_IR_reflectance_test',
+ 'vza_mask_00', 'vza_mask_10', 'vza_mask_30', 'vza_mask_50', 'sct_mask_70',
+ 'sct_mask_90', 'sct_mask_110', 'sct_mask_130', 'sct_mask_150',
+ 'sct_mask_170', 'sct_mask_180']
+
+
+_map_names = ['cloud_mask_value']
+
+def cmap_discretize(cmap, N, N2=1024):
+ """Return a discrete colormap from the continuous colormap cmap.
+
+ cmap: colormap instance, eg. cm.jet.
+ N: number of colors.
+
+ Example
+ x = resize(arange(100), (5,100))
+ djet = cmap_discretize(cm.jet, 5)
+ imshow(x, cmap=djet)
+ """
+
+ if type(cmap) == str:
+ cmap = cm.get_cmap(cmap)
+
+ colors_i = np.concatenate((np.linspace(0, 1., N), (0., 0., 0., 0.,)))
+ colors_rgba = cmap(colors_i)
+ indices = np.linspace(0, 1., N+1)
+ cdict = {}
+ for ki, key in enumerate(('red', 'green', 'blue')):
+ cdict[key] = [(indices[i], colors_rgba[i-1, ki], colors_rgba[i, ki])
+ for i in range(N+1)]
+
+ return colors.LinearSegmentedColormap(cmap.name + '_%d' % N, cdict, N2)
+
+
+def main(fname, test_name='',
+ c1='mediumblue', c2='skyblue', c3='yellowgreen', c4='olivedrab'):
+
+ #data = read_data(fname, 18, 21, 102, 107)
+ data = read_data(fname, -90, 90, -180, 180)
+ masks = create_masks(data)
+
+ for test_name in _map_names:
+ print(test_name)
+ x = masks[test_name]
+
+ zz = np.zeros((x.shape[0], x.shape[1], 4), dtype=float)
+
+ ind = np.where(x == 0)
+ zz[ind[0], ind[1], :] = np.array(colors.to_rgba(c1))
+
+ ind = np.where(x == 1)
+ zz[ind[0], ind[1], :] = np.array(colors.to_rgba(c2))
+
+ ind = np.where(x == 2)
+ zz[ind[0], ind[1], :] = np.array(colors.to_rgba(c3))
+
+ ind = np.where(x == 3)
+ zz[ind[0], ind[1], :] = np.array(colors.to_rgba(c4))
+
+ mpimg.imsave(f'{test_name}.png', np.uint8(zz*255))
+
+
+def read_data(filename, lat1, lat2, lon1, lon2):
+ f = Dataset(filename)
+ lat = f['geolocation_data/latitude'][:]
+ lon = f['geolocation_data/longitude'][:]
+ ind = np.nonzero(np.bitwise_and(
+ np.bitwise_and(lat>lat1, lat<lat2),
+ np.bitwise_and(lon>lon1, lon<lon2) ))
+ x1, x2, y1, y2 = np.min(ind[0]), np.max(ind[0]), np.min(ind[1]), np.max(ind[1])
+ qa = f['geophysical_data/Quality_Assurance'][x1:x2, y1:y2, :]
+ clearsky_conf = f['geophysical_data/Clear_Sky_Confidence'][x1:x2, y1:y2]
+ cm = f['geophysical_data/Cloud_Mask'][:, x1:x2, y1:y2]
+ vza = f['geolocation_data/sensor_zenith'][x1:x2, y1:y2]
+ sza = f['geolocation_data/solar_zenith'][x1:x2, y1:y2]
+ senaz = f['geolocation_data/sensor_azimuth'][x1:x2, y1:y2]
+ solaz = f['geolocation_data/solar_azimuth'][x1:x2, y1:y2]
+
+ f.close()
+
+ data = {'QA': qa, 'clear_sky_confidence': clearsky_conf,
+ 'cloud_mask': cm, 'latitude': lat, 'longitude': lon,
+ 'sensor_zenith': vza, 'solar_zenith': sza,
+ 'sensor_azimuth': senaz, 'solar_azimuth': solaz}
+
+ return data
+
+
+def create_masks(data):
+ # 2
+ dtr = 3.14159 / 180.0
+ rtd = 180.0 / 3.14159
+ qa = data['QA']
+ cm = data['cloud_mask']
+ vza = data['sensor_zenith']
+ sza = data['solar_zenith']
+ senaz = data['sensor_azimuth']
+ solaz = data['solar_azimuth']
+
+ array_mask = cm[0, :, :].mask
+
+ bit0 = read_bits(cm[0, :, :], 0)
+ # clear_sky_confidence = 0
+
+ # 3
+ cloudy = read_bits(cm[0, :, :], [1, 2])
+# cloudy = np.array((np.array(cm[0, :, :], np.byte) &
+# (pow(2, 1) + pow(2, 2))) / pow(2, 1))
+ cm_flag = np.array((np.array(cm[0, :, :], np.byte) &
+ (pow(2, 1) + pow(2, 2)))/pow(2, 1))
+
+ # 4
+ day = read_bits(cm[0, :, :], 3)
+ sunglint = ~read_bits(cm[0, :, :], 4)
+ snow_ice = ~read_bits(cm[0, :, :], 5)
+# land = read_bits(cm[0, :, :], [6, 7])
+ only_land = np.array((np.array(cm[0, :, :], np.byte) &
+ (pow(2, 6) + pow(2, 7)))/pow(2, 6))
+ only_land = np.array(only_land, dtype='bool') + 0
+
+ # only CM additional info
+ simpleIRtest = read_bits(cm[1, :, :], 5);
+ hicld_CO2 = read_bits(cm[1, :, :], 6);
+ hicld_67um = read_bits(cm[1, :, :], 7);
+ cld_visrefl = read_bits(cm[2, :, :], 4);
+ cld_nearIR = read_bits(cm[2, :, :], 6);
+ # 5
+ thin_cirrus_solar = read_bits(qa[:, :, 1], 1) & ~read_bits(cm[1, :, :], 1)
+ snow_cover = read_bits(qa[:, :, 1], 2) & ~read_bits(cm[1, :, :], 2)
+ thin_cirrus_IR = read_bits(qa[:, :, 1], 3) & ~read_bits(cm[1, :, :], 3)
+ cloudy_plus1 = read_bits(qa[:, :, 1], 4) & ~read_bits(cm[1, :, :], 4)
+ oceanBTthresh = read_bits(qa[:, :, 1], 5) & ~read_bits(cm[1, :, :], 5)
+ hicloud_1_38 = read_bits(qa[:, :, 2], 0) & ~read_bits(cm[2, :, :], 0)
+ hicloud_39_12 = read_bits(qa[:, :, 2], 1) & ~read_bits(cm[2, :, :], 1)
+ hicloud_11_12 = read_bits(qa[:, :, 2], 2) & ~read_bits(cm[2, :, :], 2)
+ cloud_39_11BTD = read_bits(qa[:, :, 2], 3) & ~read_bits(cm[2, :, :], 3)
+ cloud_VisNIR = read_bits(qa[:, :, 2], 4) & ~read_bits(cm[2, :, :], 4)
+ cloud_NIRVis_ratio = read_bits(qa[:, :, 2], 5) & ~read_bits(cm[2, :, :], 5)
+ shallow_clear = read_bits(qa[:, :, 2], 6) & read_bits(cm[2, :, :], 6)
+ cloud_16_21 = read_bits(qa[:, :, 2], 7) & ~read_bits(cm[2, :, :], 7)
+ hicloud_86_11BTD = read_bits(qa[:, :, 3], 0) & ~read_bits(cm[3, :, :], 0)
+ ocean_clear = read_bits(qa[:, :, 3], 1) & read_bits(cm[3, :, :], 1)
+ land_sunglint_polar = read_bits(qa[:, :, 3], 2) & read_bits(cm[3, :, :], 2)
+ cloud_sfctemp = read_bits(qa[:, :, 3], 3) & ~read_bits(cm[3, :, :], 3)
+ cloud_11umvar = read_bits(qa[:, :, 3], 6) & ~read_bits(cm[3, :, :], 6)
+ lowcloud_39_11BTD = read_bits(qa[:, :, 3], 7) & ~read_bits(cm[3, :, :], 7)
+ vzamask00 = (vza[:, :] <= 10).astype(int)
+ vzamask10 = ((vza[:, :] > 10) & (vza[:, :] <= 30)).astype(int)
+ vzamask30 = ((vza[:, :] > 30) & (vza[:, :] <= 50)).astype(int)
+ vzamask50 = (vza[:, :] > 50).astype(int)
+
+ raz = np.abs(180.0 - np.abs(senaz[:, :] - solaz[:, :]))
+ cos_sctang = -1.0 * (np.cos(sza[:, :]*dtr) *np.cos(vza[:, :]*dtr) -
+ np.sin(sza[:, :]*dtr) *np.sin(vza[:, :]*dtr) *np.cos(raz[:, :]*dtr) )
+ sct = ( np.arccos(cos_sctang[:, :]) * rtd);
+
+ sctmask70 = (sct[:, :] <= 70).astype(int)
+ sctmask90 = ((sct[:, :] > 70) & (sct[:, :] <= 90)).astype(int)
+ sctmask110 = ((sct[:, :] > 90) & (sct[:, :] <= 110)).astype(int)
+ sctmask130 = ((sct[:, :] > 110) & (sct[:, :] <= 130)).astype(int)
+ sctmask150 = ((sct[:, :] > 130) & (sct[:, :] <= 150)).astype(int)
+ sctmask170 = ((sct[:, :] > 150) & (sct[:, :] <= 170)).astype(int)
+ sctmask180 = (sct[:, :] > 170).astype(int)
+
+ masks = {'bit0': bit0,
+ 'cloudy': cloudy,
+ 'cloud_mask_value': cm_flag,
+ 'day': day,
+ 'sunglint': sunglint,
+ 'snow_ice': snow_ice,
+ 'land': only_land,
+ 'thin_cirrus_solar': thin_cirrus_solar,
+ 'snow_covered_from_anc': snow_cover,
+ 'thin_cirrus_IR': thin_cirrus_IR,
+ 'cloudy_plus_1_pixel_adj': cloudy_plus1,
+ 'ocean_BT_threshold': oceanBTthresh,
+ 'high_cloud_flag_138': hicloud_1_38,
+ 'high_cloud_flag_39_12': hicloud_39_12,
+ 'high_cloud_flag_11_12': hicloud_11_12,
+ 'cloud_flag_39_11_BTD_test': cloud_39_11BTD,
+ 'cloud_flag_VisNIR_test': cloud_VisNIR,
+ 'cloud_flag_VisNIR_ratio': cloud_NIRVis_ratio,
+ 'shallow_water_clear_sky_restoral': shallow_clear,
+ 'cloud_flag_16_21_test': cloud_16_21,
+ 'high_cloud_flag_86_11_BTD_test': hicloud_86_11BTD,
+ 'ocean_clear_sky_restoral_test': ocean_clear,
+ 'land_sunglint_polar_clearsky_restoral': land_sunglint_polar,
+ 'cloud_flag_sfc_temp_test': cloud_sfctemp,
+ 'cloud_flag_11um_var_test': cloud_11umvar,
+ 'low_cloud_flag_39_11BTD_test': lowcloud_39_11BTD,
+ 'simple_IR_test': simpleIRtest,
+ 'high_cloud_CO2_test': hicld_CO2,
+ 'high_cloud_067um_test': hicld_67um,
+ 'cloud_vis_reflectance_test': cld_visrefl,
+ 'cloud_near_IR_reflectance_test': cld_nearIR,
+ 'vza_mask_00': vzamask00,
+ 'vza_mask_10': vzamask10,
+ 'vza_mask_30': vzamask30,
+ 'vza_mask_50': vzamask50,
+ 'sct_mask_70' : sctmask70,
+ 'sct_mask_90' : sctmask90,
+ 'sct_mask_110' : sctmask110,
+ 'sct_mask_130' : sctmask130,
+ 'sct_mask_150' : sctmask150,
+ 'sct_mask_170' : sctmask170,
+ 'sct_mask_180' : sctmask180}
+
+ for k in masks.keys():
+ masks[k] = np.ma.array(masks[k], mask=array_mask)
+
+ return masks
+
+
+def read_bits(byte, bits):
+ # orig_shape = byte.shape
+ # if byte.ndim > 1:
+ # byte = byte.reshape((byte.shape[0]*byte.shape[1],))
+
+ if type(bits) is int:
+ flag = np.array((np.array(byte, np.byte) & pow(2, bits))/pow(2, bits),
+ dtype='bool')
+ else:
+ flag = (np.array(byte, np.byte) & pow(2, bits[0]) +
+ pow(2, bits[1]))/pow(2, bits[0])
+
+ flag = np.array(np.trunc(1.0-flag/2.0), dtype='bool')
+ # flag = flag.reshape((orig_shape))
+ return flag
+
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument('input')
+parser.add_argument('test_name')
+parser.add_argument('-c1', default='mediumblue')
+parser.add_argument('-c2', default='skyblue')
+parser.add_argument('-c3', default='yellowgreen')
+parser.add_argument('-c4', default='olivedrab')
+
+args = parser.parse_args()
+
+
+if __name__ == "__main__":
+
+ map_names = ['bit0', 'cloudy', 'cloud_mask_value', 'day', 'sunglint', 'snow_ice',
+ 'land', 'thin_cirrus_solar', 'snow_covered_from_anc', 'thin_cirrus_IR',
+ 'cloudy_plus_1_pixel_adj', 'ocean_BT_threshold', 'high_cloud_flag_138',
+ 'high_cloud_flag_39_12', 'high_cloud_flag_11_12',
+ 'cloud_flag_39_11_BTD_test', 'cloud_flag_VisNIR_test',
+ 'cloud_flag_VisNIR_ratio', 'shallow_water_clear_sky_restoral',
+ 'cloud_flag_16_21_test', 'high_cloud_flag_86_11_BTD_test',
+ 'ocean_clear_sky_restoral_test', 'land_sunglint_polar_clearsky_restoral',
+ 'cloud_flag_sfc_temp_test', 'cloud_flag_11um_var_test',
+ 'low_cloud_flag_39_11BTD_test', 'simple_IR_test',
+ 'high_cloud_CO2_test', 'high_cloud_067um_test',
+ 'cloud_vis_reflectance_test', 'cloud_near_IR_reflectance_test',
+ 'vza_mask_00, vza_mask_10, vza_mask_30, vza_mask_50, sct_mask_70',
+ 'sct_mask_90', 'sct_mask_110', 'sct_mask_130', 'vza_mask_150',
+ 'sct_mask170', 'sct_mask_180']
+
+# for n in map_names:
+# print(n)
+ main(args.input,
+# n,
+ args.test_name,
+ c1=args.c1,
+ c2=args.c2,
+ c3=args.c3,
+ c4=args.c4)
+
diff --git a/dev/sort_inputs.py b/dev/sort_inputs.py
new file mode 100644
index 0000000..df8a9ac
--- /dev/null
+++ b/dev/sort_inputs.py
@@ -0,0 +1,97 @@
+import os.path
+
+import numpy as np
+
+from datetime import datetime
+from glob import glob
+
+import hires_module as hm
+
+
+def create_filenames(vnp02mod_fname):
+
+ datapath = os.path.dirname(vnp02mod_fname)
+ filename = os.path.basename(vnp02mod_fname)
+
+ vnpdate = filename.split('.')[1]
+ vnptime = filename.split('.')[2]
+
+ geos_times = ['0000', '0300', '0600', '0900', '1200', '1500', '1800', '2100']
+ geos_date_from_viirs = datetime.strftime(datetime.strptime(vnpdate, 'A%Y%j'), '%Y%m%d')
+ geos_flist = glob(f'{datapath}/ancillary/GEOS.fpit.asm.inst3_2d_asm_Nx.GEOS5124.{geos_date_from_viirs}*.nc4')
+ geos_flist = np.sort(geos_flist)
+ fmt = '%H%M'
+ diff_times = [(datetime.strptime(gt, fmt) - datetime.strptime(vnptime, fmt)).total_seconds()
+ for gt in geos_times]
+
+ file_index = np.argmin(np.abs(diff_times))
+ if diff_times[file_index] <= 0:
+ geos_file1 = geos_flist[file_index]
+ geos_file2 = geos_flist[file_index + 1]
+ else:
+ geos_file1 = geos_flist[file_index - 1]
+ geos_file2 = geos_flist[file_index]
+
+ land_ocean_fnames = glob(f'{datapath}/ancillary/GEOS.fpit.asm.tavg1_2d_lnd_Nx.GEOS5124.{geos_date_from_viirs}*.nc4')
+ land_ocean_timelist = [t.split('.')[5].split('_')[1] for t in land_ocean_fnames]
+
+ diff_times = [(datetime.strptime(gt, fmt) - datetime.strptime(vnptime, fmt)).total_seconds()
+ for gt in land_ocean_timelist]
+
+ land_ocean_index = np.argmin(np.abs(diff_times))
+ geos_land_flist = glob(f'{datapath}/ancillary/GEOS.fpit.asm.tavg1_2d_lnd_Nx.GEOS5124.{geos_date_from_viirs}*.nc4')
+ geos_land_file = np.sort(geos_land_flist)[land_ocean_index]
+ geos_ocean_flist = glob(f'{datapath}/ancillary/GEOS.fpit.asm.tavg1_2d_ocn_Nx.GEOS5124.{geos_date_from_viirs}*.nc4')
+ geos_ocean_file = np.sort(geos_ocean_flist)[land_ocean_index]
+
+ vnp03mod = glob(f'{datapath}/VNP03MOD.{vnpdate}.{vnptime}*.nc')[0]
+ vnp02img = glob(f'{datapath}/VNP02IMG.{vnpdate}.{vnptime}*_bowtie_corrected.nc')[0]
+ vnp03img = glob(f'{datapath}/VNP03IMG.{vnpdate}.{vnptime}*.nc')[0]
+ cld_msk = glob(f'{datapath}/CLDMSK_L2_VIIRS_SNPP.{vnpdate}.{vnptime}*.nc')[0]
+
+ out_fnames = {'VNP03MOD': os.path.basename(vnp03mod),
+ 'VNP02IMG': os.path.basename(vnp02img),
+ 'VNP03IMG': os.path.basename(vnp03img),
+ 'GEOS_atm_1': os.path.basename(geos_file1),
+ 'GEOS_atm_2': os.path.basename(geos_file2),
+ 'GEOS_land': os.path.basename(geos_land_file),
+ 'GEOS_ocean': os.path.basename(geos_ocean_file),
+ 'GEOS_constants': 'GEOS.fp.asm.const_2d_asm_Nx.00000000_0000.V01.nc4',
+ 'SST_file': 'oisst.20220622',
+ 'ECO_file': 'goge1_2_img.v1',
+ 'NDVI_file': 'NDVI.FM.c004.v2.0.WS.00-04.177.hdf',
+ 'data_path': datapath,
+ 'cloud_mask_file': cld_msk
+ }
+ for k in list(out_fnames):
+ print(f'{k}: {out_fnames[k]}')
+ return out_fnames
+
+
+def call_mvcm():
+
+ flist = np.sort(glob('/ships19/hercules/pveglio/mvcm_viirs_hires/VNP02MOD*.nc'))
+ # flist = np.sort(glob('/ships19/hercules/pveglio/mvcm_viirs_hires/VNP02MOD.A2022173.1454*.nc'))
+
+ for fname in flist[:210]:
+ print(f'Processing {fname}...')
+ fnames_for_mvcm = create_filenames(fname)
+
+ hm.main(data_path=fnames_for_mvcm['data_path'],
+ mod02=fname,
+ mod03=fnames_for_mvcm['VNP03MOD'],
+ img02=fnames_for_mvcm['VNP02IMG'],
+ img03=fnames_for_mvcm['VNP03IMG'],
+ threshold_file=hm._threshold_file,
+ geos_atm_1=fnames_for_mvcm['GEOS_atm_1'],
+ geos_atm_2=fnames_for_mvcm['GEOS_atm_2'],
+ geos_land=fnames_for_mvcm['GEOS_land'],
+ geos_ocean=fnames_for_mvcm['GEOS_ocean'],
+ geos_constants=fnames_for_mvcm['GEOS_constants'],
+ ndvi_file=fnames_for_mvcm['NDVI_file'],
+ sst_file=fnames_for_mvcm['SST_file'],
+ cloud_mask_file=fnames_for_mvcm['cloud_mask_file'])
+
+
+if __name__ == "__main__":
+ create_filenames
diff --git a/dev/tests.py b/dev/tests.py
new file mode 100644
index 0000000..f4c817a
--- /dev/null
+++ b/dev/tests.py
@@ -0,0 +1,535 @@
+import numpy as np
+import xarray as xr
+
+from numpy.lib.stride_tricks import sliding_window_view
+from typing import Dict
+
+import functools
+
+import utils
+import conf
+import conf_xr
+import preprocess_thresholds as pt
+
+import importlib
+
+_DTR = np.pi/180
+
+# this is used for testing, eventually we want to remove it
+importlib.reload(pt)
+
+
+# ############## GROUP 1 TESTS ############## #
+
+def test_11um_old(rad, threshold):
+
+ radshape = rad.shape
+ rad = rad.reshape(np.prod(radshape))
+
+ thr = np.array(threshold['bt11'])
+ confidence = np.zeros(rad.shape)
+
+ if thr[4] == 1:
+ print("11um test running")
+ # the C code has the line below that I don't quite understand the purpose of.
+ # It seems to be setting the bit to 0 if the BT value is greater than the midpoint
+ #
+ # if (m31 >= dobt11[1]) (void) set_bit(13, pxout.testbits);
+
+ # confidence = utils.conf_test(rad, thr)
+ confidence = conf.conf_test(rad, thr)
+
+ return confidence.reshape(radshape)
+
+
+def test_11um_var(rad, threshold, var_threshold):
+
+ print("11um variability test running")
+ thr = np.array(threshold['11um_var'])
+
+ radshape = rad.shape
+ var = np.zeros((radshape[0], radshape[1], 9))
+
+ # chk_spatial2() need to figure out what this is
+ # np = rg_var.num_small_diffs * 1.0
+ test = sliding_window_view(np.pad(rad, [1, 1], mode='constant'), (3, 3)) - np.expand_dims(rad, (2, 3))
+
+ var[np.abs(test).reshape(radshape[0], radshape[1], 9) < var_threshold['dovar11']] = 1
+ var = var.sum(axis=2).reshape(np.prod(radshape))
+
+ rad = rad.reshape(np.prod(radshape))
+ confidence = np.zeros(rad.shape)
+
+ confidence[var == 9] = conf.conf_test(rad[var == 9], thr)
+
+ return confidence.reshape(radshape)
+
+
+def test_11_4diff(rad1, rad2, threshold, viirs_data, sg_thresh):
+
+ print("11um - 4um difference test running")
+ radshape = rad1.shape
+ raddiff = (rad1 - rad2).reshape(np.prod(radshape))
+
+ day = np.zeros(radshape)
+ day[viirs_data.solar_zenith <= 85] = 1
+ day = day.reshape(raddiff.shape)
+ sunglint = np.zeros(rad1.shape)
+ sunglint[viirs_data.sunglint_angle <= sg_thresh] = 1
+ sunglint = sunglint.reshape(raddiff.shape)
+ thr = np.array(threshold['test11_4lo'])
+ confidence = np.zeros(raddiff.shape)
+
+ # confidence[(day == 1) & (sunglint == 0)] = utils.conf_test(raddiff[(day == 1) & (sunglint == 0)], thr)
+ confidence[(day == 1) & (sunglint == 0)] = conf.conf_test(raddiff[(day == 1) & (sunglint == 0)], thr)
+
+ return confidence.reshape(radshape)
+
+
+def vir_refl_test(rad, threshold, viirs_data):
+
+ print('Visible reflectance test running')
+
+ thr = threshold['vis_refl_test']
+
+ radshape = rad.shape()
+ rad = rad.reshape(np.prod(radshape))
+ confidence = np.zeros(radshape)
+ vzcpow = 0.75 # THIS NEEDS TO BE READ FROM THE THRESHOLDS FILE
+
+ vza = viirs_data.sensor_zenith.values
+ dtr = np.pi/180
+ cosvza = np.cos(vza*dtr)
+
+ coeffs = utils.get_b1_thresholds()
+ coeffs[:, :3] = coeffs[:, :3] * threshold['b1_bias_adj']
+
+ # this quantity is the return of get_b1_thresholds() in the C code
+ # it's defined here to keep a consistent logic with the original source, for now
+ irtn = 0
+
+ if irtn != 0:
+ coeffs = thr
+
+ coeffs[:, :3] = coeffs[:, :3] * 1/np.power(cosvza, vzcpow)
+
+ confidence = conf.conf_test(rad, coeffs)
+
+ return confidence.reshape(radshape)
+
+
+def nir_refl_test(rad, threshold, sunglint_thresholds, viirs_data):
+
+ print("NIR reflectance test running")
+ sza = viirs_data.solar_zenith.values
+ refang = viirs_data.sunglint_angle.values
+ vza = viirs_data.sensor_zenith.values
+ dtr = np.pi/180
+ # Keep in mind that band_n uses MODIS band numbers (i.e. 2=0.86um and 7=2.1um)
+ # For VIIRS would be 2=M07 (0.865um) and 7=M11 (2.25um)
+ band_n = 2
+ vzcpow = 0.75 # THIS NEEDS TO BE READ FROM THE THRESHOLDS FILE
+
+ radshape = rad.shape
+ rad = rad.reshape(np.prod(radshape))
+ confidence = np.zeros(rad.shape)
+ sza = sza.reshape(rad.shape)
+ vza = vza.reshape(rad.shape)
+ refang = refang.reshape(rad.shape)
+ sunglint_flag = utils.sunglint_scene(refang, sunglint_thresholds).reshape(rad.shape)
+
+ # ref2 [5]
+ # b2coeffs [4]
+ # b2mid [1]
+ # b2bias_adj [1]
+ # b2lo [1]
+ # vzcpow [3] (in different place)
+
+ cosvza = np.cos(vza*dtr)
+ coeffs = threshold['b2coeffs']
+ hicut0 = np.array(coeffs[0] + coeffs[1]*sza + coeffs[2]*np.power(sza, 2) + coeffs[3]*np.power(sza, 3))
+ hicut0 = (hicut0 * 0.01) + threshold['b2adj']
+ hicut0 = hicut0 * threshold['b2bias_adj']
+ midpt0 = hicut0 + (threshold['b2mid'] * threshold['b2bias_adj'])
+ locut0 = midpt0 + (threshold['b2lo'] * threshold['b2bias_adj'])
+ thr = np.array([locut0, midpt0, hicut0, threshold['ref2'][3]*np.ones(rad.shape)])
+ # corr_thr = np.zeros((4, 4))
+ corr_thr = np.zeros((4, rad.shape[0]))
+
+ corr_thr[:3, sunglint_flag == 0] = thr[:3, sunglint_flag == 0] * (1./np.power(cosvza[sunglint_flag == 0], vzcpow))
+ corr_thr[3, sunglint_flag == 0] = thr[3, sunglint_flag == 0]
+
+ for flag in range(1, 4):
+ if len(refang[sunglint_flag == flag]) > 0:
+ sunglint_thr = utils.get_sunglint_thresholds(refang, sunglint_thresholds, band_n, flag, thr)
+ corr_thr[:3, sunglint_flag == flag] = sunglint_thr[:3, sunglint_flag == flag] * (1./np.power(cosvza[sunglint_flag == flag], vzcpow))
+ corr_thr[3, sunglint_flag == flag] = sunglint_thr[3, sunglint_flag == flag]
+
+ confidence = conf.conf_test(rad, corr_thr)
+
+ return confidence.reshape(radshape)
+
+
+def vis_nir_ratio_test(rad1, rad2, threshold, sg_threshold):
+ print("NIR-Visible ratio test running")
+ if threshold['vis_nir_ratio'][6] == 1:
+
+ radshape = rad1.shape
+ rad1 = rad1.reshape(np.prod(radshape))
+ rad2 = rad2.reshape(np.prod(radshape))
+
+ vrat = rad2/rad1
+
+ thresh = np.zeros((7,))
+
+ # temp value to avoid linter bitching at me
+ # eventually we would have the test run in two blocks as:
+ # confidence[sunglint == 1] = conf.conf_test_dble(vrat[sunglint == 1], sg_threshold['snglnt'])
+ # confidence[sunglint == 0] = conf.conf_test_dble(vrat[sunglint == 0], threshold['vis_nir_ratio'])
+ # sunglint needs to be defined somewhere
+ sunglint = 0
+ if sunglint:
+ thresh = threshold['snglnt']
+ else:
+ thresh = threshold['vis_nir_ratio']
+
+ confidence = conf.conf_test_dble(vrat, thresh)
+
+ return confidence.reshape(radshape)
+
+
+# old class, doesn't use xarray much
+class CloudTests_old:
+
+ def __init__(self, scene_ids, scene_name, thresholds):
+ self.scene = scene_ids
+ self.scene_name = scene_name
+ self.idx = np.where(scene_ids[scene_name] == 1)
+ self.threshold = thresholds[scene_name]
+
+ def single_threshold_test(self, test_name, rad, cmin):
+
+ radshape = rad.shape
+ rad = rad.reshape(np.prod(radshape))
+
+ thr = np.array(self.threshold[test_name])
+ confidence = np.zeros(radshape)
+
+ if thr[4] == 1:
+ print('test running')
+ confidence[self.idx] = conf.conf_test(rad[self.idx], thr)
+
+ cmin[self.idx] = np.minimum(cmin[self.idx], confidence[self.idx])
+ return cmin
+
+ def double_threshold_test(self):
+ pass
+
+
+# new class to try to use xarray more extensively
+class CloudTests(object):
+
+ def __init__(self,
+ data: xr.Dataset,
+ scene_name: str,
+ thresholds: Dict) -> None:
+ self.data = data
+ self.scene_name = scene_name
+ self.thresholds = thresholds
+ self.scene_idx = tuple(np.nonzero(data[scene_name] == 1))
+
+ def run_if_test_exists_for_scene(func):
+ @functools.wraps(func)
+ def wrapper(self, *args, test_name, **kwargs):
+ if test_name not in self.thresholds[self.scene_name]:
+ print('Not running test for this scene')
+ # returns cmin. This could be changed into a keyworded argument for readability
+ return args[-1]
+ return func(self, *args, test_name, **kwargs)
+ return wrapper
+
+ @run_if_test_exists_for_scene
+ def test_11um(self,
+ band: str,
+ cmin: np.ndarray,
+ test_name: str = '11um_Test') -> np.ndarray:
+ confidence = np.ones(self.data[band].shape)
+ threshold = self.thresholds[self.scene_name][test_name]
+
+ if threshold['perform'] is True:
+ print(f'Testing "{self.scene_name}"\n')
+ rad = self.data[band].values[self.scene_idx]
+ confidence[self.scene_idx] = conf.conf_test_new(rad, threshold['thr'])
+
+ cmin = np.fmin(cmin, confidence)
+
+ return cmin
+
+ @run_if_test_exists_for_scene
+ def sst_test(self,
+ band31: str,
+ band32: str,
+ cmin: np.ndarray,
+ test_name: str = 'SST_Test') -> np.ndarray:
+
+ confidence = np.ones(self.data.M01.shape)
+ threshold = self.thresholds[self.scene_name][test_name]
+
+ if threshold['perform'] is True:
+ m31 = self.data[band31].values - 273.16
+ bt_diff = self.data[band31].values - self.data[band32].values
+ sst = self.data.geos_sfct.values - 273.16
+ cosvza = np.cos(self.data.sensor_zenith.values*_DTR)
+
+ c = threshold['coeffs']
+
+ modsst = 273.16 + c[0] + c[1]*m31 + c[2]*bt_diff*sst + c[3]*bt_diff*((1/cosvza) - 1)
+ sfcdif = self.data.geos_sfct.values - modsst
+
+ print(f'Testing "{self.scene_name}"\n')
+ confidence[self.scene_idx] = conf.conf_test_new(sfcdif[self.scene_idx], threshold['thr'])
+
+ cmin = np.fmin(cmin, confidence)
+
+ return cmin
+
+ @run_if_test_exists_for_scene
+ def bt_diff_86_11um(self,
+ band: str,
+ cmin: np.ndarray,
+ test_name: str = '8.6-11um_Test') -> np.ndarray:
+
+ confidence = np.ones(self.data.M01.shape)
+ threshold = self.thresholds[self.scene_name][test_name]
+
+ if threshold['perform'] is True:
+ print(f'Testing "{self.scene_name}"\n')
+ rad = self.data[band].values[self.scene_idx]
+ confidence[self.scene_idx] = conf.conf_test_new(rad, threshold['thr'])
+
+ cmin = np.fmin(cmin, confidence)
+
+ return cmin
+
+ def run_tests_temp(self):
+ cmin_G1 = np.ones(self.data.M01.shape)
+ cmin_G2 = np.ones(self.data.M01.shape)
+
+ # Group 1
+ cmin_G1 = self.test_11um('M15', cmin_G1, test_name='11um_Test')
+ cmin_G1 = self.sst_test('M15', 'M16', cmin_G1, test_name='SST_Test')
+
+ # Group 2
+ cmin_G2 = self.bt_diff_86_11um('M14-M15', cmin_G2, '8.6-11um_Test')
+
+ cmin = cmin_G1 * cmin_G2
+
+ return cmin
+
+ def single_threshold_test(self, test_name, band, cmin):
+
+ if band == 'bad_data':
+ return cmin
+ print(f'Running test "{test_name}" for "{self.scene_name}"')
+
+ # preproc_thresholds()
+ if 'thr' in self.thresholds[self.scene_name][test_name]:
+ thr = np.array(self.thresholds[self.scene_name][test_name]['thr'])
+ else:
+ thr = np.array(self.thresholds[self.scene_name][test_name])
+ thr_xr = xr.Dataset()
+
+ if test_name == '11-12um_Cirrus_Test':
+ thr_xr['threshold'] = pt.preproc(self.data, self.thresholds[self.scene_name], self.scene_name)
+ thr = np.ones((5,)) # This is only temporary to force the logic of the code
+ # I need to find a better solution at some point
+ elif test_name == 'SST_Test':
+ thr_xr['threshold'] = (('number_of_lines', 'number_of_pixels', 'z'),
+ np.ones((self.data[band].shape[0], self.data[band].shape[1], 5))*thr)
+ elif test_name == '7.3-11um_BTD_Mid_Level_Cloud_Test':
+ thr_xr['threshold'] = pt.get_pn_thresholds(self.data, self.thresholds, self.scene_name,
+ '7.3-11um_BTD_Mid_Level_Cloud_Test')
+ thr = np.ones((5,))
+ elif test_name == 'Surface_Temperature_Test':
+ thr_xr['threshold'] = pt.preproc_surf_temp(self.data, self.thresholds[self.scene_name])
+ thr = np.ones((5,))
+ elif (test_name == '11-4um_Oceanic_Stratus_Test' and
+ self.scene_name in ['Land_Day_Desert', 'Land_Day_Desert_Coast', 'Polar_Day_Desert',
+ 'Polar_Day_Desert_Coast']):
+ thr = np.array([self.thresholds[self.scene_name][test_name][i] for i in range(8)])
+ elif test_name == 'NIR_Reflectance_Test':
+ corr_thr = pt.preproc_nir(self.data, self.thresholds, self.scene_name)
+ thr_xr['threshold'] = (('number_of_lines', 'number_of_pixels', 'z'), corr_thr)
+ elif test_name == 'Visible_Reflectance_Test':
+ thr_xr['threshold'], self.data['M128'] = pt.vis_refl_thresholds(self.data,
+ self.thresholds,
+ self.scene_name)
+ elif test_name == '1.6_2.1um_NIR_Reflectance_Test':
+ corr_thr = pt.nir_refl(self.data, self.thresholds, self.scene_name)
+ thr_xr['threshold'] = (('number_of_lines', 'number_of_pixels', 'z'), corr_thr)
+ thr = np.ones((5,))
+ elif test_name == '4-12um_BTD_Thin_Cirrus_Test':
+ thr_xr['threshold'] = pt.get_pn_thresholds(self.data, self.thresholds, self.scene_name,
+ '4-12um_BTD_Thin_Cirrus_Test')
+ thr = np.ones((5,))
+ elif test_name == 'GEMI_Test':
+ thr_xr['threshold'] = pt.GEMI_test(self.data, self.thresholds, self.scene_name)
+ thr = np.ones((5,))
+ elif (test_name == '1.38um_High_Cloud_Test' and self.scene_name in ['Ocean_Day', 'Polar_Ocean_Day']):
+ thr_xr['threshold'] = pt.test_1_38um_preproc(self.data, self.thresholds, self.scene_name)
+ thr = np.ones((5,))
+ else:
+ thr_xr['threshold'] = (('number_of_lines', 'number_of_pixels', 'z'),
+ np.ones((self.data[band].shape[0], self.data[band].shape[1], 5))*thr)
+
+ data = xr.Dataset(self.data, coords=thr_xr)
+
+ if test_name == 'SST_Test':
+ data['sfcdif'] = (('number_of_lines', 'number_of_pixels'),
+ pt.preproc_sst(data, self.thresholds[self.scene_name][test_name]).values)
+ band = 'sfcdif'
+ if test_name == '11um_Variability_Test':
+ var = pt.var_11um(self.data, self.thresholds)
+ data['11um_var'] = data.M15
+ data['11um_var'].values[var != 9] = np.nan
+
+ if thr[4] == 1:
+ print('test running...')
+ confidence = conf_xr.conf_test(data, band)
+
+ cmin = np.fmin(cmin, confidence)
+
+ return cmin
+
+ def double_threshold_test(self, test_name, band, cmin):
+ data = self.data
+ if test_name == '11-4um_BT_Difference_Test':
+ thr = pt.bt11_4um_preproc(self.data, self.thresholds, self.scene_name)
+ print('test running...')
+ confidence = conf.conf_test_dble(data['M15-M13'].values, thr)
+ confidence = confidence.reshape(data.M01.shape)
+
+ if test_name == 'Vis/NIR_Ratio_Test':
+ print('test running...')
+ thr_no_sunglint = np.array([self.thresholds[self.scene_name][test_name][i] for i in range(8)])
+ thr_sunglint = np.array([self.thresholds['Sun_Glint']['snglnt'][i] for i in range(8)])
+ vrat = data.M07.values/data.M05.values
+ _dtr = np.pi/180.0
+ sza = data.sensor_zenith.values
+ raz = data.relative_azimuth.values
+ vza = data.sensor_zenith.values
+ cos_refang = np.sin(vza*_dtr) * np.sin(sza*_dtr) * np.cos(raz*_dtr) + \
+ np.cos(vza*_dtr) * np.cos(sza*_dtr)
+ refang = np.arccos(cos_refang) * 180./np.pi
+ idx = np.nonzero((data.solar_zenith <= 85) & (refang <= data.sunglint_angle))
+
+ confidence = conf.conf_test_dble(vrat, thr_no_sunglint)
+ confidence = confidence.reshape(data.M01.shape)
+ confidence[idx] = conf.conf_test_dble(vrat[idx], thr_sunglint)
+
+ confidence = confidence.reshape(data.M01.shape)
+
+ if (test_name == '11-4um_Oceanic_Stratus_Test' and
+ self.scene_name in ['Land_Day_Desert', 'Land_Day_Desert_Coast', 'Polar_Day_Desert',
+ 'Polar_Day_Desert_Coast']):
+ thr = np.array([self.thresholds[self.scene_name][test_name][i] for i in range(8)])
+
+ print('test running...')
+ confidence = conf.conf_test_dble(data['M15-M16'].values, thr)
+ confidence = confidence.reshape(data.M01.shape)
+
+ cmin = np.fmin(cmin, confidence)
+
+ return cmin
+
+
+class ComputeTests(CloudTests):
+
+ def __init__(self,
+ data: xr.Dataset,
+ scene_name: str,
+ thresholds: Dict) -> None:
+ super().__init__(data, scene_name, thresholds)
+
+ def run_tests(self):
+ cmin_G1 = np.ones(self.data.M01.shape)
+ cmin_G2 = np.ones(self.data.M01.shape)
+
+ # Group 1
+ cmin_G1 = self.test_11um('M15', cmin_G1, test_name='11um_Test')
+ cmin_G1 = self.sst_test('M15', 'M16', cmin_G1, test_name='SST_Test')
+
+ # Group 2
+ cmin_G2 = self.bt_diff_86_11um('M14-M15', cmin_G2, test_name='8.6-11um_Test')
+
+ cmin = cmin_G1 * cmin_G2
+
+ return cmin
+
+
+def preproc_thresholds(thresholds, data):
+ thr = np.array(thresholds)
+ thr_xr = xr.Dataset()
+ thr_xr['tresholds'] = (('number_of_lines', 'number_of_pixels', 'z'),
+ np.ones((data['M01'].shape[0], data['M01'].shape[1], 5))*thr)
+
+ nl_sfct1 = thresholds['Land_Night']['Surface_Temperature_Test'][0]
+# nl_sfct2 = thresholds['Land_Night']['Surface_Temperature_Test'][1]
+# nlsfct_pfm = thresholds['Land_Night']['Surface_Temperature_Test'][2]
+ nl_df1 = thresholds['Land_Night']['Surface_Temperature_Test_difference'][0:2]
+ nl_df2 = thresholds['Land_Night']['Surface_Temperature_Test_difference'][2:]
+
+# df1 = data.M15 - data.M16
+# df2 = data.M15 - data.M13
+ thr_xr = thr_xr.where(data.desert != 1, nl_sfct1)
+ thr_xr = thr_xr.where((data['M15-M16'] > nl_df1[0]) |
+ ((data['M15-M16'] < nl_df1[0]) &
+ ((data['M15-M13'] <= nl_df2[0]) | (data['M15-M13'] >= nl_df2[1]))),
+ nl_sfct1[0])
+
+ data = xr.Dataset(data, coords=thr_xr)
+
+ return data
+
+
+def single_threshold_test(test, rad, threshold):
+
+ radshape = rad.shape
+ rad = rad.reshape(np.prod(radshape))
+
+ thr = np.array(threshold[test])
+ confidence = np.zeros(rad.shape)
+
+ if thr[4] == 1:
+ print(f"{test} test running")
+ # the C code has the line below that I don't quite understand the purpose of.
+ # It seems to be setting the bit to 0 if the BT value is greater than the midpoint
+ #
+ # if (m31 >= dobt11[1]) (void) set_bit(13, pxout.testbits);
+
+ # confidence = utils.conf_test(rad, thr)
+ confidence = conf.conf_test(rad, thr)
+
+ return confidence.reshape(radshape)
+
+
+def test():
+ rad = np.random.randint(50, size=[4, 8])
+ # coeffs = [5, 42, 20, 28, 15, 35, 1]
+ # coeffs = [20, 28, 5, 42, 15, 35, 1]
+ coeffs = [35, 15, 20, 1, 1]
+ # confidence = conf_test_dble(rad, coeffs)
+ confidence = test_11um(rad, coeffs)
+ print(rad)
+ print('\n')
+ print(confidence)
+
+
+if __name__ == "__main__":
+ test()
+
+
+
+
+
+
diff --git a/dev/write_temp_data.py b/dev/write_temp_data.py
new file mode 100644
index 0000000..ae60698
--- /dev/null
+++ b/dev/write_temp_data.py
@@ -0,0 +1,24 @@
+import main
+import read_data as rd
+
+
+def main_fcn():
+ file_names = {'MOD02': main._fname_mod02,
+ 'MOD03': main._fname_mod03,
+ 'IMG02': main._fname_img02,
+ 'IMG03': main._fname_img03,
+ 'GEOS_atm_1': main._geos_atm_1,
+ 'GEOS_atm_2': main._geos_atm_2,
+ 'GEOS_land': main._geos_land,
+ 'GEOS_ocean': main._geos_ocean,
+ 'GEOS_constants': main._geos_constants,
+ 'NDVI': main._ndvi_file,
+ 'SST': main._sst_file,
+ 'ANC_DIR': f'{main._datapath}/ancillary'}
+
+ viirs_data = rd.get_data(file_names, 40, hires=False)
+ viirs_data.to_netcdf('viirs_data.nc')
+
+
+if __name__ == "__main__":
+ main_fcn()
diff --git a/mvcm/__init__.py b/mvcm/__init__.py
new file mode 100644
index 0000000..ffcc925
--- /dev/null
+++ b/mvcm/__init__.py
@@ -0,0 +1 @@
+__version__ = '0.0.3'
diff --git a/mvcm/ancillary.pyx b/mvcm/ancillary.pyx
new file mode 100644
index 0000000..73894f8
--- /dev/null
+++ b/mvcm/ancillary.pyx
@@ -0,0 +1,205 @@
+# cython: language_level=3
+# cython: c_string_Type=unicode, c_string_encoding=utf8
+
+cdef extern void get_Reynolds_SST(float *, float *, int, char *, char *, float *)
+cdef extern void get_NDVI_background(float *, float *, int, char *, char *, float *)
+cdef extern void get_Olson_eco(float *, float *, int, char *, unsigned char *)
+cdef extern void get_GEOS(float *, float *, int, char *, char *, char *, char *, char *, char *, char *,
+ float *, float *, float *, float *, float *, float *)
+cdef extern void snow_mask(char *, unsigned char)
+cdef extern float cithr(int, float, float)
+cdef extern void check_reg_uniformity(int, int, int, int, int, unsigned char, unsigned char *,
+ float *, float *, unsigned char *, int *, int *, int *, int *)
+#cdef extern int get_b1_thresholds(float *, float *, float *, float *)
+
+import cython
+from cython.view cimport array as cvarray
+
+import numpy as np
+cimport numpy as np
+
+np.import_array()
+
+ctypedef np.float_t DTYPE_t
+DTYPE = np.float32
+
+@cython.boundscheck(False)
+@cython.wraparound(False)
+@cython.initializedcheck(False)
+def py_get_Reynolds_SST(np.ndarray[float, ndim=1] lat,
+ np.ndarray[float, ndim=1] lon, res,
+ char *anc_dir, char *sst_file, sst):
+
+ # cdef np.ndarray sst = np.zeros((3232*3200, ), order='C', dtype=np.float32)
+ if not sst.flags['C_CONTIGUOUS']:
+ sst = np.ascontiguousarray(sst)
+
+ cdef float[::1] sst_mv = sst
+
+ get_Reynolds_SST(&lat[0], &lon[0], res, anc_dir, sst_file, &sst_mv[0])
+
+ return sst
+
+
+@cython.boundscheck(False)
+@cython.wraparound(False)
+@cython.initializedcheck(False)
+def py_get_NDVI_background(np.ndarray[float, ndim=1] lat,
+ np.ndarray[float, ndim=1] lon, res,
+ char *anc_dir, char *ndvi_file, ndvi):
+
+ if not ndvi.flags['C_CONTIGUOUS']:
+ ndvi = np.ascontiguousarray(ndvi)
+
+ cdef float[::1] ndvi_mv = ndvi
+
+ get_NDVI_background(&lat[0], &lon[0], res, anc_dir, ndvi_file, &ndvi_mv[0])
+
+ return ndvi
+
+
+@cython.boundscheck(False)
+@cython.wraparound(False)
+@cython.initializedcheck(False)
+def py_get_Olson_eco(np.ndarray[float, ndim=1] lat,
+ np.ndarray[float, ndim=1] lon, res,
+ char *anc_dir, eco):
+
+ if not eco.flags['C_CONTIGUOUS']:
+ eco = np.ascontiguousarray(eco)
+
+ cdef unsigned char[::1] eco_mv = eco
+
+ get_Olson_eco(&lat[0], &lon[0], res, anc_dir, &eco_mv[0])
+
+ return eco
+
+
+@cython.boundscheck(False)
+@cython.wraparound(False)
+@cython.initializedcheck(False)
+def py_get_GEOS(np.ndarray[float, ndim=1] lat, np.ndarray[float, ndim=1] lon, int res, char *startTime,
+ char *anc_dir, char *geos1, char *geos2, char *geos_lnd, char *geos_ocn, char *geos_cnst,
+ geos_data):
+
+ for v in geos_data:
+ if not geos_data[v].flags['C_CONTIGUOUS']:
+ geos_data[v] = np.ascontiguousarray(geos_data[v])
+
+ cdef float[::1] tpw_mv = geos_data['tpw']
+ cdef float[::1] snowfr_mv = geos_data['snow_fraction']
+ cdef float[::1] icefr_mv = geos_data['ice_fraction']
+ cdef float[::1] ocnfr_mv = geos_data['ocean_fraction']
+ cdef float[::1] landicefr_mv = geos_data['land_ice_fraction']
+ cdef float[::1] sfct_mv = geos_data['surface_temperature']
+
+ get_GEOS(&lat[0], &lon[0], res, startTime, anc_dir, geos1, geos2, geos_lnd, geos_ocn, geos_cnst,
+ &tpw_mv[0], &snowfr_mv[0], &icefr_mv[0], &ocnfr_mv[0], &landicefr_mv[0], &sfct_mv[0])
+
+ geos_dict = {'tpw': geos_data['tpw'],
+ 'snow_fraction': geos_data['snow_fraction'],
+ 'ice_fraction': geos_data['ice_fraction'],
+ 'ocean_fraction': geos_data['ocean_fraction'],
+ 'land_ice_fraction': geos_data['land_ice_fraction'],
+ 'surface_temperature': geos_data['surface_temperature']
+ }
+
+ return geos_dict
+
+
+@cython.boundscheck(False)
+@cython.wraparound(False)
+@cython.initializedcheck(False)
+def py_snow_mask(char *satname, unsigned char lsf):
+ # need to have for loop here to compute all the pixels since the function, as with everything else,
+ # is run per pixel.
+ pass
+
+
+@cython.boundscheck(False)
+@cython.wraparound(False)
+@cython.initializedcheck(False)
+def py_cithr(int key, np.ndarray[float, ndim=1] sec_vza, np.ndarray[float, ndim=1] bt11):
+
+ cdef np.ndarray tci_thr = np.zeros((bt11.shape[0], ), order='C', dtype=np.float)
+
+ for i in range(bt11.shape[0]):
+ tci_thr[i] = cithr(key, sec_vza[i], bt11[i])
+
+ return tci_thr
+
+
+@cython.boundscheck(False)
+@cython.wraparound(False)
+@cython.initializedcheck(False)
+def py_check_reg_uniformity(np.ndarray[unsigned char, ndim=2] eco_type, np.ndarray[unsigned char, ndim=2] eco,
+ np.ndarray[float, ndim=2] snowfr,
+ np.ndarray[float, ndim=2] icefr, np.ndarray[unsigned char, ndim=2] lsf):
+
+ cdef int coast, land, water, loc_uniform
+ cdef int i, j
+
+ cdef np.ndarray coast_r = np.zeros((eco_type.shape[0], eco_type.shape[1]), order='C', dtype=np.int32)
+ cdef np.ndarray land_r = np.zeros((eco_type.shape[0], eco_type.shape[1]), order='C', dtype=np.int32)
+ cdef np.ndarray water_r = np.zeros((eco_type.shape[0], eco_type.shape[1]), order='C', dtype=np.int32)
+ cdef np.ndarray loc_uniform_r = np.zeros((eco_type.shape[0], eco_type.shape[1]), order='C', dtype=np.int32)
+ cdef int[:, ::1] coast_mv = coast_r
+ cdef int[:, ::1] land_mv = land_r
+ cdef int[:, ::1] water_mv = water_r
+ cdef int[:, ::1] loc_uniform_mv = loc_uniform_r
+
+ lines = eco_type.shape[0]
+ eles = eco_type.shape[1]
+ for i in range(lines):
+ for j in range(eles):
+ if (i == 0 or i == lines):
+ line_edge = 1
+ else:
+ line_edge = 0
+ if (j == 0 or j == eles):
+ elem_edge = 1
+ else:
+ elem_edge = 0
+ check_reg_uniformity(eles, line_edge, elem_edge, i, j, eco_type[i][j],
+ &eco[0, 0], &snowfr[0, 0], &icefr[0, 0], &lsf[0, 0],
+ &coast, &land, &water, &loc_uniform)
+
+ coast_mv[i][j] = coast
+ land_mv[i][j] = land
+ water_mv[i][j] = water
+ loc_uniform_mv[i][j] = loc_uniform
+
+ scene_uniformity = {'coast': coast_r,
+ 'land': land_r,
+ 'water': water_r,
+ 'loc_uniform': loc_uniform_r}
+
+ return scene_uniformity
+
+
+#@cython.boundscheck(False)
+#@cython.wraparound(False)
+#@cython.initializedcheck(False)
+#def py_get_b1_thresholds(np.ndarray[float, ndim=2] locut, np.ndarray[float, ndim=2] midpt,
+# np.ndarray[float, ndim=2] hicut, np.ndarray[float, ndim=2] power):
+#
+# cdef int i, j
+#
+# cdef np.ndarray_locut_r = np.zeros((), order='C', dtype=np.float32)
+#
+# cdef float[:, ::1] locut_mv = locut
+# cdef float[:, ::1] midpt_mv = midpt
+# cdef float[:, ::1] hicut_mv = hicut
+# cdef float[:, ::1] power_mv = power
+# lines = 0
+# eles = 0
+#
+# for i in range(lines):
+# for j in range(eles):
+#
+# get_b1_thresholds(&locut[0, 0], &midpt[0, 0], &hicut[0, 0], &power[0, 0])
+#
+# locut_mv[i][j] = locut
+# midpt_mv[i][j] = midpt
+# hicut_mv[i][j] = hicut
+# power_mv[i][j] = power
diff --git a/mvcm/c_tools/__init__.py b/mvcm/c_tools/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/mvcm/c_tools/assign_geos_vals.c b/mvcm/c_tools/assign_geos_vals.c
new file mode 100644
index 0000000..4648c19
--- /dev/null
+++ b/mvcm/c_tools/assign_geos_vals.c
@@ -0,0 +1,268 @@
+/*******************************************************************************
+Description:
+
+ Integer function assign_geos_vals.c
+ Assigns GEOS variable values to each pixel location in the input L1b granule.
+ Also spatially interpolates GEOS values to pixel locations if desired.
+ Called from get_GEOS.c
+
+Input arguments:
+ none
+
+Output arguments:
+ none
+
+Function output:
+ int return_code successful completion is zero, otherwise non-zero
+
+ Pointers to inputs are stored in ancillary.h.
+ Pointers to output variables are stored in ancillary.h.
+
+Revision History:
+ 10/2012 R. Frey Original version
+ 01/2021 R. Frey Made changes to accomodate sector processing.
+ Code cleanup.
+ 06/2021 R. Frey Modified from assign_gdas_vals.c
+ 06/2021 R. Frey Added 'snowfr' and 'icefr'.
+
+Calls:
+ none
+
+*******************************************************************************/
+
+// Includes
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+//MVCCM includes
+#include "ancillary.h"
+//#include "granule_geolocation.h"
+//#include "granule.h"
+#include "mask_processing_constants.h"
+
+/******************************************************************************/
+
+int assign_geos_vals(float *lat, float *lon, int res)
+
+{
+
+/******************************************************************************/
+
+// Declarations and initializations.
+
+ extern double fabs(double);
+
+ const int eles=3200*res;
+ const int lines=3232*res;
+
+ int row, col;
+ int i, j, ij, i1, i2, j1, j2, ic, jc;
+ int j1c, j2c, i1c, i2c;
+ int line1, line2, ele1, ele2;
+ long int n_pixels = eles * lines;
+ long int nwp_index, nwp_index_c;
+ long int nwp_index_c1, nwp_index_c2, nwp_index_c3, nwp_index_c4;
+ long int indx00, indx01, indx10, indx11;
+ long int idx, idxloc;
+ float lst_sum;
+ double x0 = -180.0;
+ double dx = 0.625;
+ double dxc = 0.3125;
+ double y0 = -90.0;
+ double dy = 0.50;
+ double dyc = 0.25;
+ double x, lat1, lat2, lat00, lat01, lat10, lat11, dyi, pyi, py;
+ double y, lon1, lon2, lon00, lon01, lon10, lon11, dxi, pxi, px;
+ double t, tx, ty, t00, t01, t10, t11;
+
+ int return_code = 0;
+
+/******************************************************************************/
+
+// Allocate memory for assigned data. Structure variables are stored in
+// ancillary.h.
+ if ((grn_anc.sfct = (float *) malloc(sizeof(float) * n_pixels)) == NULL) {
+ printf("Could not allocate memory for sfct\n");
+ return(-1);
+ }
+ if ((grn_anc.tpw = (float *) malloc(sizeof(float) * n_pixels)) == NULL) {
+ printf("Could not allocate memory for tpw\n");
+ return(-2);
+ }
+ if ((grn_anc.snowfr = (float *) malloc(sizeof(float) * n_pixels)) == NULL) {
+ printf("Could not allocate memory for snowfr\n");
+ return(-2);
+ }
+ if ((grn_anc.icefr = (float *) malloc(sizeof(float) * n_pixels)) == NULL) {
+ printf("Could not allocate memory for icefr\n");
+ return(-2);
+ }
+ if ((grn_anc.geos_ocnfr = (float *) malloc(sizeof(float) * n_pixels)) == NULL) {
+ printf("Could not allocate memory for geos_ocnfr\n");
+ return(-2);
+ }
+ if ((grn_anc.landicefr = (float *) malloc(sizeof(float) * n_pixels)) == NULL) {
+ printf("Could not allocate memory for landicefr\n");
+ return(-2);
+ }
+
+/******************************************************************************/
+
+// Get TPW, snow fraction, ice fraction, and bilinearly interpolated surface
+// temperature.
+
+/******************************************************************************/
+
+// Use 0-based line and element numbers.
+ idx = 0;
+ idxloc = 0;
+ ij = 0;
+
+ for ( row=0; row<lines; ++row) {
+ for ( col=0; col<eles; ++col) {
+
+/******************************************************************************/
+
+// Find GEOS 0.5-degree grid box corresponding to current location.
+ y = min(max(lat[idx], -90.00), 90.00);
+ j = (int) ((y - y0 + dy) / dy);
+
+ x = min(max(lon[idx], -180.00), 179.375);
+ i = (int) ((x - x0 + dx) / dx);
+
+ nwp_index = (j-1)*nwp_cols + (i-1);
+ if(nwp_index < 0 || nwp_index > nwp_cells) {
+ printf("NWP grid box out of range: %ld\n", nwp_index);
+ return (-3);
+ }
+
+// Find GEOS 0.25-degree grid box corresponding to current location.
+ jc = (int) ((y - y0 + dyc) / dyc);
+ ic = (int) ((x - x0 + dxc) / dxc);
+ nwp_index_c = ((jc-1)*nwp_cols*2) + (ic-1);
+
+/******************************************************************************/
+// Don't spatially interpolate the following variables.
+ grn_anc.tpw[idxloc] = *(ti_tpw + nwp_index);
+ grn_anc.snowfr[idxloc] = *(g_snow + nwp_index);
+ grn_anc.icefr[idxloc] = *(g_ice + nwp_index);
+ grn_anc.geos_ocnfr[idxloc] = (*(g_frocn + nwp_index_c));
+ grn_anc.landicefr[idxloc] = (*(g_frlandice + nwp_index_c));
+
+// Get non-interpolated value in case interpolation is not performed.
+ grn_anc.sfct[idxloc] = *(ti_sfct + nwp_index);
+
+/******************************************************************************/
+
+// Find four GEOS values that surround the pixel of interest. Begin by
+// finding the two surrounding latitude bands and computing the fractional
+// distance between the pixel and the closest latitude band.
+ lat1 = y0 + ((j-1) * dy);
+ lat2 = y0 + ((j+1) * dy);
+ if( fabs(y-lat1) <= fabs(y-lat2) ) {
+ j1 = j-1;
+ j2 = j;
+ }else {
+ j1 = j;
+ j2 = j+1;
+ }
+ lat00 = y0 + (j1 * dy);
+ lat10 = y0 + (j2 * dy);
+ dyi = lat00 - lat10;
+ pyi = lat00 - y;
+ if(pyi > 0.0) {
+ py = pyi / dyi;
+ }
+ else {
+ py = dy;
+ }
+ if(j1 < 0 || fabs(dyi) < 0.000001 || py > 1.00 || py < 0.00) {
+ printf("problem with j index in spatial interpolation\n");
+ printf("%d %d %ld %d %d %d %f %f %f %f %f %f\n", row,col,idx,j,j1,j2,lat00,lat10,dyi,y,pyi,py);
+ }
+
+// Find surrounding longitudes and compute the fractional distance between
+// the pixel of interest and the closest longitude box.
+ lon1 = x0 + ((i-1) * dx);
+ lon2 = x0 + ((i+1) * dx);
+// if(lon1 < 0.0) lon1 = lon1 + 360.0;
+ if( fabs(x-lon1) < fabs(x-lon2) ) {
+ i1 = i-1;
+ i2 = i;
+ }else {
+ i1 = i;
+ i2 = i+1;
+ }
+
+ lon00 = x0 + (i1 * dx);
+ lon10 = x0 + (i2 * dx);
+ dxi = lon00 - lon10;
+ pxi = lon00 - x;
+ if(pxi > 0.0) {
+ px = pxi/dxi;
+ }
+ else {
+ px = dx;
+ }
+ if(fabs(dxi) < 0.000001 || px > 1.00 || px < 0.00) {
+ printf("problem with i index in spatial interplation\n");
+ printf("%d %d %ld %d %d %d %f %f %f %f %f %f\n", row,col,idx,i,i1,i2,lon00,lon10,dxi,x,pxi,px);
+ }
+
+// Following are indices of surrounding NWP gridboxes.
+ indx00 = (j1 * nwp_cols) + i1;
+ indx01 = (j1 * nwp_cols) + i2;
+ indx10 = (j2 * nwp_cols) + i1;
+ indx11 = (j2 * nwp_cols) + i2;
+
+// Interpolate only if region contains only land or water.
+// Get ocean fraction for the four regions.
+ j1c = (j1*2)-1;
+ j2c = j2*2;
+ i1c = (i1*2)-1;
+ i2c = i2*2;
+ if(j1c < 0) j1c = 1;
+ if(i1c < 0) i1c = 1;
+ nwp_index_c1 = (j1c-1)*nwp_cols*2 + (i1c-1);
+ nwp_index_c2 = (j1c-1)*nwp_cols*2 + (i2c-1);
+ nwp_index_c3 = (j2c-1)*nwp_cols*2 + (i1c-1);
+ nwp_index_c4 = (j2c-1)*nwp_cols*2 + (i2c-1);
+ lst_sum = *(g_frocn+nwp_index_c1) + *(g_frocn+nwp_index_c2) + *(g_frocn+nwp_index_c3) + *(g_frocn+nwp_index_c4);
+// printf("\nlat,lon: %f %f %f %f %f %f\n", grn_geo.lat[idx], grn_geo.lon[idx],lat00,lat10,lon00,lon10);
+// printf("nfrocean1: %d %d %d %d %d %d %d %d\n", j1,j2,i1,i2,j1c,j2c,i1c,i2c);
+// printf("frocean2: %f %f %f %f %f\n", *(g_frocn+nwp_index_c1),*(g_frocn+nwp_index_c2),*(g_frocn+nwp_index_c3),
+// *(g_frocn+nwp_index_c4),lst_sum);
+
+ if(lst_sum == 4.0 || lst_sum == 0.0) {
+
+ t00 = ti_sfct[indx00];
+ t01 = ti_sfct[indx01];
+ tx = ((1.0 - px) * t00) + (px * t01);
+
+ t10 = ti_sfct[indx10];
+ t11 = ti_sfct[indx11];
+ ty = ((1.0 - px) * t10) + (px * t11);
+ t = ((1.0 - py) * tx) + (py * ty);
+
+ grn_anc.sfct[idxloc] = t;
+
+// if(ij < 10)
+// printf("interp: %f %f %f %f %f %f %f %f\n", lst_sum, t00, t01, tx, t10, t11, ty, t);
+// ij++;
+
+ }
+
+ idx++;
+ idxloc++;
+
+ }
+ }
+
+/******************************************************************************/
+
+ return (return_code);
+
+/******************************************************************************/
+
+}
diff --git a/mvcm/c_tools/bilinearInterpSST.c b/mvcm/c_tools/bilinearInterpSST.c
new file mode 100644
index 0000000..cea548d
--- /dev/null
+++ b/mvcm/c_tools/bilinearInterpSST.c
@@ -0,0 +1,112 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <math.h>
+
+#include "sst.h"
+
+int bilinearInterpSST(int, int, float, float, const SST *, float *);
+
+/* ===== bilinearInterpSST ================================================== */
+/*
+*| Name:
+*| bilinearInterpSST - Bi-linear interpolation scheme used with Reynolds
+*| blended SST data
+*|
+*| Interface:
+*| int
+*| bilinearInterpSST (int xindx, int yindx, float lat, float lon, const SST *sstGrid float returnValue)
+*|
+*| Input:
+*| i Grid index in the x direction
+*| j Grid index in the y direction
+*| lat Pixel latitude value
+*| lon Pixel longitude value
+*| sstGrid The SST grid from the Reynold's SST file
+*|
+*|
+*| Input and Output:
+*| none
+*|
+*| Output:
+*| t Interpolated SST Value
+*|
+*| Return values:
+*| 0 - Success.
+*| -1 - Unsuccessful
+*|
+*|
+*/
+
+int
+ bilinearInterpSST (int i, int j, float lat, float lon, const SST *sstGrid, float * t)
+{
+
+ extern float fabsf(float);
+
+ int npoints_x = 360;
+ int npoints_y = 180;
+ float dx=1.0;
+ float dy=1.0;
+ int doYInterp = 1;
+ int i1, j1;
+ float p, ip0, ip1, jp0, jp1, t00, t01, t0, t10, t11, t1, ldi, ldj;
+
+ if ( j < 0 || j > (npoints_y -1))
+ return(-1);
+ if ( i < 0 || i > (npoints_x -1))
+ return(-1);
+
+ ldi = lon - i;
+ if (ldi >= 0.5) {
+ i1 = i + 1;
+ if ( i == (npoints_x - 1))
+ i1 = 0;
+ }
+ else {
+ i1 = i - 1;
+ if ( i == 0)
+ i1 = npoints_x - 1;
+ }
+
+ ldj = lat - (j - 90.0);
+
+ if (ldj >= 0.5) {
+ j1 = j + 1;
+ if ( j == (npoints_y - 1))
+ doYInterp = 0;
+ }
+ else {
+ j1 = j - 1;
+ if ( j == 0)
+ doYInterp = 0;
+ }
+
+ p = lon - (i1 + 0.5);
+ if ( p < -1.0 )
+ p = p + 360.0;
+ if ( p > 1.0)
+ p = p - 360.0;
+ ip0 = fabsf( p / dx);
+ ip1 = dx - ip0;
+
+ t00 = sstGrid->values[j][i];
+ t01 = sstGrid->values[j][i1];
+ t0 = (ip0 * t00) + (ip1 * t01);
+
+ if( doYInterp ) {
+ p = lat - (j1 - 90.0 + 0.5);
+ jp0 = fabsf( p / dy );
+ jp1 = dy - jp0;
+
+ t10 = sstGrid->values[j1][i];
+ t11 = sstGrid->values[j1][i1];
+ t1 = (ip0 * t10) + (ip1 * t11);
+
+ *t = (jp0 * t0) + (jp1 * t1);
+ }
+ else
+ *t = t0;
+
+ return (0);
+}
+
diff --git a/mvcm/c_tools/check_reg_uniformity.c b/mvcm/c_tools/check_reg_uniformity.c
new file mode 100644
index 0000000..5fa882f
--- /dev/null
+++ b/mvcm/c_tools/check_reg_uniformity.c
@@ -0,0 +1,141 @@
+/**********************************************************************
+Description:
+
+ Integer function check_reg_uniformity.c
+ Evaluates surface uniformity.
+ Called from get_pxldat.c.
+
+Input arguments:
+ none
+
+ Inputs from mask_processing_constants.h
+
+Output arguments:
+ none
+
+ int uniform value of 1 if uniform, otherwise 0
+ (returned through function call)
+
+Revision History:
+ 05/2007 R. Frey Original version.
+ 10/2012 R. Frey Adapted to MVCCM software
+ 11/2015 R. Frey Code clean-up.
+ 06/2021 R. Frey Uses GEOS snow and ice instead of NISE.
+
+***********************************************************************/
+
+/* Includes */
+
+#include <stdio.h>
+//#include "mask_processing_constants.h"
+//#include "granule.h"
+//#include "pixel.h"
+//#include "ancillary.h"
+//#include "granule_geolocation.h"
+
+void check_reg_uniformity(int eles, int line_edge, int elem_edge, int scan, int elem, unsigned char eco_type,
+ unsigned char *eco, float *snowfr, float *icefr, unsigned char *lst,
+ int *land, int *coast, int *water, int *loc_uniform)
+
+
+{
+
+/* Declarations */
+
+ //int loc_uniform;
+ int nland;
+ int nwater;
+ int ncoast;
+ int ntotal;
+ int nl, ne;
+ int imve, imvl;
+ int pxsnow;
+
+ const int nlcntx = 3;
+ const int necntx = 3;
+ long int j, ide;
+
+ unsigned char lsf;
+
+ float snow_frc, ice_frc;
+
+/* Initializations */
+
+ nwater = 0;
+ nland = 0;
+ ncoast = 0;
+ ntotal = nlcntx * necntx;
+ *loc_uniform = 1;
+ imve = (necntx-1) / 2;
+ imvl = (nlcntx-1) / 2;
+
+// Check for edges of granule.
+ if(line_edge == 1 || elem_edge == 1) {
+
+ *loc_uniform = 0;
+
+ }
+ else {
+
+// Check region (3x3 pixel area centered on current pixel) for scene uniformity.
+
+ for ( nl=0; nl<nlcntx; ++nl) {
+ for ( ne=0; ne<necntx; ++ne) {
+
+ j = ( (scan + (nl - imvl)) * eles) + elem;
+ ide = j + (ne - imve);
+
+// Ecosystem consistency.
+ if ( (eco_type - *(eco+ide)) != 0) *loc_uniform = 0;
+
+// Snow/ice consistency.
+ snow_frc = *(snowfr+ide);
+ ice_frc = *(icefr+ide);
+ pxsnow = 0;
+ if ( (snow_frc > 0.25 && snow_frc <= 1.0) || (ice_frc > 0.25) ) pxsnow = 1;
+ if(pxsnow == 1) *loc_uniform = 0;
+
+// Count land, water, and coast pixels in region for consistency calculation
+// below.
+ lsf = *(lst+ide);
+ if(lsf == 1 || lsf == 4) {
+ nland++;
+ }
+ else if(lsf == 2 || lsf == 3) {
+ ncoast++;
+ }
+ else {
+ nwater++;
+ }
+
+ }
+ }
+
+// Check for coastlines in current region.
+ if( (nwater + ncoast) == ntotal) {
+
+ if(nwater != ntotal) {
+ *loc_uniform = 0;
+// Provide "double coastlines".
+ *coast = 1;
+ *land = 1;
+ *water = 0;
+ }
+
+ }
+ else if( (nland + ncoast) == ntotal) {
+
+ if(nland != ntotal) *loc_uniform = 0;
+
+ }
+ else {
+
+ *loc_uniform = 0;
+ *coast = 1;
+ *land = 1;
+ *water = 0;
+
+ }
+
+ }
+}
diff --git a/mvcm/c_tools/cithr.c b/mvcm/c_tools/cithr.c
new file mode 100644
index 0000000..93fbd93
--- /dev/null
+++ b/mvcm/c_tools/cithr.c
@@ -0,0 +1,230 @@
+/**********************************************************************
+Description:
+
+ Float function cithr.c
+ Computes the bi-dimensional linear or quadratic interpolation
+ (lagrange form) of a given value to a table of scan angle and 11 um
+ brightness temperature dependent 11-12 micron brightness temperature
+ differences (thin cirrus cloud test thresholds). Current look-up
+ table from J. Key (NOAA, CIMSS, UW-Madison).
+
+Input arguments:
+ int key linear (1) or quadratic (2) interpolation
+ float sec_vza secant of the viewing zenith angle
+ float bt11 11 micron measured brightness temperature (K)
+
+Output arguments:
+ none
+
+ float tci_thr thin cirrus cloud test threshold
+ (returned through function call)
+
+Revision History:
+
+ Original from MODIS Cloud Mask FORTRAN code developed at CIMSS,
+ UW-Madison (tview.f).
+
+ 05/2007 R. Frey Converted to C
+ 10/2012 R. Frey RIncluded in MVCCM software
+
+Calls:
+ none
+
+***********************************************************************/
+
+/* Includes */
+
+#include <stdio.h>
+#include <math.h>
+
+/**********************************************************************/
+
+
+float cithr(int key, float sec_vza, float bt11)
+
+
+/**********************************************************************/
+
+{
+
+/* Declarations */
+
+ int i,i0,i1,i2,ii;
+ int j,j0,j1,j2,jj;
+ int Iflag;
+
+ float lt0,lt1,lt2,lu0,lu1,lu2;
+ float p,p0,p1,p2;
+ float t,t0,t1,t2;
+ float tci_thr;
+ float u,u0,u1,u2;
+
+ float ttab[13] = {190.0, 200.0, 210.0, 220.0, 230.0, 240.0, 250.0,
+ 260.0, 270.0, 280.0, 290.0, 300.0 ,310.0 };
+
+ float utab[5] = { 2.00, 1.75, 1.50, 1.25, 1.00 };
+
+ float tab[5][13] =
+ {.559,.424,.286,.137,.123,.198,.333,.696,1.217,3.184,5.178,8.269,12.452,
+ .542,.416,.294,.162,.156,.240,.366,.704,1.184,2.926,4.854,7.885,11.985,
+ .520,.405,.305,.194,.199,.294,.409,.715,1.141,2.591,4.433,7.389,11.381,
+ .491,.391,.319,.238,.257,.367,.467,.729,1.083,2.140,3.866,6.720,10.567,
+ .450,.370,.340,.300,.340,.470,.550,.750,1.000,1.500,3.060,5.770, 9.410 };
+
+
+/* Initializations */
+
+ lt0 = 0.0;
+ lt1 = 0.0;
+ lt2 = 0.0;
+ lu0 = 0.0;
+ lu1 = 0.0;
+ lu2 = 0.0;
+ p = 0.0;
+ p0 = 0.0;
+ p1 = 0.0;
+ p2 = 0.0;
+ t = 0.0;
+ t0 = 0.0;
+ t1 = 0.0;
+ t2 = 0.0;
+ u = 0.0;
+ u1 = 0.0;
+ u2 = 0.0;
+ i0 = 0;
+ i1 = 0;
+ i2 = 0;
+ j0 = 0;
+ j1 = 0;
+ j2 = 0;
+ jj = 0;
+
+/* Bounds check. */
+
+ u = sec_vza;
+ t = bt11;
+ if (u > utab[0]) u = utab[0];
+ if (u < utab[4]) u = utab[4];
+ if (t < ttab[0]) t = ttab[0];
+ if (t > ttab[12]) t = ttab[12];
+
+ Iflag = 0;
+ i = 1;
+ while(i <= 4) {
+
+ ii = i;
+ if (u >= utab[i] && Iflag == 0) {
+ Iflag = 1;
+ if (key == 1) {
+ i0 = ii - 1;
+ i1 = ii; }
+ else {
+ if (ii == 4) {
+ i0 = ii - 2;
+ i1 = ii - 1;
+ i2 = ii; }
+ else {
+ i0 = ii - 1;
+ i1 = ii;
+ i2 = ii + 1;
+ }
+ }
+ }
+
+ i++;
+
+ }
+
+ Iflag = 0;
+ j = 1;
+ while(j <= 12) {
+
+ jj = j;
+ if (t <= ttab[j] && Iflag == 0) {
+ Iflag = 1;
+ if (key == 1) {
+ j0 = jj - 1;
+ j1 = jj; }
+ else {
+ if (jj == 12) {
+ j0 = jj - 2;
+ j1 = jj - 1;
+ j2 = jj; }
+ else {
+ j0 = jj - 1;
+ j1 = jj;
+ j2 = jj + 1;
+ }
+ }
+ }
+
+ j++;
+
+ }
+
+/* Branch on interpolation method. */
+
+ if (key == 1) {
+
+/* linear scheme */
+/* designate index values */
+
+ u0 = utab[i0];
+ u1 = utab[i1];
+ t0 = ttab[j0];
+ t1 = ttab[j1];
+
+/* lagrange polynomials */
+ lu0 = (u-u1) / (u0-u1);
+ lu1 = (u-u0) / (u1-u0);
+ lt0 = (t-t1) / (t0-t1);
+ lt1 = (t-t0) / (t1-t0);
+
+/* interpolating polynomials for the first dimension */
+ p0 = tab[i0][j0]*lu0 + tab[i1][j0]*lu1;
+ p1 = tab[i0][j1]*lu0 + tab[i1][j1]*lu1;
+
+/* interpolating polynomial for second dimension */
+ p = p0*lt0 + p1*lt1;
+
+// printf("lin cithr: %f %f %f %f %f %f %f\n", u0,u1,t0,t1,p0,p1,p);
+
+ }
+
+ else {
+
+/* quadratic scheme */
+/* designate index values */
+
+ u0 = utab[i0];
+ u1 = utab[i1];
+ u2 = utab[i2];
+ t0 = ttab[j0];
+ t1 = ttab[j1];
+ t2 = ttab[j2];
+
+/* lagrange polynomials */
+ lu0 = (u-u1) * (u-u2) / (u0-u1) / (u0-u2);
+ lu1 = (u-u0) * (u-u2) / (u1-u0) / (u1-u2);
+ lu2 = (u-u0) * (u-u1) / (u2-u0) / (u2-u1);
+ lt0 = (t-t1) * (t-t2) / (t0-t1) / (t0-t2);
+ lt1 = (t-t0) * (t-t2) / (t1-t0) / (t1-t2);
+ lt2 = (t-t0) * (t-t1) / (t2-t0) / (t2-t1);
+
+/* interpolating polynomials for the first dimension */
+ p0 = tab[i0][j0] * lu0 + tab[i1][j0] * lu1 + tab[i2][j0] * lu2;
+ p1 = tab[i0][j1] * lu0 + tab[i1][j1] * lu1 + tab[i2][j1] * lu2;
+ p2 = tab[i0][j2] * lu0 + tab[i1][j2] * lu1 + tab[i2][j2] * lu2;
+
+/* interpolating polynomial for second dimension */
+ p = p0*lt0 + p1*lt1 + p2*lt2;
+
+/* printf("quad cithr: %f %f %f %f\n", p0,p1,p2,p); */
+
+ }
+
+ tci_thr = p;
+
+ return tci_thr;
+
+}
diff --git a/mvcm/c_tools/get_GEOS.c b/mvcm/c_tools/get_GEOS.c
new file mode 100644
index 0000000..2e0692a
--- /dev/null
+++ b/mvcm/c_tools/get_GEOS.c
@@ -0,0 +1,206 @@
+/*******************************************************************************
+Description:
+
+ Integer function get_GEOS.c
+ Opens and reads selected meteorological variables from two GEOS files.
+ The first file ('geos1') is for a time < 3 hours earlier than the target
+ (granule) time. The second file ('geos2') is for a time < 3 hours later.
+ Variables are temporally interpolated to the target time.
+ Read snow and ice cover values from GEOS land and ocean files, respectively.
+ Surface temperature is then spatially interpolated.
+
+ Called from collect_inputs.c
+
+Input arguments:
+ geos1 file name of earlier GEOS data time
+ geos2 file name of later GEOS data time
+ geos_lnd file name of GEOS land properties file (for snow cover)
+ geos_ocn file name of GEOS ocean properties file (for ice cover)
+
+Output arguments:
+ none
+
+Function output:
+ int return_code successful completion is zero, otherwise non-zero
+
+ Output parameters stored in header file ancillary.h
+
+Revision History:
+ 06/2021 R. Frey Original version
+
+Calls:
+ read_GEOS.c
+ get_granule_times.c
+ get_geos_times.c
+ get_ti_weights.c
+ assign_geos_vals.c
+
+Comments:
+ Variables from the earlier GEOS time have names ending in "1" (e.g., 'sfct1').
+ Variables from the later GEOS time have names ending in "2" (e.g., 'sfct2').
+ These are stored in ancillary.h.
+
+
+*******************************************************************************/
+
+// Includes
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+//MVCCM includes
+//#include "granule_geolocation.h"
+//#include "granule.h"
+#include "ancillary.h"
+
+/******************************************************************************/
+
+void get_GEOS(float *lat, float *lon, int res, char *granule_start_time, char *anc_dir, char *geos1, char *geos2, char *geos_lnd,
+ char *geos_ocn, char *geos_cnst, float *tpw, float *snowfr, float *icefr, float *geos_ocnfr,
+ float *landicefr, float *sfct)
+
+{
+
+/******************************************************************************/
+
+// Declarations and initializations.
+
+ extern int get_granule_times(char[64], float*);
+ extern int get_geos_times(char[64], char[64], int*, int*);
+ extern int get_ti_weights(float, int, int, float*);
+ extern int get_ti_vars(float);
+ extern int assign_geos_vals(float *, float *, int);
+ extern int read_GEOS(char[256], int);
+ extern int read_GEOS_lndocn(char[256], char[256]);
+ extern int read_GEOS_constants(char[256]);
+
+ const int eles=3200*res;
+ const int lines=3232*res;
+
+ char geos1_wpath[256];
+ char geos2_wpath[256];
+ char geosland_wpath[256];
+ char geosocean_wpath[256];
+ char geosconstants_wpath[256];
+ char s1[64], s2[64];
+ char search[2];
+ char *token;
+ int irt;
+ int geoshr1;
+ int geoshr2;
+ int GEOS_time_index, ihr, imin;
+ float targhr;
+ float wt;
+
+ int return_code = 0;
+
+/******************************************************************************/
+
+/* Get complete file names. */
+ strcpy(geos1_wpath,anc_dir);strcat(geos1_wpath,"/");strcat(geos1_wpath,geos1);
+ printf("\nGEOS file1: %s\n", geos1_wpath);
+ strcpy(geos2_wpath,anc_dir);strcat(geos2_wpath,"/");strcat(geos2_wpath,geos2);
+ printf("GEOS file2: %s\n", geos2_wpath);
+ strcpy(geosland_wpath,anc_dir);strcat(geosland_wpath,"/");strcat(geosland_wpath,geos_lnd);
+ printf("GEOS LAND file: %s\n", geosland_wpath);
+ strcpy(geosocean_wpath,anc_dir);strcat(geosocean_wpath,"/");strcat(geosocean_wpath,geos_ocn);
+ printf("GEOS OCEAN file: %s\n", geosocean_wpath);
+ strcpy(geosconstants_wpath,anc_dir);strcat(geosconstants_wpath,"/");strcat(geosconstants_wpath,geos_cnst);
+ printf("GEOS CONSTANTS file: %s\n\n", geosconstants_wpath);
+
+/******************************************************************************/
+
+// Read GEOS time-independent (constants) file.
+// Get land/sea tags (fraction ocean) and surface elevation (geopotential ht.).
+ irt = read_GEOS_constants(geosconstants_wpath);
+
+// Read snow and ice fractions from GEOS land and ocean property files.
+// Variables listed in read_GEOS_lndocn.c
+// Pointers to GEOS snow and ice fractions are stored in ancillary.h.
+ irt = read_GEOS_lndocn(geosland_wpath, geosocean_wpath);
+
+// Read selected variables from GEOS 3-hourly analysis files.
+// Variables listed in read_GEOS.c
+// Pointers to GEOS variables are stored in ancillary.h.
+
+ GEOS_time_index = 1;
+ irt = read_GEOS(geos1_wpath, GEOS_time_index);
+ if(irt != 0) {
+ printf("read_GEOS unsuccessful: %s\n", geos1_wpath);
+ // return (-1);
+ }
+
+ GEOS_time_index = 2;
+ irt = read_GEOS(geos2_wpath, GEOS_time_index);
+ if(irt != 0) {
+ printf("read_GEOS unsuccessful: %s\n", geos2_wpath);
+ // return (-2);
+ }
+
+/******************************************************************************/
+
+// Time-interpolate GEOS grids (pointers to data in ancillary.h).
+
+/******************************************************************************/
+
+// Get granule start date, hour, and minute.
+ printf("GEOS interpolation target time from L1b: %s\n", granule_start_time);
+
+ (void) strcpy(search, " ");
+ token = strtok(granule_start_time, search);
+ (void) strcpy(search, ":");
+ token = strtok(NULL, search);
+ ihr = atoi(token);
+ token = strtok(NULL, search);
+ imin = atoi(token);
+ targhr = ihr + (imin / 60.0);
+// printf("hour, min: %d %d %f\n", ihr, imin, targhr);
+
+/******************************************************************************/
+
+// Get GEOS times from file names, at least for now.
+ (void) strcpy(s1, geos1);
+ (void) strcpy(s2, geos2);
+ irt = get_geos_times(s1, s2, &geoshr1, &geoshr2);
+
+/******************************************************************************/
+
+// Calculate time-interpolation weights.
+ irt = get_ti_weights(targhr, geoshr1, geoshr2, &wt);
+ printf("Time-interpolation weights: %f %d %d %f\n", targhr, geoshr1, geoshr2, wt);
+
+/******************************************************************************/
+
+// Perform time interplation. Pointers to inputs and outputs are stored in
+// ancillary.h.
+ irt = get_ti_vars(wt);
+
+/******************************************************************************/
+
+// Assign GEOS variable values to each pixel location in the input L1b granule.
+// Also spatially interpolates GEOS values to pixel locations if desired.
+// Pointers to inputs and outputs are stored in ancillary.h.
+ irt = assign_geos_vals(lat, lon, res);
+
+/******************************************************************************/
+
+ int i;
+ for (i=0; i<lines*eles; i++) {
+ tpw[i] = grn_anc.tpw[i];
+ snowfr[i] = grn_anc.snowfr[i];
+ icefr[i] = grn_anc.icefr[i];
+ geos_ocnfr[i] = grn_anc.geos_ocnfr[i];
+ landicefr[i] = grn_anc.landicefr[i];
+ sfct[i] = grn_anc.sfct[i];
+ }
+ //int i;
+ //
+ //for (i=0; i<10; i++) {
+ // printf("i: %d\ttpw: %f\tsnowfr: %f\ticefr: %f\tgeos_ocnfr: %f\tlandicefr: %f\tsfct: %f\n",
+ // i, tpw[i], snowfr[i], icefr[i], geos_ocnfr[i], landicefr[i], sfct[i]);
+ //}
+ // return ( return_code);
+
+}
+
diff --git a/mvcm/c_tools/get_NDVI_background.c b/mvcm/c_tools/get_NDVI_background.c
new file mode 100644
index 0000000..1a2b28b
--- /dev/null
+++ b/mvcm/c_tools/get_NDVI_background.c
@@ -0,0 +1,184 @@
+/*******************************************************************************
+Description:
+
+ Integer function get_NDVI_background.c
+ Opens NDVI background file and reads the data.
+ Called from collect_inputs.c.
+
+Input arguments:
+ Ancillary data directory
+ Input NDVI file name
+
+Output arguments:
+ none
+
+Function output:
+ int return_code successful completion is zero, otherwise non-zero
+
+ Output parameters in stored in ancillary.h.
+
+Revision History:
+ 10/2012 R. Frey Original version
+
+Calls:
+ none
+
+*******************************************************************************/
+
+// Includes
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+//HDF includes
+#include <hdfi.h>
+#include <hdf.h>
+#include <mfhdf.h>
+#include <HdfEosDef.h>
+
+//MVCCM includes
+//#include "granule_geolocation.h"
+//#include "granule.h"
+//#include "ancillary.h"
+
+/******************************************************************************/
+
+void get_NDVI_background(float *lat, float *lon, int res, char *anc_dir, char *NDVI_file, float *g_ndvi_background)
+
+{
+
+/******************************************************************************/
+
+// Declarations and initializations.
+
+/******************************************************************************/
+
+ char NDVI_file_wpath[256];
+ char NDVI_data[23][80];
+ char SDS_name[10];
+ char * ndvi_sub="/NDVI/";
+ float map_res, wm_lon, nm_lat;
+ //float *g_ndvi_background;
+ long int nncols = 21600;
+ long int nnrows = 10800;
+ long int xindex, yindex, indx, idx;
+ int32 start[2], stride[2], edge[2], sds_index, sds_id, irt, id;
+ int jd, xerr, yerr, row, col, numberRows;
+ int ii, jj;
+ int msgcounta = 0, msgcounta_lim = 100;
+ int msgcountb = 0, msgcountb_lim = 100;
+ int16 *ndvi_arr;
+ int fnday[23] = {1, 17, 33, 49, 65, 81, 97, 113, 129, 145, 161, 177, 193, 209, 225, 241, 257, 273, 289, 305, 321, 337, 353};
+
+ const int eles=3200*res;
+ const int lines=3232*res;
+
+ int return_code = 0;
+
+/******************************************************************************/
+
+// Construct file name.
+
+ (void) strcpy(NDVI_file_wpath, anc_dir);
+ (void) strcat(NDVI_file_wpath, "/");
+ (void) strcat(NDVI_file_wpath, NDVI_file);
+// printf("NDVI file: %s\n", NDVI_file_wpath);
+
+/******************************************************************************/
+
+// Check filename here.
+
+/******************************************************************************/
+
+// Open file. */
+ id = SDstart(NDVI_file_wpath, DFACC_READ);
+ if (id <= 0) {
+ printf("Could not open NDVI file %s\n",NDVI_file_wpath);
+ //return (-1);
+ }
+
+// Select proper SDS and read the data */
+
+ start[0] = 0;
+ start[1] = 0;
+ stride[0] = 1;
+ stride[1] = 1;
+ edge[1] = nncols;
+ edge[0] = nnrows;
+ (void) strcpy(SDS_name, "NDVI");
+
+ ndvi_arr = (int16 *) malloc(21600 * 10800 * 2);
+ sds_index = SDnametoindex(id, SDS_name);
+ sds_id = SDselect(id, sds_index);
+ irt = SDreaddata(sds_id, start, stride, edge, ndvi_arr);
+ if (irt < 0) {
+ printf("Could not read NDVI file %s\n",NDVI_file_wpath);
+ //return (-2);
+ }
+
+// Close NDVI file. */
+ id = SDend(id);
+
+ map_res = 360.0 / (float)(nncols);
+ wm_lon = (map_res / 2.0) - 180.0;
+ nm_lat = 90.0 - (map_res / 2.0);
+
+// Loop through current data and assign NDVI value for each pixel. */
+
+ //g_ndvi_background = (float *)malloc(eles*lines*sizeof(float));
+
+ idx = 0;
+ numberRows = lines;
+ for ( row=0;row<(numberRows );++row) {
+ for ( col=0;col<eles;++col) {
+
+// Initialize.
+ xerr = 0;
+ yerr = 0;
+ g_ndvi_background[idx] = 32.767;
+
+ xindex = (long int)((lon[idx] - wm_lon) / map_res);
+ yindex = (long int)((nm_lat - lat[idx]) / map_res);
+
+ if(xindex < 0 || xindex> nncols) {
+ xerr = 1;
+ if(msgcounta <= msgcounta_lim){
+ printf("xindex out of bounds on NDVI map! \n");
+ msgcounta += 1;
+ }
+ return_code = 1;
+ }
+ if(yindex < 0 || yindex> nnrows) {
+ yerr = 1;
+ if(msgcountb <= msgcountb_lim){
+ printf("yindex out of bounds on NDVI map! \n");
+ msgcountb += 1;
+ }
+ return_code = 2;
+ }
+
+ if(xerr == 0 && yerr == 0) {
+ indx = (yindex*nncols) + xindex;
+ g_ndvi_background[idx] = *(ndvi_arr + indx) * 0.001;
+
+// if (idx < 10) {
+// printf("ndvi %d %d %ld %ld %ld %f %f %d %f\n", row, col, xindex, yindex, indx, grn_geo.lat[idx], grn_geo.lon[idx], *(ndvi_arr + indx), g_ndvi_background[idx]);
+// }
+
+ }
+
+ idx++;
+
+ }
+ }
+
+ //grn_anc.ndvibk = g_ndvi_background;
+
+ free (ndvi_arr);
+
+/******************************************************************************/
+
+// return (return_code);
+
+}
diff --git a/mvcm/c_tools/get_Olson_eco.c b/mvcm/c_tools/get_Olson_eco.c
new file mode 100644
index 0000000..31f748f
--- /dev/null
+++ b/mvcm/c_tools/get_Olson_eco.c
@@ -0,0 +1,141 @@
+/*******************************************************************************
+Description:
+
+ integer function get_Olson_eco.c
+ Opens and reads Olson ecosystem file.
+ Called from collect_inputs.c.
+
+Input arguments:
+ anc_dir ancillary data directory
+
+Output arguments:
+ none
+
+Function output:
+ int return_code successful completion is zero, otherwise non-zero
+
+ Input file name (static) defined here.
+
+ Output stored in header file ancillary.h.
+
+Revision History:
+ 10/2012 R. Frey Original version
+
+Calls:
+ none
+
+*******************************************************************************/
+
+// Includes
+
+/******************************************************************************/
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+//MVCCM includes
+//#include "granule_geolocation.h"
+//#include "granule.h"
+//#include "ancillary.h"
+
+/******************************************************************************/
+
+int get_Olson_eco(float *lat, float *lon, int res, char *anc_dir, unsigned char *granule_ecotype)
+
+{
+
+/******************************************************************************/
+
+// Declarations and initializations.
+//
+/******************************************************************************/
+
+ extern long int lround(double);
+ extern void getcoord_(double*, double*, double*, double*);
+
+ char eco_file[256];
+ unsigned char eco_index;
+ //unsigned char *granule_ecotype;
+ double lt, ln, ltt, lnn;
+ double drow;
+ double dcol;
+ long int idx, irow, icol, byte_loc;
+ long int max_row = 17347;
+ long int max_col = 40031;
+ int row, col, numberRows;
+
+ const int eles=3200*res;
+ const int lines=3232*res;
+
+ size_t bytes_read;
+ FILE *eco_file_ptr;
+
+// Define static ecosystem file name.
+ char eco_file_name[] = "goge1_2_img.v1";
+
+ int return_code = 0;
+
+/******************************************************************************/
+
+/* Get file name */
+ (void) strcpy(eco_file, anc_dir);
+ (void) strcat(eco_file, "/");
+ (void) strcat(eco_file, eco_file_name);
+// printf("ecosystem file: %s \n", eco_file);
+
+/* Open file */
+ eco_file_ptr = fopen(eco_file, "r");
+ if(eco_file_ptr == NULL) {
+ printf("Cannot open ecosystem file %s \n", eco_file);
+ //return (-1);
+ }
+
+ //granule_ecotype = (unsigned char *)malloc(eles*lines*sizeof(unsigned char));
+ numberRows = lines;
+ idx = 0;
+ for ( row=0; row<(numberRows); ++row) {
+ for ( col=0; col<eles ;++col) {
+
+// Get lat/lon
+ lt = (double) lat[idx];
+ ln = (double) lon[idx];
+ ltt = lt;
+ lnn = ln;
+
+// Get row/col of data grid corresponding to lat/lon.
+ (void) getcoord_(<, &ln, &drow, &dcol);
+
+ irow = lround(drow);
+ if(irow > max_row) irow = max_row;
+ if(irow < 1) irow = 1;
+ icol = lround(dcol);
+ if(icol > max_col) icol = max_col;
+ if(icol < 1) icol = 1;
+
+ byte_loc = ( ((irow-1) * max_col) + icol) - 1;
+
+// Read file
+ fseek(eco_file_ptr, byte_loc, SEEK_SET);
+ bytes_read = fread((char *) &eco_index, sizeof(eco_index), 1, eco_file_ptr);
+
+ granule_ecotype[idx] = eco_index;
+
+// if(idx < 10)
+// printf("Olson ecosystem index: %d %d\n", idx, eco_index);
+
+ ++idx;
+
+ }
+ }
+
+/* Close file */
+ (void) fclose(eco_file_ptr);
+
+ //grn_anc.eco = granule_ecotype;
+
+/******************************************************************************/
+
+ //return (return_code);
+
+}
diff --git a/mvcm/c_tools/get_Reynolds_SST.c b/mvcm/c_tools/get_Reynolds_SST.c
new file mode 100644
index 0000000..b288e6f
--- /dev/null
+++ b/mvcm/c_tools/get_Reynolds_SST.c
@@ -0,0 +1,162 @@
+/*******************************************************************************
+Description:
+
+ Integer function get_Reynolds_SST
+ Opens and reads SSTs from Reynolds SST files.
+ Called from collect_inputs.c.
+
+Input arguments:
+ ancillary data directory
+ Reynolds SST file name
+
+Output arguments:
+ none
+
+Function output:
+ int return_code successful completion is zero, otherwise non-zero
+
+ Output parameters stored in ancillary.h
+
+Revision History:
+ 10/2012 R. Frey Original version
+
+Calls:
+ bilinearInterpSST
+ swapbyte4
+
+*******************************************************************************/
+
+// Includes
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+//MVCCM includes
+//#include "granule.h"
+#include "sst.h"
+
+/******************************************************************************/
+
+void get_Reynolds_SST(float *lat, float *lon, int res, char *anc_dir, char *SST_file, float *sstInterp)
+
+{
+
+/******************************************************************************/
+
+// Declarations and initializations.
+
+/******************************************************************************/
+
+ extern int bilinearInterpSST(int, int, float, float, const SST *, float *);
+ extern int swapbyte4(void *, int);
+ extern float fmaxf(float, float);
+ extern float fminf(float, float);
+
+ /* number of points around a latitude circle */
+ long int npoints_x = 360;
+ /* number of grid points along a meridian */
+ long int npoints_y = 180;
+ /* the 2 possible sizes of an SST file (formatted/unformatted) */
+ long int unfSize = 583264;
+ char sst_file_wpath[256];
+
+ char fileFormat[4] = "unk";
+ float xlon, ytemp;
+ //float * sstInterp;
+ float tempVal;
+ long int fileSize;
+ long int indx;
+ int i, j, xindx, yindx;
+ int ok;
+
+ const int eles=3200*res;
+ const int lines=3232*res;
+
+ FILE * sst_file_ptr;
+ size_t bytes_read;
+ SST sstGrid;
+
+ int return_code = 0;
+
+/******************************************************************************/
+
+ strcpy(sst_file_wpath, anc_dir);
+ strcat(sst_file_wpath, "/");
+ strcat(sst_file_wpath, SST_file);
+// printf("SST file: %s \n", sst_file_wpath);
+
+ /* Open file */
+ sst_file_ptr = fopen(sst_file_wpath, "rb");
+ if(sst_file_ptr == NULL) {
+ printf("Cannot open Reynolds SST file %s\n", sst_file_wpath);
+ // return (-1);
+ }
+ /* Get the file size */
+ fseek(sst_file_ptr, 0, SEEK_END);
+ fileSize = ftell(sst_file_ptr);
+ fseek(sst_file_ptr, 0, SEEK_SET);
+
+ /* read if file is unformatted only */
+ if ( fileSize == unfSize ) {
+ strcpy(fileFormat,"unf");
+ bytes_read = fread(&sstGrid, sizeof(sstGrid), 1, sst_file_ptr);
+ (void) swapbyte4(&sstGrid, 12 + npoints_x * npoints_y);
+ }
+ else {
+ /* Close file */
+ (void) fclose(sst_file_ptr);
+ // return (-1);
+ }
+ /* Close file */
+ (void) fclose(sst_file_ptr);
+
+ // to make it work with cython I don't need to allocate memory... I guess it's because cython
+ // already does it?!?
+ //sstInterp = (float *) malloc(eles*lines*sizeof(float));
+
+
+ indx = 0;
+ for ( i=0; i < lines; ++i) {
+ for ( j=0; j < eles; ++j) {
+
+ xindx = 0;
+ yindx = 0;
+ ok = -1;
+ sstInterp[indx] = 0.0;
+
+// if(imgr1->landmask[indx] != 1) {
+
+ if (lon[indx] < 0.0)
+ xlon = lon[indx] + 360.0;
+ else
+ xlon = lon[indx];
+
+ xindx = (int) fminf(fmaxf(xlon,0.0),359.99);
+ ytemp = fminf(fmaxf(lat[indx],-89.99), 89.99);
+ yindx = (int) (ytemp + 90.0);
+
+ ok = bilinearInterpSST(xindx,yindx,lat[indx],xlon,&sstGrid,&tempVal);
+
+ if (ok == 0) {
+ sstInterp[indx] = tempVal + 273.15;
+ }
+
+ if(ok != 0)
+ printf("Bad SST interpolation: %d %d %d %d %d %d %f\n", indx, i, j, xindx, yindx, ok, sstInterp[indx]);
+
+// }
+
+// if(indx < 10)
+// printf("SSTs: %d %d %d %d %f\n", indx, xindx, yindx, ok, sstInterp[indx]);
+
+ indx++;
+ }
+ }
+ // grn_anc.reynSST = sstInterp;
+
+/******************************************************************************/
+
+ //return (return_code);
+
+}
diff --git a/mvcm/c_tools/get_b1_thresholds.c b/mvcm/c_tools/get_b1_thresholds.c
new file mode 100644
index 0000000..694345a
--- /dev/null
+++ b/mvcm/c_tools/get_b1_thresholds.c
@@ -0,0 +1,191 @@
+/**********************************************************************
+Description:
+
+ Integer function get_b1_thresholds.c
+
+ Computes band 1 (0.66 micron) cloud test thresholds for daytime land
+ conditions.
+ Computes band 8 (0.412 micron) cloud test thresholds for daytime
+ desert conditions.
+
+Input arguments:
+ none
+
+ Input values stored in variables found in pixel.h
+
+Output arguments:
+ locut Zero confidence value of clear sky confidence interval
+ hicut 1.0 confidence value of clear sky confidence interval
+ midpt 0.5 confidence value of clear sky confidence interval
+ power Power of clear sky confidence curve
+
+Revision History:
+ 10/2008 R. Frey Converted to C from FORTRAN
+ 10/2012 R. Frey Included in MVCCM software
+
+calls:
+ none
+
+***********************************************************************/
+
+/* Includes */
+
+#include <stdio.h>
+#include <math.h>
+#include "thresholds.h"
+#include "pixel.h"
+
+/**********************************************************************/
+
+
+int get_b1_thresholds(float *locut, float *hicut, float *midpt,
+ float *power)
+
+
+/**********************************************************************/
+
+{
+
+/* Declarations */
+
+ extern float powf(float, float);
+
+ int interp=0;
+ int a, i, j, k, indvi, icnf, irtn;
+
+ static float delta_ndvi_bin = 0.1;
+ float x, x1, x2, y1, y2;
+ float coeff[10][3][4], thr[3], thr_adj[3];
+
+// Transfer all coefficients to one array.
+
+ i = 0;
+
+ for (j=0; j<3; j++) {
+ for( k=0; k<4; k++) {
+
+ if(pxin.ndvibk < des_ndvi[0]) {
+
+ coeff[0][j][k] = b8ndvi0[i];
+ coeff[1][j][k] = b8ndvi1[i];
+ coeff[2][j][k] = b8ndvi2[i];
+// printf("coeffs8: %d %d %d %f %f\n",i, j, k, coeff[2][j][k], b8ndvi2[i]);
+
+ }
+ else {
+
+ coeff[0][j][k] = b1ndvi0[i];
+ coeff[1][j][k] = b1ndvi1[i];
+ coeff[2][j][k] = b1ndvi2[i];
+ coeff[3][j][k] = b1ndvi3[i];
+ coeff[4][j][k] = b1ndvi4[i];
+ coeff[5][j][k] = b1ndvi5[i];
+ coeff[6][j][k] = b1ndvi6[i];
+ coeff[7][j][k] = b1ndvi7[i];
+ coeff[8][j][k] = b1ndvi8[i];
+ coeff[9][j][k] = b1ndvi9[i];
+// printf("coeffs1: %d %d %d %f %f\n",i, j, k, coeff[2][j][k], b1ndvi2[i]);
+
+ }
+
+ i++;
+
+ }
+ }
+
+// printf("\nband 1, 8 coefficients initialized \n");
+
+/* Compute thresholds for current pixel as functions of background
+ NDVI and scattering angle.
+*/
+
+// Check for special cases and fill values ('ndvibk'=32.767).
+ if(pxin.ndvibk < fill_ndvi1[0] && pxin.ndvibk > fill_ndvi2[0]) {
+
+ if(pxin.ndvibk < ndvi_bnd1[0]) {
+ indvi = 0;
+ x = 0.0;
+ y2 = 0.0;
+ interp = 0;
+ }
+ else if(pxin.ndvibk >= ndvi_bnd2[0]) {
+ indvi = 9;
+ x = 0.0;
+ y2 = 0.0;
+ interp = 0;
+ }
+ else {
+ interp = 1;
+ }
+
+ if(interp) {
+ a = ((pxin.ndvibk / delta_ndvi_bin) - 0.5);
+ if(a < 0) {
+ indvi = 0;
+ }
+ else {
+ indvi = a;
+ }
+// indvi = min(nbin_ndvi, max(0, int((pxin.ndvibk / delta_ndvi_bin) - 0.5)) );
+
+ x1 = delta_ndvi_bin*indvi + delta_ndvi_bin/2.0;
+ x2 = x1 + delta_ndvi_bin;
+ x = (pxin.ndvibk - x1) / ( x2 - x1 );
+ if(x < 0.0) x = 0.0;
+ if(x > 1.0) x = 1.0;
+
+ }
+
+ for( icnf=0; icnf<3; icnf++) {
+
+ y1 = coeff[indvi][icnf][0] +
+ (coeff[indvi][icnf][1] * pxin.sctang) +
+ (coeff[indvi][icnf][2] * powf(pxin.sctang,2)) +
+ (coeff[indvi][icnf][3] * powf(pxin.sctang,3));
+
+ if(interp) {
+ y2 = coeff[indvi+1][icnf][0] +
+ (coeff[indvi+1][icnf][1] * pxin.sctang) +
+ (coeff[indvi+1][icnf][2] * powf(pxin.sctang,2)) +
+ (coeff[indvi+1][icnf][3] * powf(pxin.sctang,3));
+ }
+
+ thr[icnf] = ((1.0 - x) * y1) + (x * y2);
+ if(pxin.ndvibk < des_ndvi[0]) {
+ thr_adj[icnf] = thr[icnf] * thr_adj_fac_desert[0];
+ }
+ else {
+ thr_adj[icnf] = thr[icnf] * thr_adj_fac_land[0];
+ }
+
+// printf("ndvi thr: %f %f %f %f %f %f %f\n", des_ndvi[0], fill_ndvi1[0], fill_ndvi2[0],
+// ndvi_bnd1[0], ndvi_bnd2[0], thr_adj_fac_desert[0], thr_adj_fac_land[0]);
+// printf("coeffs: %f %f %f %f %f %f %f %f\n", coeff[indvi][icnf][0], coeff[indvi][icnf][1],
+// coeff[indvi][icnf][2], coeff[indvi][icnf][3], coeff[indvi+1][icnf][0], coeff[indvi+1][icnf][1],
+// coeff[indvi+1][icnf][2], coeff[indvi+1][icnf][3]);
+// printf("interpolation: %d %d %f %f %f %f %f\n", indvi, icnf, pxin.sctang, y1, y2, thr[icnf], thr_adj[icnf]);
+
+ }
+
+ *(hicut) = (thr[0] + thr_adj[0]) / 100.0;
+ *(midpt) = (thr[1] + thr_adj[1]) / 100.0;
+ *(locut) = (thr[2] + thr_adj[2]) / 100.0;
+ *(power) = 2.0;
+
+ irtn = 0;
+
+ }
+ else {
+
+// No NDVI-dependent thresholds can be derived for the current pixel.
+ irtn = -1;
+
+ }
+
+// printf("\nSubroutine get_b1_thresholds\n");
+// printf("%d %d %d %f %f %f %f %f %f %f %f \n", irtn, interp, indvi, pxin.ndvibk, pxin.sctang,
+// x1, x2, x, *(locut), *(midpt), *(hicut));
+
+ return irtn;
+
+}
diff --git a/mvcm/c_tools/get_geos_times.c b/mvcm/c_tools/get_geos_times.c
new file mode 100644
index 0000000..447fc5b
--- /dev/null
+++ b/mvcm/c_tools/get_geos_times.c
@@ -0,0 +1,81 @@
+/*******************************************************************************
+Description:
+
+ Integer function get_geos_times.c
+ Parses GEOS hour from filename.
+ Called from get_GEOS.c
+
+Input arguments:
+ s1 earlier GEOS file name
+ s2 later GEOS file name
+
+Output arguments:
+ geoshr1 integer time for earlier GEOS file
+ geoshr2 integer time for later GEOS file
+
+Function output:
+ int return_code successful completion is zero, otherwise non-zero
+
+Revision History:
+ 10/2012 R. Frey Original version
+
+Calls:
+ none
+
+*******************************************************************************/
+
+// Includes
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+/******************************************************************************/
+
+int get_geos_times(char s1[64], char s2[64], int *geoshr1, int *geoshr2)
+
+{
+
+/******************************************************************************/
+
+// Declarations and initializations.
+
+ char search[4];
+ char *geos_time = "0";
+ char *token;
+
+ int return_code = 0;
+
+/******************************************************************************/
+
+// Get GEOS start hours.
+
+// printf("get time 1: %s\n", s1);
+// printf("get time 2: %s\n", s2);
+
+ (void) strcpy(search, "_");
+ token = strtok(s1, search);
+ token = strtok(NULL, search);
+ token = strtok(NULL, search);
+ token = strtok(NULL, search);
+ token = strtok(NULL, search);
+ (void) strcpy(search, "00.");
+ geos_time = strtok(token, search);
+ *(geoshr1) = atoi(geos_time);
+
+ (void) strcpy(search, "_");
+ token = strtok(s2, search);
+ token = strtok(NULL, search);
+ token = strtok(NULL, search);
+ token = strtok(NULL, search);
+ token = strtok(NULL, search);
+ (void) strcpy(search, "00.");
+ geos_time = strtok(token, search);
+ *(geoshr2) = atoi(geos_time);
+
+/******************************************************************************/
+
+ return (return_code);
+
+/******************************************************************************/
+
+}
diff --git a/mvcm/c_tools/get_granule_times.c b/mvcm/c_tools/get_granule_times.c
new file mode 100644
index 0000000..ef97765
--- /dev/null
+++ b/mvcm/c_tools/get_granule_times.c
@@ -0,0 +1,76 @@
+/*******************************************************************************
+Description:
+
+ Integer function get_granule_times.c
+ Parses granule start date, hour, and minute from IFF attribute.
+ Called from get_GDAS.c
+
+Input arguments:
+ s1 granule start time string from IFF attribute
+
+Output arguments:
+ targhr decimal granule start hour
+ (target hour for time interplation of GDAS
+ variables)
+
+Function output:
+ int return_code successful completion is zero, otherwise non-zero
+
+Revision History:
+ 10/2012 R. Frey Original version
+
+Calls:
+ none
+
+*******************************************************************************/
+
+// Includes
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+/******************************************************************************/
+
+int get_granule_times(char s1[64], float *targhr)
+
+{
+
+/******************************************************************************/
+
+// Declarations and initializations.
+
+ char s2[64];
+ char search[2];
+ char *start_time = "0";
+ char *start_date = "0";
+ char *start_hour = "0";
+ char *start_min = "0";
+ int imin;
+
+ int return_code = 0;
+
+/******************************************************************************/
+
+// Get granule start date, hour, and minute.
+
+ (void) strcpy(search, " ");
+ start_date = strtok(s1, search);
+ start_time = strtok(NULL, search);
+
+ (void) strcpy(s2, start_time);
+ (void) strcpy(search, ":");
+ start_hour = strtok(s2, search);
+ start_min = strtok(NULL, search);
+ printf("\nstart date, hour and minute: %s %s %s\n", start_date, start_hour, start_min);
+
+// Set decimal target hour for time interpolation.
+ imin = atoi(start_min);
+ *(targhr) = (float)atoi(start_hour) + ((float)imin / 60.0);
+
+/******************************************************************************/
+
+ return (return_code);
+
+/******************************************************************************/
+
+}
diff --git a/mvcm/c_tools/get_ti_vars.c b/mvcm/c_tools/get_ti_vars.c
new file mode 100644
index 0000000..b48470f
--- /dev/null
+++ b/mvcm/c_tools/get_ti_vars.c
@@ -0,0 +1,91 @@
+/*******************************************************************************
+Description:
+
+ Integer function get_ti_vars_geos.c
+ Calculates time-interpolated variables from the two GEOS times.
+ Called from get_GEOS.c
+
+Input arguments:
+ wt time interpolation weight for later GEOS time
+
+Output arguments:
+ none
+
+Function output:
+ int return_code successful completion is zero, otherwise non-zero
+
+ Inputs are taken from pointers stored in ancillary.h.
+ Pointers to time-interpolated variables are stored in ancillary.h.
+
+Revision History:
+ 10/2012 R. Frey Original version
+
+Calls:
+ none
+
+*******************************************************************************/
+
+// Includes
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+//MVCCM includes
+#include "ancillary.h"
+
+/******************************************************************************/
+
+int get_ti_vars(float wt)
+
+{
+
+/******************************************************************************/
+
+// Declarations and initializations.
+
+ long int n;
+ int return_code = 0;
+
+/******************************************************************************/
+
+// Allocate memory for time-interpolated data.
+ if ((ti_sfct = (float *) malloc(sizeof(float) * nwp_cells)) == NULL) {
+ printf("Could not allocate memory for ti_sfct\n");
+ return(-2);
+ }
+ if ((ti_tpw = (float *) malloc(sizeof(float) * nwp_cells)) == NULL) {
+ printf("Could not allocate memory for ti_tpw\n");
+ return(-4);
+ }
+
+/******************************************************************************/
+
+ for (n=0; n<nwp_cells; n++) {
+
+ ti_sfct[n] = (g_sfct1[n] >= 0.0 && g_sfct2[n] >= 0.0) ?
+ (wt*(g_sfct2[n]) + (1.0-wt)*(g_sfct1[n])) : -999.99;
+// if(n >= 1000 && n <= 1009)
+// printf("ti sfc t: %ld %f %f %f %f\n", n, wt, g_sfct1[n], g_sfct2[n], ti_sfct[n]);
+
+ ti_tpw[n] = (g_tpw1[n] >= 0.0 && g_tpw2[n] >= 0.0) ?
+ (wt*(g_tpw2[n]/10.0) + (1.0-wt)*(g_tpw1[n]/10.0)) : -999.99;
+// if(n >= 1000 && n <= 1009)
+// printf("ti tpw: %ld %f %f %f %f\n", n, wt, g_tpw1[n], g_tpw2[n], ti_tpw[n]);
+
+ }
+
+/******************************************************************************/
+
+// Free unneeded memory.
+ free(g_sfct1);
+ free(g_sfct2);
+ free(g_tpw1);
+ free(g_tpw2);
+
+/******************************************************************************/
+
+ return (return_code);
+
+/******************************************************************************/
+
+}
diff --git a/mvcm/c_tools/get_ti_vars_geos.c b/mvcm/c_tools/get_ti_vars_geos.c
new file mode 100644
index 0000000..b48470f
--- /dev/null
+++ b/mvcm/c_tools/get_ti_vars_geos.c
@@ -0,0 +1,91 @@
+/*******************************************************************************
+Description:
+
+ Integer function get_ti_vars_geos.c
+ Calculates time-interpolated variables from the two GEOS times.
+ Called from get_GEOS.c
+
+Input arguments:
+ wt time interpolation weight for later GEOS time
+
+Output arguments:
+ none
+
+Function output:
+ int return_code successful completion is zero, otherwise non-zero
+
+ Inputs are taken from pointers stored in ancillary.h.
+ Pointers to time-interpolated variables are stored in ancillary.h.
+
+Revision History:
+ 10/2012 R. Frey Original version
+
+Calls:
+ none
+
+*******************************************************************************/
+
+// Includes
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+//MVCCM includes
+#include "ancillary.h"
+
+/******************************************************************************/
+
+int get_ti_vars(float wt)
+
+{
+
+/******************************************************************************/
+
+// Declarations and initializations.
+
+ long int n;
+ int return_code = 0;
+
+/******************************************************************************/
+
+// Allocate memory for time-interpolated data.
+ if ((ti_sfct = (float *) malloc(sizeof(float) * nwp_cells)) == NULL) {
+ printf("Could not allocate memory for ti_sfct\n");
+ return(-2);
+ }
+ if ((ti_tpw = (float *) malloc(sizeof(float) * nwp_cells)) == NULL) {
+ printf("Could not allocate memory for ti_tpw\n");
+ return(-4);
+ }
+
+/******************************************************************************/
+
+ for (n=0; n<nwp_cells; n++) {
+
+ ti_sfct[n] = (g_sfct1[n] >= 0.0 && g_sfct2[n] >= 0.0) ?
+ (wt*(g_sfct2[n]) + (1.0-wt)*(g_sfct1[n])) : -999.99;
+// if(n >= 1000 && n <= 1009)
+// printf("ti sfc t: %ld %f %f %f %f\n", n, wt, g_sfct1[n], g_sfct2[n], ti_sfct[n]);
+
+ ti_tpw[n] = (g_tpw1[n] >= 0.0 && g_tpw2[n] >= 0.0) ?
+ (wt*(g_tpw2[n]/10.0) + (1.0-wt)*(g_tpw1[n]/10.0)) : -999.99;
+// if(n >= 1000 && n <= 1009)
+// printf("ti tpw: %ld %f %f %f %f\n", n, wt, g_tpw1[n], g_tpw2[n], ti_tpw[n]);
+
+ }
+
+/******************************************************************************/
+
+// Free unneeded memory.
+ free(g_sfct1);
+ free(g_sfct2);
+ free(g_tpw1);
+ free(g_tpw2);
+
+/******************************************************************************/
+
+ return (return_code);
+
+/******************************************************************************/
+
+}
diff --git a/mvcm/c_tools/get_ti_weights.c b/mvcm/c_tools/get_ti_weights.c
new file mode 100644
index 0000000..607489d
--- /dev/null
+++ b/mvcm/c_tools/get_ti_weights.c
@@ -0,0 +1,69 @@
+/*******************************************************************************
+Description:
+
+ Integer function get_ti_weights_geos.c
+ Calculates time-interpolation weights from L1b start time and the two GEOS
+ file times.
+ Computed weight is for the later of two GEOS grids. The weight of the earlier
+ one is simply (1.0 - weight).
+ Called from get_GEOS.c
+
+Input arguments:
+ targhr decimal start time of input L1b data
+ gdashr1 integer hour of earlier GEOS file
+ gdashr2 integer hour of later GEOS file
+
+Output arguments:
+ wt time-interpolation weight for later GEOS time
+
+Function output:
+ int return_code successful completion is zero, otherwise non-zero
+
+Revision History:
+ 10/2012 R. Frey Original version
+ 06/2021 R. Frey Modified to use GMAO GEOS files.
+
+Calls:
+ none
+
+*******************************************************************************/
+
+// Includes
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+/******************************************************************************/
+
+int get_ti_weights(float targhr, int geoshr1, int geoshr2, float *wt)
+
+{
+
+/******************************************************************************/
+
+// Declarations and initializations.
+
+ int return_code = 0;
+
+/******************************************************************************/
+
+// Calculate time-interpolation weight.
+
+ if(geoshr2 > geoshr1) *(wt) = (targhr-(float)geoshr1) / ((float)geoshr2-(float)geoshr1);
+ if(geoshr2 == geoshr1) *(wt) = 0.0;
+ if(geoshr2 < geoshr1) {
+ geoshr2 += 24;
+ *(wt) = (targhr-(float)geoshr1) / ((float)geoshr2-(float)geoshr1);
+ if( *(wt) < 0.0) {
+ printf("GEOS times are incorrect! T-I weight: %f %d %d %f\n", targhr, geoshr1, geoshr2, *(wt));
+ return_code = -1;
+ }
+ }
+
+/******************************************************************************/
+
+ return (return_code);
+
+/******************************************************************************/
+
+}
diff --git a/mvcm/c_tools/get_ti_weights_geos.c b/mvcm/c_tools/get_ti_weights_geos.c
new file mode 100644
index 0000000..607489d
--- /dev/null
+++ b/mvcm/c_tools/get_ti_weights_geos.c
@@ -0,0 +1,69 @@
+/*******************************************************************************
+Description:
+
+ Integer function get_ti_weights_geos.c
+ Calculates time-interpolation weights from L1b start time and the two GEOS
+ file times.
+ Computed weight is for the later of two GEOS grids. The weight of the earlier
+ one is simply (1.0 - weight).
+ Called from get_GEOS.c
+
+Input arguments:
+ targhr decimal start time of input L1b data
+ gdashr1 integer hour of earlier GEOS file
+ gdashr2 integer hour of later GEOS file
+
+Output arguments:
+ wt time-interpolation weight for later GEOS time
+
+Function output:
+ int return_code successful completion is zero, otherwise non-zero
+
+Revision History:
+ 10/2012 R. Frey Original version
+ 06/2021 R. Frey Modified to use GMAO GEOS files.
+
+Calls:
+ none
+
+*******************************************************************************/
+
+// Includes
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+/******************************************************************************/
+
+int get_ti_weights(float targhr, int geoshr1, int geoshr2, float *wt)
+
+{
+
+/******************************************************************************/
+
+// Declarations and initializations.
+
+ int return_code = 0;
+
+/******************************************************************************/
+
+// Calculate time-interpolation weight.
+
+ if(geoshr2 > geoshr1) *(wt) = (targhr-(float)geoshr1) / ((float)geoshr2-(float)geoshr1);
+ if(geoshr2 == geoshr1) *(wt) = 0.0;
+ if(geoshr2 < geoshr1) {
+ geoshr2 += 24;
+ *(wt) = (targhr-(float)geoshr1) / ((float)geoshr2-(float)geoshr1);
+ if( *(wt) < 0.0) {
+ printf("GEOS times are incorrect! T-I weight: %f %d %d %f\n", targhr, geoshr1, geoshr2, *(wt));
+ return_code = -1;
+ }
+ }
+
+/******************************************************************************/
+
+ return (return_code);
+
+/******************************************************************************/
+
+}
diff --git a/mvcm/c_tools/getcoord.c b/mvcm/c_tools/getcoord.c
new file mode 100644
index 0000000..8b5872b
--- /dev/null
+++ b/mvcm/c_tools/getcoord.c
@@ -0,0 +1,650 @@
+/*
+!C **********************************************************************
+!Description:
+ GIHLS211 transforms latitude, longitude coordinates to line, sample
+ Renamed GIHLS211 to GETCOORD !!
+ coordinates for an image in the Goode's Interrupted Homolosine
+ projection. This routine was complied and run using the C compiler on
+ SunOS 4.2 (UNIX). Results were accurate at the time of testing, but they
+ are by no means guaranteed!
+
+!Input Parameters:
+lt latitude (double precision)
+ln longitude (double precision)
+
+!Output Parameters:
+lne line number of 1km mask for given latitude/longitude
+smp element number of 1km maks for given latitude/longitude
+
+!Revision History:
+
+!Team-unique Header:
+
+!References and Credits:
+
+1. Snyder, John P. and Voxland, Philip M., "An Album of Map Projections",
+ U.S. Geological Survey Professional Paper 1453 , United State Government
+ Printing Office, Washington D.C., 1989.
+
+2. Snyder, John P., "Map Projections--A Working Manual", U.S. Geological
+ Survey Professional Paper 1395 (Supersedes USGS Bulletin 1532), United
+ State Government Printing Office, Washington D.C., 1987.
+
+3. Goode, J.P., 1925, The Homolosine projection: a new device for
+ portraying the Earth's surface entire: Assoc. Am. Geographers, Annals,
+ v. 15, p. 119-125
+
+4. Steinwand, Daniel R., "Mapping Raster Imagery to the Interrupted
+ Goode Homolosine Projection", 1993. In press--IJRS.
+
+!Design Notes:
+!END************************************************************************/
+#include <stdio.h>
+#include <math.h>
+
+
+/* Modified by JC Guu 08/27/96 */
+/* function prototype is given here for the */
+/* functions in this module. */
+
+extern void getcoord_(double *lt, double *ln, double *lne, double *smp);
+/*void getcoord_(double *lt, double *ln, double *lne, double *smp);*/
+void bad_input_parms();
+void goode_init(double r);
+int goode_forward(double lon, double lat, double* x, double* y);
+/*
+void ptitle(char* A);
+void radius(double A);
+*/
+void giherror(char* what, char* where);
+void getcoord_sincos(double val, double* sin_val, double* cos_val);
+int getcoord_sign(double x);
+double getcoord_adjust_lon(double x);
+double getcoord_e0fn(double x);
+double getcoord_e1fn(double x);
+double getcoord_e2fn(double x);
+double getcoord_mlfn(double e0,double e1,double e2,double phi);
+
+
+#define PI 3.141592653589793238
+#define HALF_PI PI*0.5
+#define TWO_PI PI*2.0
+#define EPSLN 1.0e-10
+#define R2D 57.2957795131
+#define D2R 0.0174532925199
+
+#define OK 1
+#define ERROR -1
+#define IN_BREAK -2
+
+/* Variables common to all subroutines in this code file
+ -----------------------------------------------------*/
+static double R; /* Radius of the earth (sphere) */
+static double lon_center[12]; /* Central meridians, one for each region */
+static double feast[12]; /* False easting, one for each region */
+
+/* Transformation routine
+ -------------------------------*/
+void getcoord_(double *lt, double *ln, double *lne, double *smp)
+{
+float pixsiz;
+double x,y;
+double lat, lon, line, samp;
+/*
+int nl,ns;
+*/
+double ul_x, ul_y;
+
+/* Modified by JC Guu 08/30/96 */
+/* The variable status is added here to */
+/* process the return code of */
+/* goode_forward(). */
+
+int status;
+
+/* Modified by JC Guu 08/26/96 */
+/* The type of constant is specified. */
+
+pixsiz = 1000.0F;
+
+/* Report parameters and image geometric characteristics to the user
+ -----------------------------------------------------------------*/
+/*printf("Converting latitude, longitude to line, sample coordinates\n\n");
+printf("Pixel size is %f km\n\n", pixsiz/1000.0); */
+
+/* Modified by JC Guu 08/26/96 */
+/* The equality comparision */
+/* "pixsiz == 1000.0" is changed to an */
+/* inequality comparision */
+/* "fabs(pixsiz-1000.0F) < 0.000001F". */
+
+
+if (fabs(pixsiz-1000.0F) < 0.000001F) {
+ ul_x = -20015000.0;
+ ul_y = 8673000.0;
+/*
+ nl = 17347;
+ ns = 40031;
+*/
+ }
+else {
+ ul_x = -20011500.0;
+ ul_y = 8669500.0;
+/*
+ nl = 2168;
+ ns = 5004;
+*/
+ }
+/*printf("Image size is %d lines by %d samples with an upper left\n",nl,ns);
+printf("corner of UL_X = %lf and UL_Y = %lf meters.\n", ul_x, ul_y); */
+
+/* Initialize the Interrupted Goode Homolosine projection
+ ------------------------------------------------------*/
+goode_init(6370997.0);
+
+/* Process point
+ ------------------*/
+ lat = *lt;
+ lon = *ln;
+/* printf("%12.6f %12.6f\n",lat, lon);*/
+ lat *= D2R;
+ lon *= D2R;
+
+/* Modified by JC Guu 08/30/96 */
+/* The variable status is used to */
+/* capture the return value form */
+/* goode_forward(). */
+/* Valid return code to process the */
+/* return status could be added in the */
+/* furture. */
+
+ status=goode_forward(lon, lat, &x, &y);
+ if(status==ERROR)
+ /*printf("Newton-Raphson method failed to converge\n");*/
+ ;
+
+ line = (ul_y - y) / pixsiz + 1.0;
+ samp = (x - ul_x) / pixsiz + 1.0;
+/* printf("%12.6f %12.6f\n",lat, lon);
+ printf("%12.6f %12.6f\n",line, samp); */
+
+ *lne = line;
+ *smp = samp;
+ *lt = lat;
+ *ln = lon;
+
+ return;
+}
+
+
+
+/* Modified by JC Guu 08/26/96 */
+/* The return type void is given. */
+
+void bad_input_parms() {
+
+/* Modified by JC Guu 08/30/96 */
+/* The required prolog components are added */
+/* here. */
+
+/*
+!C
+!Description: Function to report bad parameters.
+!Input Parameters: none
+!Output Parameters: none
+!Revision History:
+!Team-unique Header:
+!END
+*/
+
+/* Modified by JC Guu 09/03/96 */
+/* The prohibited printf() code is */
+/* commented out. */
+
+/* printf("Syntax: gihll2ls pixsize\n");
+printf(" pixsize in km = 1 or 8\n\n");*/
+}
+
+
+
+/* Modified by JC Guu 08/26/96 */
+/* The old K&R style function definition */
+/* is changed to the modern form and the */
+/* return type is given as void. */
+
+void goode_init(double r)
+{
+
+/* Modified by JC Guu 08/30/96 */
+/* The required prolog components are added */
+/* here. */
+
+/*
+!C
+!Description: Initialize the Goode`s Homolosine projection.
+ Place parameters in static storage for common use.
+!Input Parameters: r Radius of the earth (sphere)
+!Output Parameters: lon_center[12] Central meridians, one for
+ each region
+ feast[12]; False easting, one for each
+ region
+!Revision History:
+!Team-unique Header:
+!END
+*/
+
+/* Place parameters in static storage for common use
+ -------------------------------------------------*/
+
+R = r;
+
+/* Initialize central meridians for each of the 12 regions
+ -------------------------------------------------------*/
+lon_center[0] = -1.74532925199; /* -100.0 degrees */
+lon_center[1] = -1.74532925199; /* -100.0 degrees */
+lon_center[2] = 0.523598775598; /* 30.0 degrees */
+lon_center[3] = 0.523598775598; /* 30.0 degrees */
+lon_center[4] = -2.79252680319; /* -160.0 degrees */
+lon_center[5] = -1.0471975512; /* -60.0 degrees */
+lon_center[6] = -2.79252680319; /* -160.0 degrees */
+lon_center[7] = -1.0471975512; /* -60.0 degrees */
+lon_center[8] = 0.349065850399; /* 20.0 degrees */
+lon_center[9] = 2.44346095279; /* 140.0 degrees */
+lon_center[10] = 0.349065850399; /* 20.0 degrees */
+lon_center[11] = 2.44346095279; /* 140.0 degrees */
+
+/* Initialize false eastings for each of the 12 regions
+ ----------------------------------------------------*/
+feast[0] = R * -1.74532925199;
+feast[1] = R * -1.74532925199;
+feast[2] = R * 0.523598775598;
+feast[3] = R * 0.523598775598;
+feast[4] = R * -2.79252680319;
+feast[5] = R * -1.0471975512;
+feast[6] = R * -2.79252680319;
+feast[7] = R * -1.0471975512;
+feast[8] = R * 0.349065850399;
+feast[9] = R * 2.44346095279;
+feast[10] = R * 0.349065850399;
+feast[11] = R * 2.44346095279;
+
+/* Report parameters to the user
+ -----------------------------*/
+/*ptitle("Goode`s Homolosine Equal Area");
+radius(r): */
+
+/* Modified by JC Guu 08/26/96 */
+/* The function should not return anything */
+/* here. */
+
+}
+
+
+
+/* Modified by JC Guu 08/26/96 */
+/* The old K&R style function definition */
+/* is changed to the modern form and the */
+/* return type is given as void. */
+
+int goode_forward(double lon, double lat, double* x, double* y)
+
+/* Modified by JC Guu 08/30/96 */
+/* The required prolog components are added */
+/* here. */
+
+/*
+!C
+!Description:Goode`s Homolosine forward equations--mapping
+ lat,long to x,y
+!Input Parameters: lon Longitude
+ lat Latitude
+!Output Parameters: x X projection coordinate
+ y Y projection coordinate
+!Revision History:
+!Team-unique Header:
+!END
+*/
+
+{
+
+/* Modified by JC Guu 08/30/96 */
+/* Function prototype is provided for */
+/* getcoord_adjust_lon() therefore the following */
+/* declaration is not needed. */
+
+/* double getcoord_adjust_lon();*/ /* Function to adjust longitude to -180 - 180 */
+double delta_lon; /* Delta longitude (Given longitude - center */
+double theta;
+double delta_theta;
+double constant;
+int i;
+int region;
+
+/* Forward equations
+ -----------------*/
+if (lat >= 0.710987989993) /* if on or above 40 44' 11.8" */
+ {
+ if (lon <= -0.698131700798) region = 0; /* If to the left of -40 */
+ else region = 2;
+ }
+else if (lat >= 0.0) /* Between 0.0 and 40 44' 11.8" */
+ {
+ if (lon <= -0.698131700798) region = 1; /* If to the left of -40 */
+ else region = 3;
+ }
+else if (lat >= -0.710987989993) /* Between 0.0 & -40 44' 11.8" */
+ {
+ if (lon <= -1.74532925199) region = 4; /* If between -180 and -100 */
+ else if (lon <= -0.349065850399) region = 5; /* If between -100 and -20 */
+ else if (lon <= 1.3962634016) region = 8; /* If between -20 and 80 */
+ else region = 9; /* If between 80 and 180 */
+ }
+else /* Below -40 44' */
+ {
+ if (lon <= -1.74532925199) region = 6; /* If between -180 and -100 */
+ else if (lon <= -0.349065850399) region = 7; /* If between -100 and -20 */
+ else if (lon <= 1.3962634016) region = 10; /* If between -20 and 80 */
+ else region = 11; /* If between 80 and 180 */
+ }
+
+if (region==1||region==3||region==4||region==5||region==8||region==9)
+ {
+ delta_lon = getcoord_adjust_lon(lon - lon_center[region]);
+ *x = feast[region] + R * delta_lon * cos(lat);
+ *y = R * lat;
+ }
+else
+ {
+ delta_lon = getcoord_adjust_lon(lon - lon_center[region]);
+ theta = lat;
+ constant = PI * sin(lat);
+
+/* Iterate using the Newton-Raphson method to find theta
+ -----------------------------------------------------*/
+ for (i=0;;i++)
+ {
+ delta_theta = -(theta + sin(theta) - constant) / (1.0 + cos(theta));
+ theta += delta_theta;
+ if (fabs(delta_theta) < EPSLN) break;
+ if (i >= 30)
+ {giherror("Iteration failed to converge","Goode-forward");return(ERROR);}
+ }
+ theta /= 2.0;
+ *x = feast[region] + 0.900316316158 * R * delta_lon * cos(theta);
+
+/* Modified by JC Guu */
+/* The explicit cast is provided. */
+
+ *y = R * (1.4142135623731 * sin(theta) - 0.0528035274542 * ((double) getcoord_sign(lat)));
+ }
+
+
+return(OK);
+}
+
+
+
+/* Modified by JC Guu 08/27/96 */
+/* The function definition style is changed */
+/* the K&R style to the modern form. */
+
+/*
+void ptitle(char* A)
+*/
+
+/* Modified by JC Guu 08/30/96 */
+/* The required prolog components are added */
+/* here. */
+
+/*
+!C
+!Description: Functions to report projection parameter
+!Input Parameters: none
+!Output Parameters: none
+!Revision History:
+!Team-unique Header:
+!END
+*/
+
+/* Modified by JC Guu 09/03/96 */
+/* printf() is a prohibited function. */
+
+/* {printf("\n%s Projection Parameters:\n\n",A);} */
+
+
+
+/*
+void radius(double A)
+*/
+
+/* Modified by JC Guu 08/30/96 */
+/* The required prolog components are added */
+/* here. */
+
+/*
+!C
+!Description:
+!Input Parameters: none
+!Output Parameters: none
+!Revision History:
+!Team-unique Header:
+!END
+*/
+
+/* Modified by JC Guu 08/27/96 */
+/* The non-standard format specification */
+/* %lf is changed to %f. */
+
+/* Modified by JC Guu 09/03/96 */
+/* The prohibited function ptintf() is */
+/* commented out. */
+
+/*{ printf(" Radius of Sphere: %f meters\n",A); } */
+
+
+
+/* Modified by JC Guu 08/26/96 */
+/* The old K&R style function definition */
+/* is changed to the modern form and the */
+/* return type is given as void. */
+
+void giherror(char* what, char* where)
+
+/* Modified by JC Guu 08/30/96 */
+/* The required prolog components are added */
+/* here. */
+
+/*
+!C
+!Description: Function to report errors
+!Input Parameters: what cause of the error
+ where where the error occurs
+!Output Parameters: none
+!Revision History:
+!Team-unique Header:
+!END
+*/
+
+/* Modified by JC Guu 09/03/96 */
+/* printf() is a prohibited function. */
+/* commented out by fhliang on 12/08/97 */
+{
+/* printf("[%s] %s\n",where,what); */
+}
+
+
+#ifndef sun
+
+
+
+/* Modified by JC Guu 08/26/96 */
+/* The old K&R style function definition */
+/* is changed to the modern form and the */
+/* return type is given as void. */
+
+void getcoord_sincos(double val, double* sin_val, double* cos_val)
+
+/* Modified by JC Guu 08/30/96 */
+/* The required prolog components are added */
+/* here. */
+
+/*
+!C
+!Description: Function to calculate the sine and cosine in
+ one call. Some computer systems have implemented
+ this function, resulting in a faster implementation
+ than calling each function separately. It is
+ provided here for those computer systems which
+ don`t implement this function.
+
+!Input Parameters: val the value to be operated on
+!Output Parameters: sin_val the sine of val
+ cos_val the cosine of val
+!Revision History:
+!Team-unique Header:
+!END
+*/
+
+/* Modified by JC Guu 08/26/96 */
+/* A return statement is not needed here. */
+
+{ *sin_val = sin(val); *cos_val = cos(val); }
+#endif
+
+
+
+/* Modified by JC Guu 08/27/96 */
+/* The old K&R style function definition */
+/* is changed to the modern form and the */
+/* return type is given as int. */
+
+int getcoord_sign(double x)
+{ if (x < 0.0) return(-1); else return(1);}
+
+/* Modified by JC Guu 08/30/96 */
+/* The required prolog components are added */
+/* here. */
+
+/*
+!C
+!Description: Function to return the sign of an argument
+!Input Parameters: x a floating point number
+!Output Parameters: the sign of x
+!Revision History:
+!Team-unique Header:
+!END
+*/
+
+
+
+/* Modified by JC Guu 08/27/96 */
+/* The old K&R style function definition */
+/* is changed to the modern. */
+
+double getcoord_adjust_lon(double x)
+
+/* Modified by JC Guu 08/30/96 */
+/* The required prolog components are added */
+/* here. */
+
+/*
+!C
+!Description: Function to adjust longitude to -180 to 180
+!Input Parameters: x
+!Output Parameters:
+!Revision History:
+!Team-unique Header:
+!END
+*/
+
+/* Modified by JC Guu 08/27/96 */
+/* The explicit cast is provided. */
+
+{x=(fabs(x)<PI)?x:(x-(((double) getcoord_sign(x))*TWO_PI));return(x);}
+
+
+
+/* Modified by JC Guu 08/27/96 */
+/* The style of function definition is changed */
+/* from the K&R style to the modern form for */
+/* the following functions. */
+
+double getcoord_e0fn(double x)
+
+/* Modified by JC Guu 08/30/96 */
+/* The required prolog components are added */
+/* here. */
+
+/*
+!C
+!Description: Function to compute e0
+!Input Parameters: x
+!Output Parameters:
+!Revision History:
+!Team-unique Header:
+!END
+*/
+
+{return(1.0-0.25*x*(1.0+x/16.0*(3.0+1.25*x)));}
+
+
+
+double getcoord_e1fn(double x)
+
+/* Modified by JC Guu 08/30/96 */
+/* The required prolog components are added */
+/* here. */
+
+/*
+!C
+!Description: Function to compute e1
+!Input Parameters: x
+!Output Parameters:
+!Revision History:
+!Team-unique Header:
+!END
+*/
+
+{return(0.375*x*(1.0+0.25*x*(1.0+0.46875*x)));}
+
+
+
+double getcoord_e2fn(double x)
+
+/* Modified by JC Guu 08/30/96 */
+/* The required prolog components are added */
+/* here. */
+
+/*
+!C
+!Description: Function to compute e2
+!Input Parameters: x
+!Output Parameters:
+!Revision History:
+!Team-unique Header:
+!END
+*/
+
+{return(0.05859375*x*x*(1.0+0.75*x));}
+
+
+
+double getcoord_mlfn(double e0,double e1,double e2,double phi)
+
+/* Modified by JC Guu 08/30/96 */
+/* The required prolog components are added */
+/* here. */
+
+/*
+!C
+!Description:
+!Input Parameters: e0, e1, e2 and phi
+!Output Parameters:
+!Revision History:
+!Team-unique Header:
+!END
+*/
+
+{
+return(e0*phi-e1*sin(2.0*phi)+e2*sin(4.0*phi));
+}
diff --git a/mvcm/c_tools/include/ancillary.h b/mvcm/c_tools/include/ancillary.h
new file mode 100644
index 0000000..5c30bf1
--- /dev/null
+++ b/mvcm/c_tools/include/ancillary.h
@@ -0,0 +1,41 @@
+/*******************************************************************************
+Description:
+
+ Header file ancillary.h
+ Contains pointers to all ancillary data necessary for processing granule.
+ Access through structure variable 'grn_anc'.
+
+Revision History:
+ 10/2012 R. Frey Original version
+
+*******************************************************************************/
+
+// For obtaininig and processing ancillary data.
+float *g_sfct1;
+float *g_sfct2;
+float *g_snow;
+float *g_ice;
+float *g_frocn;
+float *g_frlandice;
+float *g_phis;
+float *g_tpw1;
+float *g_tpw2;
+float *g_lst;
+float *ti_sfct;
+float *ti_tpw;
+int nwp_rows;
+int nwp_cols;
+long int nwp_cells;
+
+// Ancillary data for entire granule.
+struct granule_ancillary {
+ float *reynSST;
+ float *ndvibk;
+ unsigned char *eco;
+ float *sfct;
+ float *tpw;
+ float *snowfr;
+ float *landicefr;
+ float *icefr;
+ float *geos_ocnfr;
+} grn_anc;
diff --git a/mvcm/c_tools/include/mask_processing_constants.h b/mvcm/c_tools/include/mask_processing_constants.h
new file mode 100644
index 0000000..bc1b6da
--- /dev/null
+++ b/mvcm/c_tools/include/mask_processing_constants.h
@@ -0,0 +1,39 @@
+/*******************************************************************************
+ Description:
+
+ Header file mask_processing_constants.h
+ Contains definitions for various processing constants.
+
+ Revision History:
+ 09/2012 R. Frey Original version
+ 11/2012 R. Frey Changed 'badsza_data' to 'badgeo_data'.
+ 08/2013 R. Frey Eliminated 'max_vza' and
+ added 'MODIS_max_vza' and 'VIIRS_max_vza'.
+ 08/2015 R. Frey Added "min" and "max" macros.
+
+*******************************************************************************/
+
+#define nlcntx 3
+#define necntx 3
+#define lines_in_edge (nlcntx-1)/2
+#define elems_in_edge (necntx-1)/2
+#define num_diffs_surr (nlcntx * necntx) - 1
+#define num_test_groups 5
+#define min(a,b) ((a) < (b) ? (a) : (b))
+#define MODIS_max_vza 65.49
+#define VIIRS_max_vza 70.13
+#define dtr (3.14159 / 180.0)
+#define rtd (180.0 / 3.14159)
+#define bad_data -999.0
+#define badgeo_data -327.67
+#define conf_clr_thresh 0.99
+#define prob_clr_thresh 0.95
+#define prob_cld_thresh 0.66
+
+#ifndef min
+#define min(a,b) ((a) < (b) ? (a) : (b))
+#endif
+
+#ifndef max
+#define max(a,b) ((a) < (b) ? (b) : (a))
+#endif
diff --git a/mvcm/c_tools/include/sst.h b/mvcm/c_tools/include/sst.h
new file mode 100644
index 0000000..2ba3d37
--- /dev/null
+++ b/mvcm/c_tools/include/sst.h
@@ -0,0 +1,26 @@
+/**********************************************************************
+Description:
+
+ Header file sst.h
+ Defines Reynolds "sst" structures.
+
+Revision History:
+ 10/2012 R. Frey Original version
+
+***********************************************************************/
+
+typedef struct {
+ int headerLen1;
+ int startYear;
+ int startMonth;
+ int startDay;
+ int endYear;
+ int endMonth;
+ int endDay;
+ int unknown1;
+ int unknown2;
+ int headerLen2;
+ int numSSTValues1;
+ float values[180][360];
+ int numSSTValues2;
+} SST;
diff --git a/mvcm/c_tools/read_GEOS.c b/mvcm/c_tools/read_GEOS.c
new file mode 100644
index 0000000..5e94a41
--- /dev/null
+++ b/mvcm/c_tools/read_GEOS.c
@@ -0,0 +1,219 @@
+/******************************************************************************
+
+Description:
+
+ Integer function read_GEOS.f
+ Reads desired variables from input file (in calling argument list).
+
+Input arguments:
+ geos_file input GEOS file name
+ geos_time_index indicates earlier (=1) or later (=2) GEOS file time
+
+Output arguments:
+ none
+
+Function output:
+ int return_code successful completion is zero, otherwise non-zero
+
+ Output variables in ancillary.h
+
+Revision History:
+ 06/2021 R. Frey Original version
+
+Calls:
+ none
+
+*******************************************************************************/
+
+// Includes
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <math.h>
+#include <float.h>
+
+//MVCCM includes
+//#include "granule_geolocation.h"
+//#include "granule.h"
+#include "ancillary.h"
+
+//netcdf includes
+#include <netcdf.h>
+
+/******************************************************************************/
+
+int read_GEOS(char *geos_file, int geos_time_index)
+
+{
+
+/******************************************************************************/
+
+// Declarations and initializations.
+
+/******************************************************************************/
+
+ size_t lat_len, lon_len, start[3], edge[3];
+ int status, idf, latln_id, lonln_id, nx, ny, nxny, varid;
+ int return_code = 0;
+
+/******************************************************************************/
+
+ start[0] = 0;
+ start[1] = 0;
+ start[2] = 0;
+ edge[0] = 0;
+ edge[1] = 0;
+ edge[2] = 0;
+
+/******************************************************************************/
+
+// Open file.
+// printf("Opening GEOS file: %s\n", geos_file);
+ status = nc_open(geos_file, NC_NOWRITE, &idf);
+ if (status != NC_NOERR) {
+ printf("Could not open GEOS file: %s %d %d\n", geos_file, idf, status);
+ return (-1);
+ }
+
+/******************************************************************************/
+
+// Get data array dimensions. All meteorologial variables are the same size.
+// First dimension (time) is 1.
+
+ status = nc_inq_dimid(idf, "lat", &latln_id);
+ if(status < 0) {
+ printf("Could not acquire lat-dimension data: %s\n", geos_file);
+ return_code = -2;
+ return (return_code);
+ }
+ status = nc_inq_dimlen(idf, latln_id, &lat_len);
+
+ status = nc_inq_dimid(idf, "lon", &lonln_id);
+ if(status < 0) {
+ printf("Could not acquire lon-dimension data: %s\n", geos_file);
+ return_code = -3;
+ return (return_code);
+ }
+ status = nc_inq_dimlen(idf, lonln_id, &lon_len);
+
+/******************************************************************************/
+
+ nx = lon_len;
+ ny = lat_len;
+ nxny = lat_len * lon_len;
+ nwp_cells = nxny;
+ nwp_cols = nx;
+ nwp_rows = ny;
+
+// printf("dims: %d %d %d %d %d %d\n", idf, latln_id, lonln_id, lat_len, lon_len, nxny);
+// printf("time index: %d\n", geos_time_index);
+
+ if(geos_time_index == 1) {
+
+ if ((g_sfct1 = (float *) malloc(sizeof(float) * nxny)) == NULL) {
+ printf("Could not allocate memory for g_sfct1\n");
+ return(-4);
+ }
+ if ((g_tpw1 = (float *) malloc(sizeof(float) * nxny)) == NULL) {
+ printf("Could not allocate memory for g_tpw1\n");
+ return(-5);
+ }
+
+ }
+ else if (geos_time_index == 2) {
+
+ if ((g_sfct2 = (float *) malloc(sizeof(float) * nxny)) == NULL) {
+ printf("Could not allocate memory for g_sfct2\n");
+ return(-6);
+ }
+ if ((g_tpw2 = (float *) malloc(sizeof(float) * nxny)) == NULL) {
+ printf("Could not allocate memory for g_tpw2\n");
+ return(-7);
+ }
+
+ }
+
+/******************************************************************************/
+
+ edge[0] = 1;
+ edge[1] = lat_len;
+ edge[2] = lon_len;
+
+// Read skin temperatures.
+
+ status = nc_inq_varid (idf, "TS", &varid);
+ if(status != NC_NOERR) {
+ printf("Could not find TS variable in %s\n", geos_file);
+ return_code = -8;
+ return (return_code);
+ }
+
+ if(geos_time_index == 1) {
+ status = nc_get_vara_float(idf, varid, start, edge, g_sfct1);
+ }
+ else if (geos_time_index == 2) {
+ status = nc_get_vara_float(idf, varid, start, edge, g_sfct2);
+ }
+ if (status != NC_NOERR) {
+ printf("could not get TS data in %s\n", geos_file);
+ printf("status: %d\n", status);
+ return_code = -9;
+ return (return_code);
+ }
+
+// Read total precipitable water.
+
+ status = nc_inq_varid (idf, "TQV", &varid);
+ if(status != NC_NOERR) {
+ printf("Could not find TQV variable in %s\n", geos_file);
+ return_code = -10;
+ return (return_code);
+ }
+
+ if(geos_time_index == 1) {
+ status = nc_get_vara_float(idf, varid, start, edge, g_tpw1);
+ }
+ else if (geos_time_index == 2) {
+ status = nc_get_vara_float(idf, varid, start, edge, g_tpw2);
+ }
+ if (status != NC_NOERR) {
+ printf("could not get TQV data in %s\n", geos_file);
+ printf("status: %d\n", status);
+ return_code = -11;
+ return (return_code);
+ }
+
+/******************************************************************************/
+
+// if(geos_time_index == 1) {
+// printf("GEOS1 data: %f %f %f\n", g_sfct1[0], g_sfct1[9], g_sfct1[nxny-1]);
+// printf("GEOS1 data: %f %f %f\n", g_tpw1[0], g_tpw1[9], g_tpw1[nxny-1]);
+// printf("GEOS1 data: %f %f %f %f %f %f %f %f %f\n",
+// g_sfct1[0], g_lst[0], g_tpw1[0],
+// g_sfct1[32580], g_lst[32580], g_tpw1[32580],
+// g_sfct1[65159], g_lst[65159], g_tpw1[65159]);
+// }
+// if(geos_time_index == 2) {
+// printf("GEOS2 data: %f %f %f\n", g_sfct2[0], g_sfct2[9], g_sfct2[nxny-1]);
+// printf("GEOS2 data: %f %f %f\n", g_tpw2[0], g_tpw2[9], g_tpw2[nxny-1]);
+// printf("GEOS2 data: %f %f %f %f %f %f\n",
+// g_sfct2[0], g_tpw2[0],
+// g_sfct2[32580], g_tpw2[32580],
+// g_sfct2[65159], g_tpw2[65159]);
+// }
+// fflush;
+
+/******************************************************************************/
+
+// Close file.
+ status = nc_close(idf);
+
+/******************************************************************************/
+
+ return (return_code);
+
+/******************************************************************************/
+
+}
diff --git a/mvcm/c_tools/read_GEOS_constants.c b/mvcm/c_tools/read_GEOS_constants.c
new file mode 100644
index 0000000..9b8d06d
--- /dev/null
+++ b/mvcm/c_tools/read_GEOS_constants.c
@@ -0,0 +1,200 @@
+/******************************************************************************
+
+Description:
+
+ Integer function read_GEOS_constants.c
+ Reads desired variables from time-invariant (constant) file (calling argument).
+
+Input arguments:
+ geos_file_cnst input GEOS constants file name
+
+Output arguments:
+ none
+
+Function output:
+ int return_code successful completion is zero, otherwise non-zero
+
+ Output variables in ancillary.h
+
+Revision History:
+ 07/2021 R. Frey Original version
+
+Calls:
+ none
+
+*******************************************************************************/
+
+// Includes
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <math.h>
+#include <float.h>
+
+//MVCCM includes
+//#include "granule_geolocation.h"
+//#include "granule.h"
+#include "ancillary.h"
+
+//netcdf includes
+#include <netcdf.h>
+
+/******************************************************************************/
+
+int read_GEOS_constants(char *geos_file_cnst)
+
+{
+
+/******************************************************************************/
+
+// Declarations and initializations.
+
+/******************************************************************************/
+
+ size_t lat_len, lon_len, start[3], edge[3];
+ int status, idf, latln_id, lonln_id, nxc, nyc, nxnyc, varid;
+ int return_code = 0;
+
+/******************************************************************************/
+
+ start[0] = 0;
+ start[1] = 0;
+ start[2] = 0;
+ edge[0] = 0;
+ edge[1] = 0;
+ edge[2] = 0;
+
+/******************************************************************************/
+
+// Open file.
+// printf("Opening GEOS constants file: %s\n", geos_file_cnst);
+ status = nc_open(geos_file_cnst, NC_NOWRITE, &idf);
+ if (status != NC_NOERR) {
+ printf("Could not open GEOS constants file: %s %d %d\n", geos_file_cnst,
+ idf, status);
+ return (-1);
+ }
+
+/******************************************************************************/
+
+// Get data array dimensions. All meteorologial variables are the same size.
+// First dimension (time) is 1.
+
+ status = nc_inq_dimid(idf, "lat", &latln_id);
+ if(status < 0) {
+ printf("Could not acquire lat-dimension data: %s\n", geos_file_cnst);
+ return_code = -2;
+ return (return_code);
+ }
+ status = nc_inq_dimlen(idf, latln_id, &lat_len);
+
+ status = nc_inq_dimid(idf, "lon", &lonln_id);
+ if(status < 0) {
+ printf("Could not acquire lon-dimension data: %s\n", geos_file_cnst);
+ return_code = -3;
+ return (return_code);
+ }
+ status = nc_inq_dimlen(idf, lonln_id, &lon_len);
+
+/******************************************************************************/
+
+ nxc = lon_len;
+ nyc = lat_len;
+ nxnyc = lat_len * lon_len;
+
+ if ((g_frocn = (float *) malloc(sizeof(float) * nxnyc)) == NULL) {
+ printf("Could not allocate memory for g_frocn\n");
+ return(-4);
+ }
+ if ((g_frlandice = (float *) malloc(sizeof(float) * nxnyc)) == NULL) {
+ printf("Could not allocate memory for g_frlandice\n");
+ return(-4);
+ }
+ if ((g_phis = (float *) malloc(sizeof(float) * nxnyc)) == NULL) {
+ printf("Could not allocate memory for g_glst\n");
+ return(-5);
+ }
+
+/******************************************************************************/
+
+ edge[0] = 1;
+ edge[1] = lat_len;
+ edge[2] = lon_len;
+
+/******************************************************************************/
+
+// Read ocean fractions.
+
+ status = nc_inq_varid (idf, "FROCEAN", &varid);
+ if(status != NC_NOERR) {
+ printf("Could not find FROCEAN variable in %s\n", geos_file_cnst);
+ return_code = -8;
+ return (return_code);
+ }
+
+ status = nc_get_vara_float(idf, varid, start, edge, g_frocn);
+ if (status != NC_NOERR) {
+ printf("could not get FROCEAN data in %s\n", geos_file_cnst);
+ printf("status: %d\n", status);
+ return_code = -9;
+ return (return_code);
+ }
+
+/******************************************************************************/
+
+// Read permanent land ice fractions.
+
+ status = nc_inq_varid (idf, "FRLANDICE", &varid);
+ if(status != NC_NOERR) {
+ printf("Could not find FRLANDICE variable in %s\n", geos_file_cnst);
+ return_code = -8;
+ return (return_code);
+ }
+
+ status = nc_get_vara_float(idf, varid, start, edge, g_frlandice);
+ if (status != NC_NOERR) {
+ printf("could not get FROCEAN data in %s\n", geos_file_cnst);
+ printf("status: %d\n", status);
+ return_code = -9;
+ return (return_code);
+ }
+
+/******************************************************************************/
+
+// Read surface geopotential heights.
+
+ status = nc_inq_varid (idf, "PHIS", &varid);
+ if(status != NC_NOERR) {
+ printf("Could not find PHIS variable in %s\n", geos_file_cnst);
+ return_code = -10;
+ return (return_code);
+ }
+
+ status = nc_get_vara_float(idf, varid, start, edge, g_phis);
+ if (status != NC_NOERR) {
+ printf("could not get PHIS data in %s\n", geos_file_cnst);
+ printf("status: %d\n", status);
+ return_code = -11;
+ return (return_code);
+ }
+
+/******************************************************************************/
+
+// printf("GEOS frc ocn data: %f %f %f\n", g_frocn[0], g_frocn[9], g_frocn[nxnyc-1]);
+// printf("GEOS PHIS data: %f %f %f\n", g_phis[0], g_phis[9], g_phis[nxnyc-1]);
+// fflush;
+
+/******************************************************************************/
+
+// Close files.
+ status = nc_close(idf);
+
+/******************************************************************************/
+
+ return (return_code);
+
+/******************************************************************************/
+
+}
diff --git a/mvcm/c_tools/read_GEOS_lndocn.c b/mvcm/c_tools/read_GEOS_lndocn.c
new file mode 100644
index 0000000..88dc7ba
--- /dev/null
+++ b/mvcm/c_tools/read_GEOS_lndocn.c
@@ -0,0 +1,187 @@
+/******************************************************************************
+
+Description:
+
+ Integer function read_GEOS_lndocn.c
+ Reads desired variables from input files (in calling argument list).
+
+Input arguments:
+ geos_file_lnd input GEOS land property file name
+ geos_file_ocn input GEOS ocean property file name
+
+Output arguments:
+ none
+
+Function output:
+ int return_code successful completion is zero, otherwise non-zero
+
+ Output variables in ancillary.h
+
+Revision History:
+ 06/2021 R. Frey Original version
+
+Calls:
+ none
+
+*******************************************************************************/
+
+// Includes
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <math.h>
+#include <float.h>
+
+//MVCCM includes
+//#include "granule_geolocation.h"
+//#include "granule.h"
+#include "ancillary.h"
+
+//netcdf includes
+#include <netcdf.h>
+
+/******************************************************************************/
+
+int read_GEOS_lndocn(char *geos_file_lnd, char *geos_file_ocn)
+
+{
+
+/******************************************************************************/
+
+// Declarations and initializations.
+
+/******************************************************************************/
+
+ size_t lat_len, lon_len, start[3], edge[3];
+ int status, idfl, idfo, latln_id, lonln_id, nx, ny, nxny, varid;
+ int return_code = 0;
+
+/******************************************************************************/
+
+ start[0] = 0;
+ start[1] = 0;
+ start[2] = 0;
+ edge[0] = 0;
+ edge[1] = 0;
+ edge[2] = 0;
+
+/******************************************************************************/
+
+// Open files.
+// printf("Opening GEOS land file: %s\n", geos_file_lnd);
+ status = nc_open(geos_file_lnd, NC_NOWRITE, &idfl);
+ if (status != NC_NOERR) {
+ printf("Could not open GEOS land property file: %s %d %d\n", geos_file_lnd,
+ idfl, status);
+ return (-1);
+ }
+// printf("Opening GEOS ocean file: %s\n", geos_file_ocn);
+ status = nc_open(geos_file_ocn, NC_NOWRITE, &idfo);
+ if (status != NC_NOERR) {
+ printf("Could not open GEOS ocean property file: %s %d %d\n", geos_file_ocn,
+ idfo, status);
+ return (-1);
+ }
+
+/******************************************************************************/
+
+// Get data array dimensions. All meteorologial variables are the same size.
+// First dimension (time) is 1.
+
+ status = nc_inq_dimid(idfl, "lat", &latln_id);
+ if(status < 0) {
+ printf("Could not acquire lat-dimension data: %s\n", geos_file_lnd);
+ return_code = -2;
+ return (return_code);
+ }
+ status = nc_inq_dimlen(idfl, latln_id, &lat_len);
+
+ status = nc_inq_dimid(idfl, "lon", &lonln_id);
+ if(status < 0) {
+ printf("Could not acquire lon-dimension data: %s\n", geos_file_lnd);
+ return_code = -3;
+ return (return_code);
+ }
+ status = nc_inq_dimlen(idfl, lonln_id, &lon_len);
+
+/******************************************************************************/
+
+ nx = lon_len;
+ ny = lat_len;
+ nxny = lat_len * lon_len;
+ nwp_cells = nxny;
+ nwp_cols = nx;
+ nwp_rows = ny;
+
+// printf("dims: %d %d %d %d %d %d\n", idfl, latln_id, lonln_id, lat_len, lon_len, nxny);
+
+ if ((g_snow = (float *) malloc(sizeof(float) * nxny)) == NULL) {
+ printf("Could not allocate memory for g_snow\n");
+ return(-4);
+ }
+ if ((g_ice = (float *) malloc(sizeof(float) * nxny)) == NULL) {
+ printf("Could not allocate memory for g_ice\n");
+ return(-5);
+ }
+
+/******************************************************************************/
+
+ edge[0] = 1;
+ edge[1] = lat_len;
+ edge[2] = lon_len;
+
+// Read snow fractions.
+
+ status = nc_inq_varid (idfl, "FRSNO", &varid);
+ if(status != NC_NOERR) {
+ printf("Could not find FRSNO variable in %s\n", geos_file_lnd);
+ return_code = -8;
+ return (return_code);
+ }
+
+ status = nc_get_vara_float(idfl, varid, start, edge, g_snow);
+ if (status != NC_NOERR) {
+ printf("could not get FRSNO data in %s\n", geos_file_lnd);
+ printf("status: %d\n", status);
+ return_code = -9;
+ return (return_code);
+ }
+
+// Read ice fractions.
+
+ status = nc_inq_varid (idfo, "FRSEAICE", &varid);
+ if(status != NC_NOERR) {
+ printf("Could not find FRSEAICE variable in %s\n", geos_file_ocn);
+ return_code = -10;
+ return (return_code);
+ }
+
+ status = nc_get_vara_float(idfo, varid, start, edge, g_ice);
+ if (status != NC_NOERR) {
+ printf("could not get FRSEAICE data in %s\n", geos_file_ocn);
+ printf("status: %d\n", status);
+ return_code = -11;
+ return (return_code);
+ }
+
+/******************************************************************************/
+
+// printf("GEOS snow data: %f %f %f\n", g_snow[0], g_snow[9], g_snow[nxny-1]);
+// printf("GEOS ice data: %f %f %f\n", g_ice[0], g_ice[9], g_ice[nxny-1]);
+// fflush;
+
+/******************************************************************************/
+
+// Close files.
+ status = nc_close(idfl);
+ status = nc_close(idfo);
+
+/******************************************************************************/
+
+ return (return_code);
+
+/******************************************************************************/
+
+}
diff --git a/mvcm/c_tools/snow_mask.c b/mvcm/c_tools/snow_mask.c
new file mode 100644
index 0000000..f7c1c21
--- /dev/null
+++ b/mvcm/c_tools/snow_mask.c
@@ -0,0 +1,226 @@
+/**********************************************************************
+Description:
+
+ Function snow_mask
+
+ Subroutine which implements a quick and dirty version of the snow
+ algorithm by Dorothy Hall, George Riggs and Vince Salmonson.
+ Called from get_pxldat.c.
+
+ Version 2 adds snow from NDSI=0.4 to NDSI=0.1 but also as a function
+ of NDVI (NDVI=0.1 to NDVI=1.0).
+
+Input arguments:
+ satname Instrument/platform
+ MODIS_Aqua
+ MODIS_Terra
+ VIIRS_NPP
+ VIIRS_N20
+
+ Input values stored in variables found in pixel.h, granule.h,
+ mask_processing_constants.h, and thresholds.h.
+
+Output arguments:
+ none
+
+ Output stored in 'pxin.ndsi_snow' found in pixel.h.
+
+Revision History:
+ 10/2007 R. Frey Converted to C
+ 10/2012 R. Frey Adapted for MVCCM software.
+ 11/2012 R. Frey Adapted for MVCCM algorithm.
+ 04/2014 R. Frey Added Klein et al. NDSI extension as a function of
+ NDVI
+ 10/2014 R. Frey Added "mountain" snow test from Thompson et al., 2014
+ 01/2015 R. Frey Added surface temperature test to snow detection and
+ added 'lsf' to calling argument list
+ 10/2015 R. Frey Added 'Grnlnd_ndsi', 'sm85_11hel', 'sm_lsfcdif', 'sm_wsfcdif',
+ 'sm_bt1
+ 11/2015 R. Frey Code clean-up.
+ 04/2018 R. Frey Added 'prd_ndvi_const'.
+ 02/2020 R. Frey Added logic to include NOAA-20.
+
+***********************************************************************/
+
+/* Includes */
+
+#include <stdio.h>
+#include <math.h>
+//#include "granule.h"
+//#include "pixel.h"
+//#include "mask_processing_constants.h"
+//#include "thresholds.h"
+
+void snow_mask(char *satname, unsigned char lsf)
+
+{
+
+// Declarations
+
+ extern float rintf(float);
+ extern int strcmp(char *, char *);
+
+ int I_bad;
+ int hi_elev_snow;
+
+ float m04;
+ float m01;
+ float m02;
+ float m20;
+ float m29;
+ float m31;
+ float m35;
+ float nirband;
+ float ndsi;
+ float ndvi;
+ float pred_ndvi;
+ float diff;
+ float diff2;
+ float sth85_11;
+ float sth37_11;
+ float sfcdif;
+ float bad_data = -999.0;
+
+// Initializations
+ m04 = pxin.rad[pxin.bnd555];
+ m01 = pxin.rad[pxin.bnd065];
+ m02 = pxin.rad[pxin.bnd086];
+ m20 = pxin.rad[pxin.bnd375];
+ m29 = pxin.rad[pxin.bnd855];
+ m31 = pxin.rad[pxin.bnd11];
+
+ diff = 0.0;
+ diff2 = 0.0;
+ hi_elev_snow = 0;
+
+// Select NIR channel for NDSI based on platform name.
+// Band 6 for Terra, band 7 for Aqua.
+// if(strcmp(satname, "MODIS_Terra") == 0) {
+// nirband = pxin.rad[pxin.bnd164];
+// }
+// else if(strcmp(satname, "MODIS_Aqua") == 0) {
+// nirband = pxin.rad[pxin.bnd213];
+// }
+ if(strcmp(satname, "VIIRS_NPP") == 0) {
+ nirband = pxin.rad[pxin.bnd164];
+ }
+ else if(strcmp(satname, "VIIRS_N20") == 0) {
+ nirband = pxin.rad[pxin.bnd164];
+ }
+ else {
+ printf("Platform name not recognized in snow_mask \n");
+ }
+
+ I_bad = rintf(bad_data);
+
+// Begin snow/ice detection.
+
+// First, check the 11 micron brightness temperature.
+ if (rintf(m31) != I_bad && m31 <= sm_bt11[0]) {
+
+// Get NDSI index
+ if (rintf(m04) != I_bad && rintf(nirband) != I_bad) {
+
+ ndsi = (m04 - nirband) / (m04 + nirband);
+ ndvi = (m02 - m01) / (m02 + m01);
+ pred_ndvi = prd_ndvi_const[0] + (3.0 * ndsi);
+
+ if(pxin.land && (!pxin.Greenland) && (!pxin.Antarctic) ) {
+ if(pxin.elev > 1000.0) {
+ hi_elev_snow = 1;
+ if(ndsi > 0.0) pxin.ndsi_snow = 1;
+ }
+ else if ( (ndsi > sm_ndsi[0] && m02 > sm_ref2[0]) ||
+ (ndsi > 0.1 && ndsi <= sm_ndsi[0] && ndvi <= pred_ndvi) ) {
+ pxin.ndsi_snow = 1;
+ }
+ }
+ else if( pxin.land && pxin.Greenland) {
+ if( ndsi > Grnlnd_ndsi[0] ) pxin.ndsi_snow = 1;
+ }
+ else {
+ if (ndsi > sm_ndsi[0] && m02 > sm_ref2[0]) pxin.ndsi_snow = 1;
+ }
+
+// printf("ndsi: %f %f %f %f %f %f %f %f %d\n", m01, m02, m04, nirband, ndsi, ndvi, prd_ndvi_const[0], pred_ndvi, pxin.ndsi_snow);
+
+ if(pxin.ndsi_snow) {
+
+// If in sunglint region, disregard if between -60 and 50 degrees latitude.
+ if(pxin.water && pxin.sunglint && pxin.lat <= 50.0 && pxin.lat >= -60.0) {
+ pxin.ndsi_snow = 0;
+ }
+
+// Check for ice clouds mis-identified as snow. Note modified BTD thresholds for Greenland.
+ diff = -99.0;
+ diff2 = -99.0;
+ sth85_11 = -99.0;
+ sth37_11 = -99.0;
+ if(rintf(m29) != I_bad && rintf(m31) != I_bad) {
+ diff = m29 - m31;
+ sth85_11 = sm85_11[0];
+ if(rintf(m20) != I_bad && !(pxin.Greenland && pxin.hi_elev)) {
+ diff2 = m20 - m31;
+ sth37_11 = sm37_11[0];
+ if(diff >= sth85_11 && diff2 >= sth37_11) {
+ pxin.ndsi_snow = 0;
+ }
+ }
+ else {
+ if(pxin.Greenland && pxin.hi_elev) {
+ sth85_11 = sm85_11hel[0];
+ }
+ else {
+ sth85_11 = sm85_11[0];
+ }
+ if(diff >= sth85_11) {
+ pxin.ndsi_snow = 0;
+ }
+ }
+ }
+// printf("ice chk: %d %d %f %f\n\n", pxin.Greenland, pxin.hi_elev, diff, sth85_11);
+
+// Check for water clouds mis-identified as snow.
+ diff = -99.0;
+ sth37_11 = -99.0;
+ if(m20 != I_bad && m31 != I_bad) {
+ if( !(pxin.hi_elev && pxin.Greenland) ) {
+ diff = m20 - m31;
+ if(pxin.hi_elev) {
+ sth37_11 = sm37_11hel[0];
+ }
+ else {
+ sth37_11 = sm37_11[0];
+ }
+ if(diff >= sth37_11) {
+ pxin.ndsi_snow = 0;
+ }
+ }
+ }
+
+// Check value of NIR reflectance.
+ if( !pxin.hi_elev && !hi_elev_snow) {
+ if(nirband > sm_mnir[0]) {
+ pxin.ndsi_snow = 0;
+ }
+
+// Check 11 um vs. surface temperature.
+ sfcdif = pxin.sfctmp - m31;
+ if(lsf == 1 || lsf == 2) {
+// Land or coast.
+ if(sfcdif > sm_lsfcdif[0]) pxin.ndsi_snow = 0;
+ }
+ else if(lsf == 0 || lsf > 2) {
+// Water.
+ if(sfcdif > sm_wsfcdif[0] && pxin.sfctmp > sm_bt1[0]) pxin.ndsi_snow = 0;
+ }
+
+ }
+
+
+ }
+ }
+ }
+
+
+}
diff --git a/mvcm/c_tools/swap_bytes.c b/mvcm/c_tools/swap_bytes.c
new file mode 100644
index 0000000..47fd705
--- /dev/null
+++ b/mvcm/c_tools/swap_bytes.c
@@ -0,0 +1,126 @@
+# include <stdlib.h>
+
+int swapbyte4(void *, int);
+void swbyt4_(void *, int *);
+void fbyte4_(void *, int *);
+
+/* ===== swapbyte4 ================================================== */
+/*
+*| Name:
+*| swapbyte4 - Flips 4 byte words into network byte order.
+*|
+*| Interface:
+*| int
+*| swapbyte4 (void *buf, int n_words)
+*|
+*| Input:
+*| buf - Buffer to be flipped.
+*| n_words - Number of words to flip.
+*|
+*| Input and Output:
+*| none
+*|
+*| Output:
+*| none
+*|
+*| Return values:
+*| 0 - Success.
+*| -1 - buf pointer is NULL.
+*|
+*| Remarks:
+*| Network byte order is big endian.
+*|
+*| Categories:
+*| converter
+*| utility
+*/
+
+int
+ swapbyte4 (void *buf, int n_words)
+{
+ int *b_ptr = (int *) buf;
+ int nw = (int) n_words;
+
+ if (b_ptr == (int *) NULL)
+ {
+ return (-1);
+ }
+
+ if (n_words > 0)
+ {
+ swbyt4_ (b_ptr, &nw);
+ }
+ return (0);
+}
+
+/*
+*$ Name:
+*$ swbyt4 - Re-orders bytes if internal representation is not
+*$ big-endian.
+*$
+*$ Interface:
+*$ subroutine
+*$ swbyt4(integer buf(*), integer n)
+*$
+*$ Input:
+*$ n - Number of 4 byte swaps to be made if order is not
+*$ big-endian.
+*$
+*$ Input and Output:
+*$ buf - Array containing bytes to be swapped.
+*$
+*$ Output:
+*$ none
+*$
+*$ Return values:
+*$ none
+*$
+*$ Remarks:
+*$ none
+*$
+*$ Categories:
+*$ utility
+*/
+
+void
+swbyt4_(void *buf, int *n)
+{
+ /* determine byte order from first principles */
+ union
+ {
+ char bytes[sizeof(int)];
+ int word;
+ } q;
+
+ q.word = 1;
+ if (q.bytes[3] == 1)
+ return;
+
+ /* swap bytes */
+ fbyte4_(buf, n);
+}
+
+ /* buffer - INTEGER*4 array to switch bytes */
+ /* n - INTEGER number of 4 byte switches */
+
+void
+fbyte4_(void *buffer, int *num)
+{
+ char *cbuf = (char *)buffer;
+ int i, n;
+
+ n = *num;
+ for (i = 0; i < n; i++)
+ {
+ char b;
+
+ b = cbuf[0];
+ cbuf[0] = cbuf[3];
+ cbuf[3] = b;
+ b = cbuf[1];
+ cbuf[1] = cbuf[2];
+ cbuf[2] = b;
+ cbuf += 4;
+ }
+}
+
diff --git a/mvcm/conf.py b/mvcm/conf.py
new file mode 100644
index 0000000..bc586ef
--- /dev/null
+++ b/mvcm/conf.py
@@ -0,0 +1,274 @@
+import numpy as np
+
+
+def test(flipped=False):
+ bt = np.arange(265, 275)
+ if flipped is False:
+ thr = np.array([267, 270, 273, 1, 1])
+ else:
+ thr = np.array([273, 270, 267, 1, 1])
+ c = conf_test(bt, thr)
+ print(c)
+
+
+def test_dble(flipped=False):
+ bt = np.arange(260, 282)
+ if flipped is False:
+ thr = np.array([264, 267, 270, 273, 276, 279, 1, 1])
+ else:
+ thr = np.array([279, 276, 273, 270, 267, 264, 1, 1])
+ c = conf_test_dble(bt, thr)
+ print(c)
+
+
+def conf_test_new(rad: np.ndarray,
+ thr: np.ndarray) -> np.ndarray:
+ """Assuming a linear function between min and max confidence level, the plot below shows
+ how the confidence (y axis) is computed as function of radiance (x axis).
+ This case illustrates alpha < gamma, obviously in case alpha > gamma, the plot would be
+ flipped.
+ gamma
+ c 1 ________
+ o | /
+ n | /
+ f | /
+ i | beta /
+ d 1/2 |....../
+ e | /
+ n | /
+ c | /
+ e 0________/
+ | alpha
+ --------- radiance ---------->
+ """
+
+ radshape = rad.shape
+ rad = rad.ravel()
+ thr = np.array(thr)
+ if thr.ndim == 1:
+ thr = np.full((rad.shape[0], 4), thr[:4]).T
+
+ coeff = np.power(2, (thr[3, :] - 1))
+ locut = thr[0, :]
+ beta = thr[1, :]
+ hicut = thr[2, :]
+ power = thr[3, :]
+ confidence = np.zeros(rad.shape)
+
+ alpha, gamma = np.empty(rad.shape), np.empty(rad.shape)
+ flipped = np.zeros(rad.shape)
+
+ gamma[hicut > locut] = thr[2, hicut > locut]
+ alpha[hicut > locut] = thr[0, hicut > locut]
+ flipped[hicut > locut] = 0
+ gamma[hicut < locut] = thr[0, hicut < locut]
+ alpha[hicut < locut] = thr[2, hicut < locut]
+ flipped[hicut < locut] = 1
+
+ # Rad between alpha and beta
+ range_ = 2. * (beta - alpha)
+ s1 = (rad - alpha) / range_
+ idx = np.nonzero((rad <= beta) & (flipped == 0))
+ confidence[idx] = coeff[idx] * np.power(s1[idx], power[idx])
+ idx = np.nonzero((rad <= beta) & (flipped == 1))
+ confidence[idx] = 1.0 - coeff[idx] * np.power(s1[idx], power[idx])
+
+ # Rad between beta and gamma
+ range_ = 2. * (beta - gamma)
+ s1 = (rad - gamma) / range_
+ idx = np.nonzero((rad > beta) & (flipped == 0))
+ confidence[idx] = 1.0 - coeff[idx] * np.power(s1[idx], power[idx])
+ idx = np.nonzero((rad > beta) & (flipped == 1))
+ confidence[idx] = coeff[idx] * np.power(s1[idx], power[idx])
+
+ # Rad outside alpha-gamma interval
+ confidence[(rad > gamma) & (flipped is False)] = 1
+ confidence[(rad < alpha) & (flipped is False)] = 0
+ confidence[(rad > gamma) & (flipped is True)] = 0
+ confidence[(rad < alpha) & (flipped is True)] = 1
+
+ confidence = np.clip(confidence, 0, 1)
+
+ return confidence.reshape(radshape)
+
+
+def conf_test(rad, thr):
+ '''
+ Assuming a linear function between min and max confidence level, the plot below shows
+ how the confidence (y axis) is computed as function of radiance (x axis).
+ This case illustrates alpha < gamma, obviously in case alpha > gamma, the plot would be
+ flipped.
+ gamma
+ c 1 ________
+ o | /
+ n | /
+ f | /
+ i | beta /
+ d 1/2 |....../
+ e | /
+ n | /
+ c | /
+ e 0________/
+ | alpha
+ --------- radiance ---------->
+ '''
+
+ radshape = rad.shape
+ rad = rad.reshape(np.prod(radshape))
+
+ if thr.ndim == 1:
+ thr = np.full((rad.shape[0], 4), thr[:4]).T
+
+ coeff = np.power(2, (thr[3] - 1))
+ hicut = thr[2, :]
+ beta = thr[1, :]
+ locut = thr[0, :]
+ power = thr[3, :]
+ confidence = np.zeros(rad.shape)
+
+ alpha, gamma = np.empty(rad.shape), np.empty(rad.shape)
+ flipped = np.zeros(rad.shape)
+
+ gamma[hicut > locut] = thr[2, hicut > locut]
+ alpha[hicut > locut] = thr[0, hicut > locut]
+ flipped[hicut > locut] = 0
+ gamma[hicut < locut] = thr[0, hicut < locut]
+ alpha[hicut < locut] = thr[2, hicut < locut]
+ flipped[hicut < locut] = 1
+
+ # Rad between alpha and beta
+ range_ = 2. * (beta - alpha)
+ s1 = (rad - alpha) / range_
+ idx = np.nonzero((rad <= beta) & (flipped == 0))
+ confidence[idx] = coeff[idx] * np.power(s1[idx], power[idx])
+ idx = np.nonzero((rad <= beta) & (flipped == 1))
+ confidence[idx] = 1.0 - coeff[idx] * np.power(s1[idx], power[idx])
+
+ # Rad between beta and gamma
+ range_ = 2. * (beta - gamma)
+ s1 = (rad - gamma) / range_
+ idx = np.nonzero((rad > beta) & (flipped == 0))
+ confidence[idx] = 1.0 - coeff[idx] * np.power(s1[idx], power[idx])
+ idx = np.nonzero((rad > beta) & (flipped == 1))
+ confidence[idx] = coeff[idx] * np.power(s1[idx], power[idx])
+
+ # Rad outside alpha-gamma interval
+ confidence[(rad > gamma) & (flipped is False)] = 1
+ confidence[(rad < alpha) & (flipped is False)] = 0
+ confidence[(rad > gamma) & (flipped is True)] = 0
+ confidence[(rad < alpha) & (flipped is True)] = 1
+
+ confidence[confidence > 1] = 1
+ confidence[confidence < 0] = 0
+
+ return confidence
+
+
+def conf_test_dble(rad, coeffs):
+ # '''
+ # gamma1 gamma2
+ # c 1_______ ________
+ # o | \ /
+ # n | \ /
+ # f | \ /
+ # i | \ beta1 beta2 /
+ # d 1/2 \....| |...../
+ # e | \ /
+ # n | \ /
+ # c | \ /
+ # e 0 \_____________/
+ # | alpha1 alpha2
+ # --------------------- radiance ------------------------->
+ # '''
+
+ coeffs = np.array(coeffs)
+ radshape = rad.shape
+ rad = rad.ravel()
+ confidence = np.zeros(rad.shape)
+
+ alpha1, gamma1 = np.empty(rad.shape), np.empty(rad.shape)
+ alpha2, gamma2 = np.empty(rad.shape), np.empty(rad.shape)
+
+ if coeffs.ndim == 1:
+ coeffs = np.full((rad.shape[0], 7), coeffs[:7]).T
+
+ gamma1 = coeffs[0, :]
+ beta1 = coeffs[1, :]
+ alpha1 = coeffs[2, :]
+ alpha2 = coeffs[3, :]
+ beta2 = coeffs[4, :]
+ gamma2 = coeffs[5, :]
+ power = coeffs[6, :]
+
+ coeff = np.power(2, (power - 1))
+ # radshape = rad.shape
+ # rad = rad.reshape((rad.shape[0]*rad.shape[1]))
+
+ # ## Find if interval between inner cutoffs passes or fails test
+
+ # Inner region fails test
+
+ # Value is within range of lower set of limits
+ range_ = 2. * (beta1 - alpha1)
+ s1 = (rad - alpha1) / range_
+ idx = np.nonzero((rad <= alpha1) & (rad >= beta1) & (alpha1 - gamma1 > 0))
+ confidence[idx] = coeff[idx] * np.power(s1[idx], power[idx])
+
+ range_ = 2. * (beta1 - gamma1)
+ s1 = (rad - gamma1) / range_
+ idx = np.nonzero((rad >= gamma1) & (rad < beta1) & (alpha1 - gamma1 > 0))
+ confidence[idx] = 1.0 - coeff[idx] * np.power(s1[idx], power[idx])
+
+ # Value is within range of upper set of limits
+ range_ = 2. * (beta2 - alpha2)
+ s1 = (rad - alpha2) / range_
+ idx = np.nonzero((rad > alpha1) & (rad <= beta2) & (alpha1 - gamma1 > 0))
+ confidence[idx] = coeff[idx] * np.power(s1[idx], power[idx])
+
+ range_ = 2. * (beta2 - gamma2)
+ s1 = (rad - gamma2) / range_
+ idx = np.nonzero((rad > alpha1) & (rad > beta2) & (alpha1 - gamma1 > 0))
+ confidence[idx] = 1.0 - coeff[idx] * np.power(s1[idx], power[idx])
+
+ # Check for value beyond function range
+ confidence[(alpha1 - gamma1 > 0) & (rad > alpha1) & (rad < alpha2)] = 0
+ confidence[(alpha1 - gamma1 > 0) & ((rad < gamma1) | (rad > gamma2))] = 1
+
+ ###
+
+ # Inner region passes test
+ print("I NEED TO REVIEW THIS TO WRITE IT MORE CLEARLY")
+ # FOR NOW ALPHA AND GAMMA ARE SWITCHED BECAUSE OF HOW THE ARRAYS ARE DEFINED.
+ # THINK ON HOW THIS COULD BE WRITTEN SO THAT IT'S EASIER TO UNDERSTAND (AND DEBUG)
+ # Value is within range of lower set of limits
+ range_ = 2 * (beta1 - alpha1)
+ s1 = (rad - alpha1) / range_
+ idx = np.nonzero((rad > alpha1) & (rad <= gamma1) & (rad <= beta1) & (alpha1 - gamma1 <= 0))
+ confidence[idx] = 1.0 - coeff[idx] * np.power(s1[idx], power[idx])
+
+ range_ = 2 * (beta1 - gamma1)
+ s1 = (rad - gamma1) / range_
+ idx = np.nonzero((rad > alpha1) & (rad <= gamma1) & (rad > beta1) & (alpha1 - gamma1 <= 0))
+ confidence[idx] = coeff[idx] * np.power(s1[idx], power[idx])
+
+ # Values is within range of upper set of limits
+ range_ = 2 * (beta2 - alpha2)
+ s1 = (rad - alpha2) / range_
+ idx = np.nonzero((rad > gamma2) & (rad < alpha2) & (rad >= beta2) & (alpha1 - gamma1 <= 0))
+ confidence[idx] = 1.0 - coeff[idx] * np.power(s1[idx], power[idx])
+
+ range_ = 2 * (beta2 - gamma2)
+ s1 = (rad - gamma2) / range_
+ idx = np.nonzero((rad > gamma2) & (rad < alpha2) & (rad < beta2) & (alpha1 - gamma1 <= 0))
+ confidence[idx] = coeff[idx] * np.power(s1[idx], power[idx])
+
+ confidence[(alpha1 - gamma1 <= 0) & ((rad > gamma1) | (rad < gamma2))] = 0
+ confidence[(alpha1 - gamma1 <= 0) & (rad <= alpha1) & (rad >= alpha2)] = 1
+
+ # confidence = np.clip(confidence, 0, 1)
+
+ return confidence.reshape(radshape)
+
+
+if __name__ == "__main__":
+ test_dble()
diff --git a/mvcm/main.py b/mvcm/main.py
new file mode 100644
index 0000000..27fa0ed
--- /dev/null
+++ b/mvcm/main.py
@@ -0,0 +1,532 @@
+try:
+ from ruamel import yaml as yml
+except ImportError:
+ import ruamel_yaml as yml
+
+import numpy as np
+
+from glob import glob
+
+import read_data as rd
+import spectral_tests as tst
+import scene as scn
+import restoral
+
+# #################################################################### #
+# TEST CASE
+# data:
+_datapath = '/ships19/hercules/pveglio/mvcm_viirs_hires'
+_fname_mod02 = glob(f'{_datapath}/VNP02MOD.A2022173.1454.001.*.uwssec_bowtie_restored.nc')[0]
+_fname_mod03 = glob(f'{_datapath}/VNP03MOD.A2022173.1454.001.*.uwssec.nc')[0]
+_fname_img02 = glob(f'{_datapath}/VNP02IMG.A2022173.1454.001.*.uwssec_bowtie_restored.nc')[0]
+_fname_img03 = glob(f'{_datapath}/VNP03IMG.A2022173.1454.001.*.uwssec.nc')[0]
+
+# thresholds:
+_threshold_file = '/home/pveglio/mvcm/thresholds.mvcm.snpp.v0.0.1.yaml'
+
+# ancillary files:
+_geos_atm_1 = 'GEOS.fpit.asm.inst3_2d_asm_Nx.GEOS5124.20220622_1200.V01.nc4'
+_geos_atm_2 = 'GEOS.fpit.asm.inst3_2d_asm_Nx.GEOS5124.20220622_1500.V01.nc4'
+_geos_land = 'GEOS.fpit.asm.tavg1_2d_lnd_Nx.GEOS5124.20220622_1430.V01.nc4'
+_geos_ocean = 'GEOS.fpit.asm.tavg1_2d_ocn_Nx.GEOS5124.20220622_1430.V01.nc4'
+_geos_constants = 'GEOS.fp.asm.const_2d_asm_Nx.00000000_0000.V01.nc4'
+_ndvi_file = 'NDVI.FM.c004.v2.0.WS.00-04.177.hdf'
+_sst_file = 'oisst.20220622'
+_eco_file = 'goge1_2_img.v1'
+
+# #################################################################### #
+
+
+def main(satellite: str = 'snpp',
+ sensor: str = 'viirs',
+ data_path: str = _datapath,
+ mod02: str = _fname_mod02,
+ mod03: str = _fname_mod03,
+ img02: str = None,
+ img03: str = None,
+ threshold_file: str = _threshold_file,
+ geos_atm_1: str = _geos_atm_1,
+ geos_atm_2: str = _geos_atm_2,
+ geos_land: str = _geos_land,
+ geos_ocean: str = _geos_ocean,
+ geos_constants: str = _geos_constants,
+ ndvi_file: str = _ndvi_file,
+ sst_file: str = _sst_file) -> None:
+ """Main function for the MVCM.
+
+ Parameters
+ ----------
+ satellite: str
+ satellite name, not case-sensitive. Available options: [snpp, ]
+ sensor: str
+ sensor name, not case-sensitive. Available options: [viirs, ]
+ data_path: str
+ path where the data is stored
+ mod02: str
+ VIIRS MOD02 file name
+ mod03: str
+ VIIRS MOD03 file name
+ img02: [optional] str
+ VIIRS IMG02 file name
+ img03: [optional] str
+ VIIRS IMG03 file name
+ threshold_file: str
+ thresholds file name
+ geos_atm_1: str
+ file name of first GEOS5 file for atmospheric parameters
+ goes_atm_2: str
+ file name of second GEOS5 file for atmospheric parameters
+ geos_land: str
+ file name of GEOS5 land parameters
+ geos_ocean: str
+ file name of GEOS5 ocean parameters
+ geos_constants: str
+ file name of GEOS5 constants
+ ndvi_file: str
+ NDVI file name
+ sst_file:
+ Sea surface temperature file name
+
+ Returns
+ -------
+ None
+ """
+
+ file_names = {'MOD02': f'{mod02}',
+ 'MOD03': f'{mod03}',
+ 'IMG02': f'{img02}',
+ 'IMG03': f'{img03}',
+ 'GEOS_atm_1': f'{geos_atm_1}',
+ 'GEOS_atm_2': f'{geos_atm_2}',
+ 'GEOS_land': f'{geos_land}',
+ 'GEOS_ocean': f'{geos_ocean}',
+ 'GEOS_constants': f'{geos_constants}',
+ 'NDVI': f'{ndvi_file}',
+ 'SST': f'{sst_file}',
+ 'ANC_DIR': f'{data_path}/ancillary'
+ }
+
+ with open(threshold_file) as f:
+ text = f.read()
+ thresholds = yml.safe_load(text)
+
+ sunglint_angle = thresholds['Sun_Glint']['bounds'][3]
+
+ viirs_data = rd.get_data(satellite, sensor, file_names, sunglint_angle)
+
+ # Initialize the confidence arrays for the various test groups
+ cmin_G1 = np.ones(viirs_data.M01.shape)
+ cmin_G2 = np.ones(viirs_data.M01.shape)
+ cmin_G3 = np.ones(viirs_data.M01.shape)
+ cmin_G4 = np.ones(viirs_data.M01.shape)
+ cmin_G5 = np.ones(viirs_data.M01.shape)
+
+ ##########################################################
+
+ """
+ for scene_name in ['Land_Day', 'Land_Day_Coast', 'Land_Day_Desert_Coast',
+ 'Ocean_Day', 'Ocean_Night', 'Polar_Day_Ocean', 'Polar_Night_Ocean',
+ 'Polar_Day_Land', 'Polar_Day_Coast', 'Polar_Day_Desert',
+ 'Polar_Day_Desert_Coast', 'Polar_Day_Snow', 'Land_Night',
+ 'Polar_Night_Land', 'Polar_Night_Snow', 'Day_Snow', 'Night_Snow']:
+ """
+ for scene_name in ['Ocean_Day']:
+ MyScene = tst.CloudTests(viirs_data, scene_name, thresholds)
+
+ cmin_G1, bit1 = MyScene.test_11um('M15', cmin_G1, test_name='11um_Test')
+ cmin_G1, bit2 = MyScene.sst_test('M15', 'M16', cmin_G1, test_name='SST_Test')
+
+ cmin_G2, bit3 = MyScene.bt_diff_86_11um('M14-M15', cmin_G2, test_name='8.6-11um_Test')
+ cmin_G2, bit4 = MyScene.test_11_12um_diff('M15-M16', cmin_G2, test_name='11-12um_Cirrus_Test')
+ cmin_G2, bit5 = MyScene.oceanic_stratus_11_4um_test('M15-M12', cmin_G2,
+ test_name='11-4um_Oceanic_Stratus_Test')
+
+ cmin_G3, bit6 = MyScene.nir_reflectance_test('M07', cmin_G3, test_name='NIR_Reflectance_Test')
+ cmin_G3, bit7 = MyScene.vis_nir_ratio_test('M07-M05ratio', cmin_G3, test_name='Vis/NIR_Ratio_Test')
+ cmin_G3, bit8 = MyScene.nir_reflectance_test('M10', cmin_G3, test_name='1.6_2.1um_NIR_Reflectance_Test')
+ cmin_G3, bit9 = MyScene.visible_reflectance_test('M128', cmin_G3, test_name='Visible_Reflectance_Test')
+ cmin_G3, bit10 = MyScene.gemi_test('GEMI', cmin_G3, test_name='GEMI_Test')
+
+ cmin_G4, bit11 = MyScene.test_1_38um_high_clouds('M09', cmin_G4, test_name='1.38um_High_Cloud_Test')
+
+ cmin_G5, bit12 = MyScene.thin_cirrus_4_12um_BTD_test('M13-M16', cmin_G5,
+ test_name='4-12um_BTD_Thin_Cirrus_Test')
+
+ cmin = cmin_G1 * cmin_G2 * cmin_G3 * cmin_G4 * cmin_G5
+
+ if scene_name in ['Ocean_Day']:
+ # total_bit = np.full(viirs_data.M01.shape, 4)
+ total_bit = bit1 + bit2 + bit4 + bit11
+ sunglint_angle = thresholds['Sun_Glint']['bounds'][3]
+ scene_flags = scn.find_scene(viirs_data, sunglint_angle)
+
+ # idx = np.nonzero((scene_flags['water'] == 1) & (scene_flags['ice'] == 0) &
+ # (scene_flags['uniform'] == 1) & (cmin <= 0.99) & (cmin >= 0.05))
+ # cmin[idx] = restoral.spatial(viirs_data, thresholds['Sun_Glint'], scene_flags, cmin)[idx]
+
+ idx = np.nonzero((scene_flags['water'] == 1) & (scene_flags['sunglint'] == 1) &
+ (scene_flags['uniform'] == 1) & (cmin <= 0.95))
+ cmin[idx] = restoral.sunglint(viirs_data, thresholds['Sun_Glint'], total_bit, cmin)[idx]
+
+ # MVCM = tst.ComputeTests(viirs_data, scene_name, thresholds)
+ # cmin = MVCM.clear_sky_restoral(cmin)
+
+ # return cmin
+ np.savez('test_ams', confidence=cmin, lat=viirs_data.latitude.values, lon=viirs_data.longitude.values)
+
+ ##########################################################
+
+ """
+ # ------------------- #
+ # ### Testing Setup ###
+ # ------------------- #
+
+ perform = {'11um BT Test': True,
+ 'CO2 High Clouds Test': False,
+ 'Water Vapor High Clouds Test': False,
+ 'Surface Temperature Test': False,
+ 'SST Test': False,
+ '8.6-11um BT Difference Test': False,
+ '11-12um BTD Thin Cirrus Test': False,
+ '11-4um BT Difference Test': False,
+ '7.3-11um BT Difference Mid-level Clouds': False,
+ 'Water Vapor Cloud Test': False,
+ '11um BT Variability Test': False,
+ '11-4um BTD Oceanic Stratus': False,
+ 'NIR Reflectance Test': False,
+ 'Vis/NIR Ratio Test': False,
+ '1.6um or 2.1um NIR Reflectance Test': False,
+ 'Visible Reflectance Test': False,
+ 'GEMI Test': False,
+ '1.38um High Cloud Test': False,
+ '4-12um BTD Thin Cirrus Test': False
+ }
+
+ # --------------------- #
+ # ### Group 1 Tests ### #
+ # --------------------- #
+
+ if perform['11um BT Test'] is True:
+ # 11um BT Test
+ for scene_name in ['Ocean_Day', 'Ocean_Night', 'Polar_Ocean_Day', 'Polar_Ocean_Night']:
+ SceneType = CloudTests(viirs_data, scene_name, thresholds)
+ cmin_G1 = SceneType.single_threshold_test('11um_Test', 'M15', cmin_G1)
+
+ if perform['CO2 High Clouds Test'] is True:
+ # CO2 High Clouds Test
+ for scene_name in ['Land_Day', 'Land_Night', 'Land_Day_Coast', 'Land_Day_Desert',
+ 'Land_Day_Desert_Coast', 'Ocean_Day', 'Ocean_Night', 'Day_Snow', 'Night_Snow']:
+ SceneType = CloudTests(viirs_data, scene_name, thresholds)
+ cmin_G1 = SceneType.single_threshold_test('CO2_High_Clouds_tests', 'bad_data', cmin_G1)
+
+ if perform['Water Vapor High Clouds Test'] is True:
+ # Water Vapor High Clouds Test
+ for scene_name in ['Land_Day', 'Land_Night', 'Land_Day_Coast', 'Land_Day_Desert',
+ 'Land_Day_Desert_Coast', 'Ocean_Day', 'Ocean_Night', 'Polar_Day_Land',
+ 'Polar_Night_Land', 'Polar_Day_Coast', 'Polar_Day_Desert',
+ 'Polar_Day_Desert_Coast', 'Polar_Day_Snow', 'Polar_Night_Snow', 'Polar_Ocean_Day',
+ 'Polar_Ocean_Night', 'Day_Snow', 'Night_Snow', 'Antarctic_Day']:
+ SceneType = CloudTests(viirs_data, scene_name, thresholds)
+ cmin_G1 = SceneType.single_threshold_test('Water_Vapor_High_Clouds_tests', 'bad_data', cmin_G1)
+
+ if perform['Surface Temperature Test'] is True:
+ # Surface Temperature Test
+ # NOTE: this test requires the thresholds to be preprocessed
+ for scene_name in ['Land_Night', 'Polar_Night_Land']:
+ SceneType = CloudTests(viirs_data, scene_name, thresholds)
+ cmin_G1 = SceneType.single_threshold_test('Surface_Temperature_Test', 'M15-M16', cmin_G1)
+
+ if perform['SST Test'] is True:
+ # SST Test
+ for scene_name in ['Ocean_Day', 'Ocean_Night', 'Polar_Ocean_Day', 'Polar_Ocean_Night']:
+ SceneType = CloudTests(viirs_data, scene_name, thresholds)
+ cmin_G1 = SceneType.single_threshold_test('SST_Test', 'M15-M16', cmin_G1)
+
+ # --------------------- #
+ # ### Group 2 tests ### #
+ # --------------------- #
+
+ if perform['8.6-11um BT Difference Test'] is True:
+ # 8.6-11um BT Difference Test
+ for scene_name in ['Ocean_Day', 'Ocean_Night', 'Polar_Ocean_Day', 'Polar_Ocean_Night']:
+ SceneType = CloudTests(viirs_data, scene_name, thresholds)
+ cmin_G2 = SceneType.single_threshold_test('8.6-11um_Test', 'M15-M14', cmin_G2)
+
+ if perform['11-12um BTD Thin Cirrus Test'] is True:
+ # 11-12um BT BTD Transmissive Cirrus Test
+ # NOTE: some of the tests have some differences in how the thresholds are derived
+ # The commented list scene_name is the complete list, the one currently in use is to test that
+ # the template code works at least with a subset of scenes that have the same way of deriving the
+ # thresholds
+ # for scene_name in ['Land_Day', 'Land_Day_Coast', 'Land_Day_Desert', 'Land_Day_Desert_Coast',
+ # 'Ocean_Day', 'Ocean_Night', 'Polar_Day_Land', 'Polar_Day_Coast',
+ # 'Polar_Day_Desert', 'Polar_Day_Desert_Coast', 'Polar_Day_Snow', 'Polar_Night_Snow',
+ # 'Polar_Ocean_Day', 'Polar_Ocean_Night', 'Day_Snow', 'Night_Snow']:
+ for scene_name in ['Land_Day', 'Land_Day_Coast', 'Land_Day_Desert', 'Land_Day_Desert_Coast',
+ 'Ocean_Day', 'Ocean_Night', 'Polar_Ocean_Day', 'Polar_Ocean_Night']:
+ SceneType = CloudTests(viirs_data, scene_name, thresholds)
+ cmin_G2 = SceneType.single_threshold_test('11-12um_Cirrus_Test', 'M15-M16', cmin_G2)
+
+ if perform['11-4um BT Difference Test'] is True:
+ # 11-4um BT Difference Test
+ for scene_name in ['Ocean_Night', 'Polar_Ocean_Night']:
+ pass
+
+ if perform['7.3-11um BT Difference Mid-level Clouds'] is True:
+ for scene_name in ['Land_Night', 'Polar_Night_Land', 'Polar_Night_Snow', 'Night_Snow']:
+ pass
+
+ if perform['Water Vapor Cloud Test'] is True:
+ for scene_name in ['Polar_Night_Ocean']:
+ pass
+
+ if perform['11um BT Variability Test'] is True:
+ for scene_name in ['Ocean_Night', 'Polar_Night_Ocean']:
+ pass
+
+ if perform['11-4um BTD Oceanic Stratus'] is True:
+ # 11-4um BT Difference for oceanic stratus (low emissivity water cloud) Test
+ for scene_name in ['Land_Day', 'Land_Day_Coast', 'Land_Day_Desert', 'Land_Day_Desert_Coast',
+ 'Ocean_Day', 'Ocean_Night', 'Polar_Day_Land', 'Polar_Day_Coast',
+ 'Polar_Day_Desert', 'Polar_Day_Desert_Coast', 'Polar_Day_Snow', 'Polar_Night_Snow',
+ 'Polar_Ocean_Day', 'Polar_Ocean_Night', 'Day_Snow', 'Night_Snow']:
+ SceneType = CloudTests(viirs_data, scene_name, thresholds)
+ cmin_G2 = SceneType.single_threshold_test('11-4um_Oceanic_Stratus_Test', 'M15-M13', cmin_G2)
+
+ # --------------------- #
+ # ### Group 3 tests ### #
+ # --------------------- #
+
+ if perform['NIR Reflectance Test'] is True:
+ for scene_name in ['Ocean_Day', 'Polar_Ocean_Day']:
+ SceneType = CloudTests(viirs_data, scene_name, thresholds)
+ cmin_G3 = SceneType.single_thredhold_test('NIR_Reflectance_Test', 'M07', cmin_G3)
+
+ if perform['Vis/NIR Ratio Test'] is True:
+ for scene_name in ['Ocean_Day', 'Polar_Ocean_Day']:
+ pass
+
+ if perform['1.6um or 2.1um NIR Reflectance Test'] is True:
+ for scene_name in ['Ocean_Day', 'Polar_Ocean_Day']:
+ pass
+
+ if perform['Visible Reflectance Test'] is True:
+ for scene_name in ['Land_Day', 'Land_Day_Coast', 'Land_Day_Desert', 'Land_Day_Desert_Coast',
+ 'Polar_Day_Land', 'Polar_Day_Coast', 'Polar_Day_Desert', 'Polar_Day_Desert_Coast']:
+ pass
+
+ if perform['GEMI Test'] is True:
+ for scene_name in ['Land_Day_Desert', 'Polar_Day_Desert']:
+ pass
+
+ # --------------------- #
+ # ### Group 4 tests ### #
+ # --------------------- #
+
+ if perform['1.38um High Cloud Test'] is True:
+ # for scene_name in ['Land_Day', 'Land_Day_Coast', 'Land_Day_Desert', 'Land_Day_Desert_Coast',
+ # 'Ocean_Day', 'Polar_Day_Land', 'Polar_Day_Coast', 'Polar_Day_Desert',
+ # 'Polar_Day_Desert_Coast', 'Polar_Ocean_Day', 'Day_Snow']:
+ for scene_name in ['Land_Day', 'Land_Day_Coast', 'Land_Day_Desert', 'Land_Day_Desert_Coast',
+ 'Polar_Day_Land', 'Polar_Day_Coast', 'Polar_Day_Desert',
+ 'Polar_Day_Desert_Coast', 'Day_Snow']:
+ # The current loop is missing Ocean_Day and Polar_Ocean_Day because they need some
+ # preprocessing to compute the thresholds. Once things are implemented I can use the commented
+ # loop
+ SceneType = CloudTests(viirs_data, scene_name, thresholds)
+ cmin_G4 = SceneType.single_threshold_test('1.38um_High_Cloud_Test', 'M09', cmin_G4)
+
+ # --------------------- #
+ # ### Group 5 tests ### #
+ # --------------------- #
+
+ if perform['4-12um BTD Thin Cirrus Test'] is True:
+ for scene_name in ['Land_Night', 'Polar_Night_Land', 'Polar_Night_Snow', 'Night_Snow']:
+ pass
+
+ cmin_total = cmin_G1 * cmin_G2 * cmin_G3 * cmin_G4 * cmin_G5
+ return cmin_total
+ """
+
+ '''
+ Land_Day = CloudTests(viirs_data, 'Land_Day', thresholds)
+ Land_Night = CloudTests(viirs_data, 'Land_Night', thresholds)
+ Land_Day_Coast = CloudTests(viirs_data, 'Land_Day_Coast', thresholds)
+ Land_Day_Desert = CloudTests(viirs_data, 'Land_Day_Desert', thresholds)
+ Land_Day_Desert_Coast = CloudTests(viirs_data, 'Land_Day_Desert_Coast', thresholds)
+ Ocean_Day = CloudTests(viirs_data, 'Ocean_Day', thresholds)
+ Ocean_Night = CloudTests(viirs_data, 'Ocean_Night', thresholds)
+ Polar_Day_Land = CloudTests(viirs_data, 'Polar_Day_Land', thresholds)
+ Polar_Night_Land = CloudTests(viirs_data, 'Polar_Night_Land', thresholds)
+ Polar_Day_Coast = CloudTests(viirs_data, 'Polar_Day_Coast', thresholds)
+ Polar_Day_Desert = CloudTests(viirs_data, 'Polar_Day_Desert', thresholds)
+ Polar_Day_Desert_Coast = CloudTests(viirs_data, 'Polar_Day_Desert_Coast', thresholds)
+ Polar_Day_Snow = CloudTests(viirs_data, 'Polar_Day_Snow', thresholds)
+ Polar_Night_Snow = CloudTests(viirs_data, 'Polar_Night_Snow', thresholds)
+ Polar_Ocean_Day = CloudTests(viirs_data, 'Polar_Ocean_Day', thresholds)
+ Polar_Ocean_Night = CloudTests(viirs_data, 'Polar_Ocean_Night', thresholds)
+ Day_Snow = CloudTests(viirs_data, 'Day_Snow', thresholds)
+ Night_Snow = CloudTests(viirs_data, 'Night_Snow', thresholds)
+ Antarctic_Day = CloudTests(viirs_data, 'Antarctic_Day', thresholds)
+
+ # 11um BT Test
+ cmin_G1 = Ocean_Day.single_threshold_test('11um_Test', 'M15', cmin_G1)
+ cmin_G1 = Polar_Ocean_Day.single_threshold_test('11um_Test', 'M15', cmin_G1)
+ cmin_G1 = Polar_Ocean_Night.single_threshold_test('11um_Test', 'M15', cmin_G1)
+
+ # CO2 High Clouds Test
+ # NOTE: VIIRS doesn't have the MODIS equivalent of B35 so this test is not performed
+ cmin_G1 = Land_Day.single_threshold_test('CO2_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Land_Night.single_threshold_test('CO2_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Land_Day_Coast.single_threshold_test('CO2_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Land_Day_Desert.single_threshold_test('CO2_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Land_Day_Desert_Coast.single_threshold_test('CO2_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Ocean_Day.single_threshold_test('CO2_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Ocean_Night.single_threshold_test('CO2_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Day_Snow.single_threshold_test('CO2_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Night_Snow.single_threshold_test('CO2_High_Clouds_Test', 'bad_data', cmin_G1)
+
+ # Water Vapor High Clouds Test
+ # NOTE: VIIRS doesn't have the MODIS equivalent of B27 so this test is not performed
+ cmin_G1 = Land_Day.single_threshold_test('Water_Vapor_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Land_Night.single_threshold_test('Water_Vapor_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Land_Day_Coast.single_threshold_test('Water_Vapor_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Land_Day_Desert.single_threshold_test('Water_Vapor_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Land_Day_Desert_Coast.single_threshold_test('Water_Vapor_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Ocean_Day.single_threshold_test('Water_Vapor_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Ocean_Night.single_threshold_test('Water_Vapor_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Polar_Day_Land.single_threshold_test('Water_Vapor_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Polar_Night_Land.single_threshold_test('Water_Vapor_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Polar_Day_Coast.single_threshold_test('Water_Vapor_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Polar_Day_Desert.single_threshold_test('Water_Vapor_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Polar_Day_Desert_Coast.single_threshold_test('Water_Vapor_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Polar_Day_Snow.single_threshold_test('Water_Vapor_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Polar_Night_Snow.single_threshold_test('Water_Vapor_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Polar_Ocean_Day.single_threshold_test('Water_Vapor_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Polar_Ocean_Night.single_threshold_test('Water_Vapor_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Day_Snow.single_threshold_test('Water_Vapor_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Night_Snow.single_threshold_test('Water_Vapor_High_Clouds_Test', 'bad_data', cmin_G1)
+ cmin_G1 = Antarctic_Day.single_threshold_test('Water_Vapor_High_Clouds_Test', 'bad_data', cmin_G1)
+
+ # Surface Temperature Test
+ # ## NOTE: This requires some calculations for the thresholds.
+ # Moreover this test is using the 11um - 12um difference instead of a single channel
+ # Also, look at the test carefully for these two cases. Polar_Night_Land uses some hardcoded coeffs
+ # (i.e. *_df1 and *_df2) that might be worth moving in the thresholds file for consistency
+ cmin_G1 = Land_Night.single_threshold_test('Surface_Temperature_Test', 'M15-M16', cmin_G1)
+ cmin_G1 = Polar_Night_Land.single_threshold_test('Surface_Temperature_Test', 'M15-M16', cmin_G1)
+
+ # SST Test
+ # ## NOTE: This requires some calculations for the thresholds.
+ # Moreover this test is using the 11um - 12um difference instead of a single channel
+ cmin_G1 = Ocean_Day.single_threshold_test('SST_Test', 'M15-M16', cmin_G1)
+ cmin_G1 = Polar_Ocean_Day.single_threshold_test('SST_Test', 'M15-M16', cmin_G1)
+ cmin_G1 = Polar_Ocean_Night.single_threshold_test('SST_Test', 'M15-M16', cmin_G1)
+ # the following test uses a different set of coefficients compared to the others above
+ cmin_G1 = Ocean_Night.single_threshold_test('SST_Test', 'M15-M16', cmin_G1)
+
+ # 11-8.6um BT Difference Test
+ cmin_G2 = Ocean_Day.single_threshold_test('11-8.6um_Test', 'M15-M14', cmin_G2)
+ cmin_G2 = Ocean_Night.single_threshold_test('11-8.6um_Test', 'M15-M14', cmin_G2)
+ cmin_G2 = Polar_Ocean_Day.single_threshold_test('11-8.6um_Test', 'M15-M14', cmin_G2)
+ cmin_G2 = Polar_Ocean_Night.single_threshold_test('11-8.6um_Test', 'M15-M14', cmin_G2)
+
+ # 11-12um BT Difference
+ cmin_G1 = Ocean_Day.single_threshold_test('11-12BT_diff',
+ viirs_data.M15.values-viirs_data.M16.values,
+ cmin_G1)
+
+ Ocean_Night = CloudTests(viirs_data, 'Ocean_Night', thresholds)
+
+ cmin_G1 = Ocean_Night.single_threshold_test('11um_test', viirs_data.M15.values, cmin_G1)
+ cmin_G1 = Ocean_Night.single_threshold_test('11-12um_diff',
+ viirs_data.M15.values-viirs_data.M16.values,
+ cmin_G1)
+
+ c = np.ones((9, viirs_data['M01'].shape[0], viirs_data['M01'].shape[1]))
+ cmin1, c[0, :, :], bit1 = odt.simple_test(viirs_data.M15.values, thresholds['Daytime_Ocean']['bt11'], cmin1)
+ cmin1, c[1, :, :], bit2 = odt.sst_test(viirs_data.M15.values, viirs_data.M16.values,
+ viirs_data.sensor_zenith.values, viirs_data.geos_sfct.values,
+ thresholds['Daytime_Ocean']['sst'], cmin1)
+ cmin2, c[2, :, :], bit3 = odt.simple_test(viirs_data.M14.values-viirs_data.M15.values,
+ thresholds['Daytime_Ocean']['diff_11_86um'], cmin2)
+ cmin2, c[3, :, :], bit4 = odt.test_11_12_diff(viirs_data, thresholds['Daytime_Ocean']['diff11_12um'], cmin2)
+ cmin2, c[4, :, :] = odt.test_11_4_diff(viirs_data.M15.values, viirs_data.M12.values,
+ thresholds['Daytime_Ocean']['test11_4lo'], scene_flags, cmin2)
+ cmin3, c[5, :, :] = odt.nir_refl_test(viirs_data.M07.values, thresholds['Daytime_Ocean'],
+ thresholds['Sun_Glint'], viirs_data, cmin3)
+ cmin3, c[6, :, :] = odt.vis_nir_ratio_test(viirs_data.M05.values, viirs_data.M07.values,
+ thresholds, scene_flags, cmin3)
+ cmin3, c[7, :, :] = odt.nir_refl_test(viirs_data.M10.values, thresholds['Daytime_Ocean'],
+ thresholds['Sun_Glint'], viirs_data, cmin3)
+#
+# confidence[0, :, :] = tests.test_11um(viirs_data.M15.values, thresholds['Daytime_Ocean'])
+# confidence[1, :, :] = tests.test_11_4diff(viirs_data.M15.values, viirs_data.M13.values,
+# thresholds['Daytime_Ocean'], viirs_data,
+# thresholds['Sun_Glint']['bounds'][3])
+#
+# confidence[2, :, :] = tests.nir_refl_test(viirs_data.M07.values, thresholds['Daytime_Ocean'],
+# thresholds['Sun_Glint'], viirs_data)
+#
+# # Note that here I'm using M05/M07 but the corresponding hi-res channels are I1/I2
+# # IMPORTANT: conf_test_dble() needs to be verified. I don't think it's working as intended at the moment
+# confidence[3, :, :] = tests.vis_nir_ratio_test(viirs_data.M05.values, viirs_data.M07.values,
+# thresholds['Daytime_Ocean'], thresholds['Sun_Glint'])
+#
+# # This test needs to be verified, for the granule I'm running everything is zero
+# confidence[4, :, :] = tests.test_11um_var(viirs_data.M15.values, thresholds['Nighttime_Ocean'],
+# thresholds['Daytime_Ocean_Spatial_Variability'])
+
+ total_bit = bit1 + bit2 + bit4
+ temp_confidence = cmin1 * cmin2 * cmin3 * cmin4
+ confidence = cmin1 * cmin2 * cmin3 * cmin4
+ # idx = np.nonzero((scene_flags['water'] == 1) & (scene_flags['ice'] == 0) & (scene_flags['uniform'] == 1) &
+ # (confidence <= 0.99) & (confidence >= 0.05))
+ # confidence[idx] = restoral.spatial(viirs_data, thresholds['Sun_Glint'], scene_flags, confidence)[idx]
+
+ idx = np.nonzero((scene_flags['water'] == 1) & (scene_flags['sunglint'] == 1) &
+ (scene_flags['uniform'] == 1) & (confidence <= 0.95))
+ confidence[idx] = restoral.sunglint(viirs_data, thresholds['Sun_Glint'], total_bit, temp_confidence)[idx]
+
+ temp = np.zeros((viirs_data.M01.shape[0], viirs_data.M01.shape[1]))
+ temp[idx] = 1
+ c[8, :, :] = temp
+
+ np.savez('test_confidence', confidence=confidence, conf_test=c,
+ lat=viirs_data.latitude.values, lon=viirs_data.longitude.values)
+
+ return confidence
+ '''
+
+
+def test_main():
+
+ rad1 = [[255, 260, 265, 248, 223],
+ [278, 285, 270, 268, 256],
+ [275, 273, 266, 254, 259]]
+ rad2 = [[270, 273, 271, 268, 265],
+ [277, 286, 275, 277, 269],
+ [280, 281, 272, 270, 267]]
+ thresh_file = '/home/pveglio/mvcm_leo/thresholds/new_thresholds.mvcm.snpp.v1.0.0.yaml'
+
+ with open(thresh_file) as f:
+ text = f.read()
+
+ rad1 = np.array(rad1)
+ rad2 = np.array(rad2)
+
+ confidence = np.zeros((2, rad1.shape[0], rad1.shape[1]))
+
+ thresholds = yml.safe_load(text)
+
+ confidence[0, :, :] = tests.test_11um(rad1, thresholds['Daytime_Ocean'])
+ confidence[1, :, :] = tests.test_11_4diff(rad1, rad2, thresholds['Daytime_Ocean'])
+
+ print(f'Confidence[0,:,:]: \n {confidence[0, :, :]}')
+ print(f'Confidence[1,:,:]: \n {confidence[1, :, :]}')
+
+ return confidence
+
+
+if __name__ == "__main__":
+ main()
diff --git a/mvcm/preprocess_thresholds.py b/mvcm/preprocess_thresholds.py
new file mode 100644
index 0000000..56d12a1
--- /dev/null
+++ b/mvcm/preprocess_thresholds.py
@@ -0,0 +1,567 @@
+import numpy as np
+import xarray as xr
+
+import ancillary_data as anc
+import utils
+
+from numpy.lib.stride_tricks import sliding_window_view
+from typing import Dict
+
+_dtr = np.pi/180
+_DTR = np.pi/180
+
+
+def prepare_11_12um_thresholds(thresholds: np.ndarray,
+ dim1: int) -> Dict:
+
+ coeff_values = np.empty((dim1, 2))
+ coeff_values[:, 0] = np.full(dim1, thresholds['coeffs'][0])
+ coeff_values[:, 1] = np.full(dim1, thresholds['coeffs'][1])
+ cmult_values = np.full(dim1, thresholds['cmult'])
+ adj_values = np.full(dim1, thresholds['adj'])
+ if 'bt1' in list(thresholds):
+ bt1 = np.full(dim1, thresholds['bt1'])
+ else:
+ bt1 = np.full(dim1, -999)
+ if 'lat' in list(thresholds):
+ lat = np.full(dim1, thresholds['lat'])
+ else:
+ lat = np.full(dim1, -999)
+ thr_dict = {'coeffs': coeff_values,
+ 'cmult': cmult_values,
+ 'adj': adj_values,
+ 'bt1': bt1,
+ 'lat': lat,
+ }
+
+ return thr_dict
+
+
+# function was called preproc
+def thresholds_11_12um(data: xr.Dataset,
+ thresholds: np.ndarray,
+ scene: str,
+ scene_idx: np.ndarray) -> np.ndarray:
+ cosvza = np.cos(data.sensor_zenith.values[scene_idx].ravel() * _DTR)
+ schi = np.full(cosvza.shape, 99.0)
+ schi[cosvza > 0] = 1/cosvza
+ schi = np.array(schi, dtype=np.float32) # this is because the C function expects a float
+
+ m15 = data.M15.values[scene_idx].ravel()
+ latitude = data.latitude.values[scene_idx].ravel()
+ thr = anc.py_cithr(1, schi, m15)
+ thr_dict = prepare_11_12um_thresholds(thresholds, m15.shape[0])
+
+ midpt = np.full(m15.shape[0], thr)
+ idx = np.nonzero((thr < 0.1) | (np.abs(schi-99) < 0.0001))
+ midpt[idx] = thr_dict['coeffs'][idx, 0]
+ locut = midpt + (thr_dict['cmult'] * midpt)
+ if scene in ['Land_Day', 'Land_Day_Coast', 'Land_Day_Desert', 'Land_Day_Desert_Coast',
+ 'Ocean_Day', 'Ocean_Night', 'Polar_Day_Ocean', 'Polar_Night_Ocean']:
+ hicut = midpt - thr_dict['adj']
+ elif scene in ['Polar_Day_Land', 'Polar_Day_Coast', 'Polar_Day_Desert',
+ 'Polar_Day_Desert_Coast', 'Polar_Day_Snow']:
+ hicut = midpt - (thr_dict['adj'] * midpt)
+ elif scene in ['Land_Night', 'Polar_Night_Land', 'Polar_Night_Snow', 'Day_Snow', 'Night_Snow']:
+ _coeffs = {'Land_Night': 0.3, 'Polar_Night_Land': 0.3, 'Polar_Night_Snow': 0.3,
+ 'Day_Snow': 0.0, 'Night_Snow': 0.3}
+ midpt = midpt - (_coeffs[scene] * locut)
+ if scene in ['Polar_Night_Land', 'Polar_Night_Snow', 'Night_Snow']:
+ hicut = np.full(m15.shape, midpt - 1.25)
+ idx = np.nonzero(m15 < thr_dict['bt1'])
+ hicut[idx] = midpt[idx] - (0.2 * locut[idx])
+ elif scene in ['Land_Night']:
+ hicut = np.full(m15.shape, 1.25)
+ idx = np.nonzero((m15 < thr_dict['bt1']) & (latitude > thr_dict['lat']))
+ hicut[idx] = -0.1 - np.power(90.0 - np.abs(latitude[idx])/60, 4) * 1.15
+ elif scene in ['Day_Snow']:
+ hicut = locut - (thr_dict['cmult'] * locut)
+ else:
+ print('Scene not recognized\n')
+ else:
+ print('Scene not recognized\n')
+
+ thr_out = np.dstack((locut, midpt, hicut, np.ones(locut.shape), np.ones(locut.shape)))
+ return np.squeeze(thr_out.T)
+
+
+def thresholds_NIR(data, thresholds, scene, test_name, scene_idx):
+
+ sza = data.solar_zenith.values[scene_idx].ravel()
+ band_n = 2
+ # NOTE: the visud condition in the C code is equivalent to having sza <= 85
+ # For the time being the visud filtering is not implemented
+
+ c = np.array(thresholds[scene][test_name]['coeffs'])
+ vzcpow = thresholds['VZA_correction']['vzcpow'][0]
+ refang = data.sunglint_angle.values[scene_idx].ravel()
+ sunglint_thresholds = thresholds['Sun_Glint']
+ sunglint_flag = utils.sunglint_scene(refang, sunglint_thresholds)
+ nir_thresh = thresholds[scene][test_name]
+
+ if test_name == 'NIR_Reflectance_Test':
+ hicut = c[0] + c[1]*sza + c[2]*np.power(sza, 2) + c[3]*np.power(sza, 3)
+ elif test_name == '1.6_2.1um_NIR_Reflectance_Test':
+ hicut = c[0] + c[1]*sza + c[2]*np.power(sza, 2) + c[3]*np.power(sza, 3) + c[4]*np.power(sza, 4)
+ else:
+ pass
+ hicut = (hicut * 0.01) + nir_thresh['adj']
+ hicut = (hicut * nir_thresh['bias'])
+ midpt = hicut + (nir_thresh['midpt_coeff'] * nir_thresh['bias'])
+ locut = midpt + (nir_thresh['locut_coeff'] * nir_thresh['bias'])
+ thr = np.array([locut, midpt, hicut, nir_thresh['thr'][3]*np.ones(refang.shape)])
+
+ cosvza = np.cos(data.sensor_zenith.values[scene_idx]*_DTR).ravel()
+
+ corr_thr = np.zeros((4, refang.shape[0]))
+ corr_thr[:3, sunglint_flag == 0] = thr[:3, sunglint_flag == 0] * (1./np.power(cosvza[sunglint_flag == 0],
+ vzcpow))
+ corr_thr[3, sunglint_flag == 0] = thr[3, sunglint_flag == 0]
+ for flag in range(1, 4):
+ if len(refang[sunglint_flag == flag]) > 0:
+ dosgref = utils.get_sunglint_thresholds(refang, sunglint_thresholds, band_n, flag, thr)
+ corr_thr[:3, sunglint_flag == flag] = dosgref[:3, sunglint_flag == flag] * \
+ (1./np.power(cosvza[sunglint_flag == flag], vzcpow))
+ corr_thr[3, sunglint_flag == flag] = dosgref[3, sunglint_flag == flag]
+
+ return corr_thr
+
+
+def thresholds_surface_temperature(data, thresholds, scene_idx):
+ # def preproc_surf_temp(data, thresholds):
+ thr_sfc1 = thresholds['desert_thr']
+ thr_sfc2 = thresholds['regular_thr']
+ thr_df1 = thresholds['channel_diff_11-12um_thr']
+ thr_df2 = thresholds['channel_diff_11-4um_thr']
+ max_vza = 70.13 # This values is set based on sensor. Check mask_processing_constants.h for MODIS value
+
+ df1 = (data.M15 - data.M16).values[scene_idx].ravel()
+ df2 = (data.M15 - data.M13).values[scene_idx].ravel()
+ desert_flag = data.Desert.values[scene_idx].ravel()
+ thresh = np.ones(df1.shape) * thr_sfc1
+
+ idx = np.where((df1 >= thr_df1[0]) | ((df1 < thr_df1[0]) & ((df2 <= thr_df2[0]) | (df2 >= thr_df2[1]))))
+ thresh[idx] = thr_sfc2
+ idx = np.where(desert_flag == 1)
+ thresh[idx] == thr_sfc1
+
+ midpt = thresh
+ idx = np.where(df1 >= thr_df1[1])
+ midpt[idx] = thresh[idx] + 2.0*df1[idx]
+
+ corr = np.power(data.sensor_zenith.values[scene_idx].ravel()/max_vza, 4) * 3.0
+ midpt = midpt + corr
+ locut = midpt + 2.0
+ hicut = midpt - 2.0
+
+ thr_out = np.dstack((locut, midpt, hicut, np.ones(locut.shape), np.ones(locut.shape)))
+
+ return np.squeeze(thr_out.T)
+
+
+# This function is currently not used
+def preproc_sst(data, thresholds):
+ m31c = data.M15 - 273.16
+ m32c = data.M16 - 273.16
+ m31c_m32c = m31c - m32c
+ sstc = data.geos_sfct - 273.16
+ cosvza = np.cos(data.sensor_zenith*_DTR)
+
+ a = thresholds['coeffs']
+
+ modsst = 273.16 + a[0] + a[1]*m31c + a[2]*m31c_m32c*sstc + a[3]*m31c_m32c*((1/cosvza) - 1)
+ sfcdif = data.geos_sfct - modsst
+
+ return sfcdif
+
+
+def var_11um(data, thresholds):
+ rad = data.M15.values
+ var = np.zeros((rad.shape[0], rad.shape[1], 9))
+ var_thr = thresholds['Daytime_Ocean_Spatial_Variability']['dovar11']
+
+ test = sliding_window_view(np.pad(rad, [1, 1], mode='constant'), (3, 3)) - np.expand_dims(rad, (2, 3))
+
+ var[np.abs(test).reshape(rad.shape[0], rad.shape[1], 9) < var_thr] = 1
+ var = var.sum(axis=2)
+
+ return var
+
+
+def get_b1_thresholds(data, thresholds, scene_idx):
+ ndvi = data.ndvi.values[scene_idx].ravel()
+ sctang = data.scattering_angle.values[scene_idx].ravel()
+
+ # this is hardcoded in the function
+ delta_ndvi_bin = 0.1
+
+ des_ndvi = thresholds['Misc']['des_ndvi']
+ thr_adj_fac_desert = thresholds['Misc']['adj_fac_desert']
+ thr_adj_fac_land = thresholds['Misc']['adj_fac_land']
+ ndvi_bnd1 = thresholds['Misc']['ndvi_bnd1']
+ ndvi_bnd2 = thresholds['Misc']['ndvi_bnd2']
+ fill_ndvi = thresholds['Misc']['fill_ndvi']
+
+ coeff1 = np.array(thresholds['Coeffs_Band1_land_thresh']).reshape(10, 3, 4)
+ coeff2 = np.zeros((10, 3, 4))
+ coeff2[:3, :, :] = np.array(thresholds['Coeffs_Band8_land_thresh']).reshape(3, 3, 4)
+ coeff = np.stack((coeff1, coeff2))
+
+ indvi = np.zeros(ndvi.shape)
+ indvi[ndvi >= ndvi_bnd2] = 9
+
+ x, y2 = np.zeros(ndvi.shape), np.zeros(ndvi.shape)
+
+ # this is equivalent to interp=1 in the C code
+ idx = np.nonzero((ndvi >= ndvi_bnd1) & (ndvi < ndvi_bnd2))
+
+ indvi[idx] = (ndvi[idx]/delta_ndvi_bin) - 0.5
+ indvi[ndvi < 0] = 0
+
+ x1 = delta_ndvi_bin*indvi + delta_ndvi_bin/2.0
+ x2 = x1 + delta_ndvi_bin
+ x[idx] = (ndvi[idx] - x1[idx])/(x2[idx] - x1[idx])
+ x = np.clip(x, 0, 1)
+
+ indvi = np.array(indvi, dtype=np.int)
+ thr = np.empty((ndvi.shape[0], 4))
+ thr_adj = np.empty((ndvi.shape[0], 4))
+
+ for i in range(3):
+ y1 = coeff[0, indvi, i, 0] + coeff[0, indvi, i, 1]*sctang + \
+ coeff[0, indvi, i, 2]*sctang**2 + coeff[0, indvi, i, 3]*sctang**3
+ des = np.nonzero(ndvi < des_ndvi)
+ y1[des] = coeff[1, indvi[des], i, 0] + coeff[1, indvi[des], i, 1]*sctang[des] + \
+ coeff[1, indvi[des], i, 2]*sctang[des]**2 + coeff[1, indvi[des], i, 3]*sctang[des]**3
+
+ y2[idx] = coeff[0, indvi[idx], i, 0] + \
+ coeff[0, indvi[idx], i, 1]*sctang[idx] + \
+ coeff[0, indvi[idx], i, 2]*sctang[idx]**2 + \
+ coeff[0, indvi[idx], i, 3]*sctang[idx]**3
+ idxdes = np.nonzero((ndvi >= ndvi_bnd1) & (ndvi < ndvi_bnd2) & (ndvi < des_ndvi))
+ y2[idxdes] = coeff[0, indvi[idxdes], i, 0] + \
+ coeff[0, indvi[idxdes], i, 1]*sctang[idxdes] + \
+ coeff[0, indvi[idxdes], i, 2]*sctang[idxdes]**2 + \
+ coeff[0, indvi[idxdes], i, 3]*sctang[idxdes]**3
+
+ thr[:, i] = (1.0 - x) + (x + y2)
+ thr_adj[:, i] = thr[:, i] * thr_adj_fac_desert
+ thr_adj[ndvi >= des_ndvi, i] = thr[ndvi >= des_ndvi, i] * thr_adj_fac_land
+
+ hicut = ((thr[:, 0] + thr_adj[:, 0])/100) # .reshape(data.ndvi.shape)
+ midpt = ((thr[:, 1] + thr_adj[:, 1])/100) # .reshape(data.ndvi.shape)
+ locut = ((thr[:, 2] + thr_adj[:, 2])/100) # .reshape(data.ndvi.shape)
+
+ idx = np.nonzero((ndvi >= fill_ndvi[0]) | (ndvi <= fill_ndvi[1]))
+ hicut[idx] = -999
+ midpt[idx] = -999
+ locut[idx] = -999
+
+# out_thr = xr.DataArray(data=np.dstack((locut, midpt, hicut, np.ones(data.ndvi.shape),
+# np.full(data.ndvi.shape, 2))),
+# dims=('number_of_lines', 'number_of_pixels', 'z'))
+#
+# return out_thr
+ return locut, midpt, hicut
+
+
+def polar_night_thresholds(data, thresholds, scene, test_name, scene_idx):
+
+ thresholds = thresholds[scene]
+ if ((test_name == '4-12um_BTD_Thin_Cirrus_Test') and (scene in ['Land_Night', 'Night_Snow']) or
+ (test_name == '7.3-11um_BTD_Mid_Level_Cloud_Test') and (scene == 'Land_Night')):
+ locut = thresholds[test_name]['thr'][0] * np.ones(data.M15.values[scene_idx].ravel())
+ midpt = thresholds[test_name]['thr'][1] * np.ones(data.M15.values[scene_idx].ravel())
+ hicut = thresholds[test_name]['thr'][2] * np.ones(data.M15.values[scene_idx].ravel())
+ power = thresholds[test_name]['thr'][3] * np.ones(data.M15.values[scene_idx].ravel())
+
+ # out_thr = xr.DataArray(data=np.dstack((locut, midpt, hicut, np.ones(data.ndvi.shape), power)),
+ # dims=('number_of_lines', 'number_of_pixels', 'z'))
+ out_thr = np.dstack((locut, midpt, hicut, np.ones(locut.shape), power))
+
+ return out_thr.T
+
+ rad = data.M15.values[scene_idx].ravel()
+ bt_bounds = thresholds[test_name]['bt11_bounds']
+
+ locut, midpt = np.empty(rad.shape), np.empty(rad.shape)
+ hicut, power = np.empty(rad.shape), np.empty(rad.shape)
+ lo, hi = np.empty(rad.shape), np.empty(rad.shape)
+ lo_thr, hi_thr = np.empty(rad.shape), np.empty(rad.shape)
+ conf_range = np.empty(rad.shape)
+
+ idx = np.nonzero(rad < bt_bounds[0])
+ locut[idx] = thresholds[test_name]['low'][0]
+ midpt[idx] = thresholds[test_name]['low'][1]
+ hicut[idx] = thresholds[test_name]['low'][2]
+ power[idx] = thresholds[test_name]['low'][3]
+
+ idx = np.nonzero(rad > bt_bounds[3])
+ locut[idx] = thresholds[test_name]['high'][0]
+ midpt[idx] = thresholds[test_name]['high'][1]
+ hicut[idx] = thresholds[test_name]['high'][2]
+ power[idx] = thresholds[test_name]['high'][3]
+
+ # # # # #
+ idx = np.nonzero((rad >= bt_bounds[0]) & (rad <= bt_bounds[3]) &
+ (bt_bounds[1] == 0) & (bt_bounds[2] == 0))
+ lo[idx] = thresholds[test_name]['bt11_bounds'][0]
+ hi[idx] = thresholds[test_name]['bt11_bounds'][3]
+ lo_thr[idx] = thresholds[test_name]['mid1'][0]
+ hi_thr[idx] = thresholds[test_name]['mid1'][1]
+ power[idx] = thresholds[test_name]['mid1'][3]
+ conf_range[idx] = thresholds[test_name]['mid1'][2]
+
+ idx = np.nonzero((rad >= bt_bounds[0]) & (rad < bt_bounds[1]))
+ lo[idx] = thresholds[test_name]['bt11_bounds'][0]
+ hi[idx] = thresholds[test_name]['bt11_bounds'][1]
+ lo_thr[idx] = thresholds[test_name]['mid1'][0]
+ hi_thr[idx] = thresholds[test_name]['mid1'][1]
+ power[idx] = thresholds[test_name]['mid1'][3]
+ conf_range[idx] = thresholds[test_name]['mid1'][2]
+
+ idx = np.nonzero((rad >= bt_bounds[1]) & (rad < bt_bounds[2]))
+ lo[idx] = thresholds[test_name]['bt11_bounds'][1]
+ hi[idx] = thresholds[test_name]['bt11_bounds'][2]
+ lo_thr[idx] = thresholds[test_name]['mid2'][0]
+ hi_thr[idx] = thresholds[test_name]['mid2'][1]
+ power[idx] = thresholds[test_name]['mid2'][3]
+ conf_range[idx] = thresholds[test_name]['mid2'][2]
+
+ idx = np.nonzero((rad >= bt_bounds[2]) & (rad < bt_bounds[3]))
+ lo[idx] = thresholds[test_name]['bt11_bounds'][2]
+ hi[idx] = thresholds[test_name]['bt11_bounds'][3]
+ lo_thr[idx] = thresholds[test_name]['mid3'][0]
+ hi_thr[idx] = thresholds[test_name]['mid3'][1]
+ power[idx] = thresholds[test_name]['mid3'][3]
+ conf_range[idx] = thresholds[test_name]['mid3'][2]
+
+ idx = np.nonzero(((rad >= bt_bounds[0]) & (rad < bt_bounds[3])) |
+ (bt_bounds[1] == 0.0) | (bt_bounds[2] == 0))
+
+ a = (rad[idx] - lo[idx])/(hi[idx] - lo[idx])
+ midpt[idx] = lo_thr[idx] + (a*(hi_thr[idx] - lo_thr[idx]))
+ hicut[idx] = midpt[idx] - conf_range[idx]
+ locut[idx] = midpt[idx] + conf_range[idx]
+
+ # locut = locut.reshape(data.M15.shape)
+ # midpt = midpt.reshape(data.M15.shape)
+ # hicut = hicut.reshape(data.M15.shape)
+ # power = power.reshape(data.M15.shape)
+
+ # out_thr = xr.DataArray(data=np.dstack((locut, midpt, hicut, np.ones(data.ndvi.shape), power)),
+ # dims=('number_of_lines', 'number_of_pixels', 'z'))
+ out_thr = np.dstack((locut, midpt, hicut, np.ones(locut.shape), power))
+
+ return out_thr.T
+
+
+# get_nl_thresholds
+def land_night_thresholds(data, threshold, coast=True):
+ if coast is False:
+ lo_val = threshold['bt_diff_bounds'][0]
+ hi_val = threshold['bt_diff_bounds'][1]
+ lo_val_thr = threshold['thr_mid'][0]
+ hi_val_thr = threshold['thr_mid'][1]
+ conf_range = threshold['thr_mid'][2]
+
+ a = (data['M15-M16'].values - lo_val) / (hi_val - lo_val)
+ midpt = lo_val_thr + a*(hi_val_thr - lo_val_thr)
+ hicut = midpt - conf_range
+ locut = midpt + conf_range
+
+ power = np.full(midpt.shape, threshold['thr_mid'][3])
+
+ idx = np.nonzero(data['M15-M16'].values > threshold['bt_diff_bounds'][0])
+ locut[idx] = threshold['thr_low'][0]
+ midpt[idx] = threshold['thr_low'][1]
+ hicut[idx] = threshold['thr_low'][2]
+ power[idx] = threshold['thr_low'][3]
+
+ idx = np.nonzero(data['M15-M16'].values < threshold['bt_diff_bounds'][1])
+ locut[idx] = threshold['thr_hi'][0]
+ midpt[idx] = threshold['thr_hi'][1]
+ hicut[idx] = threshold['thr_hi'][2]
+ power[idx] = threshold['thr_hi'][3]
+
+ out_thr = np.dstack((locut.ravel(), midpt.ravel(), hicut.ravel(),
+ np.ones(locut.ravel().shape), power.ravel()))
+ return np.squeeze(out_thr.T)
+ else:
+ b0 = threshold['coeffs'][0]
+ b1 = threshold['coeffs'][1]
+ b2 = threshold['coeffs'][2]
+
+ thr = b0 + threshold['int_adj'] + b1*data.geos_tpw.values + b2*data.geos_tpw.values**2
+ hicut = np.empty((2, thr.shape[0], thr.shape[1]))
+ midpt = np.empty((2, thr.shape[0], thr.shape[1]))
+ locut = np.empty((2, thr.shape[0], thr.shape[1]))
+ hicut[0, :] = thr - 1
+ hicut[1, :] = thr + 1
+ midpt[0, :] = hicut[0, :] - 0.5
+ midpt[1, :] = hicut[1, :] + 0.5
+ locut[0, :] = hicut[0, :] - 1
+ locut[1, :] = hicut[1, :] + 1
+
+ out_thr = np.dstack((hicut[0, :], midpt[0, :], locut[0, :], locut[1, :], midpt[1, :], hicut[1, :],
+ np.ones(locut[0, :].shape), np.ones(locut[0, :].shape)))
+ return np.moveaxis(out_thr, -1, 0)
+
+
+def vis_refl_thresholds(data, thresholds, scene, scene_idx):
+
+ locut, midpt, hicut = get_b1_thresholds(data, thresholds, scene_idx)
+ bias_adj = thresholds[scene]['Visible_Reflectance_Test']['adj']
+ ndvi = data.ndvi.values[scene_idx].ravel()
+ m01 = data.M05.values[scene_idx].ravel()
+ m02 = data.M07.values[scene_idx].ravel()
+ m08 = data.M01.values[scene_idx].ravel()
+
+ m128 = m01
+
+ b1_locut = locut * bias_adj
+ b1_midpt = midpt * bias_adj
+ b1_hicut = hicut * bias_adj
+
+ if ((scene == 'Land_Day_Desert') | (scene == 'Land_Day_Desert_Coast')):
+ ndvi_desert_thr = thresholds[scene]['Visible_Reflectance_Test']['ndvi_thr']
+ idx = np.nonzero(ndvi < ndvi_desert_thr)
+
+ b1_locut[idx] = locut[idx]
+ b1_midpt[idx] = midpt[idx]
+ b1_hicut[idx] = hicut[idx]
+ m128[idx] = m08[idx]
+
+ b1_power = np.full(b1_locut.shape, 2)
+
+ idx = np.nonzero(locut == -999)
+ b1_locut[idx] = thresholds[scene]['Visible_Reflectance_Test']['thr'][0]
+ b1_midpt[idx] = thresholds[scene]['Visible_Reflectance_Test']['thr'][1]
+ b1_hicut[idx] = thresholds[scene]['Visible_Reflectance_Test']['thr'][2]
+ b1_power[idx] = thresholds[scene]['Visible_Reflectance_Test']['thr'][3]
+ m128[idx] = m02[idx]
+
+ cosvza = np.cos(data.sensor_zenith.values[scene_idx] * _DTR).ravel()
+
+ vzcpow = thresholds['VZA_correction']['vzcpow'][0]
+ b1_locut = (b1_locut * (1.0 / np.power(cosvza, vzcpow)))
+ b1_midpt = (b1_midpt * (1.0 / np.power(cosvza, vzcpow)))
+ b1_hicut = (b1_hicut * (1.0 / np.power(cosvza, vzcpow)))
+
+ # out_thr = xr.DataArray(data=np.dstack((b1_locut, b1_midpt, b1_hicut, np.ones(data.ndvi.shape),
+ # b1_power.reshape(data.ndvi.shape))),
+ # dims=('number_of_lines', 'number_of_pixels', 'z'))
+ # out_rad = xr.DataArray(data=m128.reshape(data.M01.shape), dims=('number_of_lines', 'number_of_pixels'))
+ out_thr = np.dstack((b1_locut, b1_midpt, b1_hicut, b1_power))
+ out_rad = m128
+ return np.squeeze(out_thr.T), out_rad
+
+
+def gemi_thresholds(data, thresholds, scene_name, scene_idx):
+ ndvi = data.ndvi.values[scene_idx].ravel()
+ gemi_thr = np.ones((ndvi.shape[0], 5))
+
+ idx = np.nonzero(ndvi < 0.1)
+ gemi_thr[idx, :3] = thresholds['gemi0'][:3]
+ idx = np.nonzero((ndvi >= 0.1) & (ndvi < 0.2))
+ gemi_thr[idx, :3] = thresholds['gemi1'][:3]
+ idx = np.nonzero((ndvi >= 0.2) & (ndvi < 0.3))
+ gemi_thr[idx, :3] = thresholds['gemi2'][:3]
+
+ # thr_out = xr.DataArray(data=np.dstack((gemi_thr[:, :, 0], gemi_thr[:, :, 1], gemi_thr[:, :, 2],
+ # np.ones(gemi_thr[:, :, 0].shape),
+ # np.ones(gemi_thr[:, :, 0].shape))),
+ # dims=('number_of_lines', 'number_of_pixels', 'z'))
+
+ return gemi_thr.T
+
+
+def bt_diff_11_4um_thresholds(data, threshold, scene_idx):
+ c = threshold['coeffs']
+ tpw = data.geos_tpw.values[scene_idx].ravel()
+
+ thr = c[0] + threshold['corr'] + c[1]*tpw + c[2]*np.power(tpw, 2)
+
+ hicut0 = thr + threshold['hicut_coeff'][0]
+ hicut1 = thr + threshold['hicut_coeff'][1]
+ midpt0 = hicut0 + threshold['midpt_coeff'][0]
+ midpt1 = hicut1 + threshold['midpt_coeff'][1]
+ locut0 = hicut0 + threshold['locut_coeff'][0]
+ locut1 = hicut1 + threshold['locut_coeff'][1]
+
+ thr_out = np.dstack([locut0, midpt0, hicut0, hicut1, midpt1, locut1,
+ np.ones(hicut0.shape), np.ones(hicut0.shape)])
+ return np.squeeze(thr_out.T)
+
+
+def thresholds_1_38um_test(data, thresholds, scene_name, scene_idx):
+ sza = data.solar_zenith.values[scene_idx]
+ vza = data.sensor_zenith.values[scene_idx]
+ thresh = thresholds[scene_name]['1.38um_High_Cloud_Test']
+ c = thresh['coeffs']
+ vzcpow = thresholds['VZA_correction']['vzcpow'][1]
+
+ hicut = c[0] + c[1]*sza + c[2]*np.power(sza, 2) + c[3]*np.power(sza, 3) + \
+ c[4]*np.power(sza, 4) + c[5]*np.power(sza, 5)
+ hicut = hicut*0.01 + (np.maximum((sza/90.)*thresh['szafac']*thresh['adj'], thresh['adj']))
+ midpt = hicut + 0.001
+ locut = midpt + 0.001
+
+ cosvza = np.cos(vza*_DTR)
+
+ locut = locut * (1/np.power(cosvza, vzcpow))
+ midpt = midpt * (1/np.power(cosvza, vzcpow))
+ hicut = hicut * (1/np.power(cosvza, vzcpow))
+
+ # out_thr = xr.DataArray(data=np.dstack((locut, midpt, hicut, np.ones(data.ndvi.shape),
+ # np.ones(data.ndvi.shape))),
+ # dims=('number_of_lines', 'number_of_pixels', 'z'))
+ out_thr = np.dstack((locut, midpt, hicut, np.ones(locut.shape), np.ones(locut.shape)))
+
+ return np.squeeze(out_thr.T)
+
+# NOTE: 11-12um Cirrus Test
+# hicut is computed in different ways depending on the scene
+# 1. midpt - adj
+# - Land_Day
+# - Land_Day_Coast
+# - Land_Day_Desert
+# - Land_Day_Desert_Coast
+# - Ocean_Day
+# - Ocean_Night
+# - Polar_Day_Ocean
+# - Polar_Night_Ocean
+#
+# 2. midpt - (btd_thr * adj)
+# - Polar_Day_Land
+# - Polar_Day_Coast
+# - Polar_Day_Desert
+# - Polar_Day_Desert_Coast
+# - Polar_Day_Snow
+#
+# 3. Others
+# - Land_Night
+# - Polar_Night_Land
+# - Polar_Night_Snow
+# - Day_Snow
+# - Night_Snow
+
+# NOTE: 1.38um High Cloud Test
+# thresholds are not always computed the same way. In group 1 there's no preprocessing required,
+# in group 2 some calcuations are needed
+# 1.
+# - Land_Day
+# - Land_Day_Coast
+# - Land_Day_Desert
+# - Land_Day_Desert_Coast
+# - Polar_Day_Land
+# - Polar_Day_Coast
+# - Polar_Day_Desert
+# - Polar_Day_Desert_Coast
+# - Polar_Day_Snow
+# - Day_Snow
+#
+# 2.
+# - Ocean_Day
+# - Polar_Ocean_Day
+
diff --git a/mvcm/read_data.py b/mvcm/read_data.py
new file mode 100644
index 0000000..4e95231
--- /dev/null
+++ b/mvcm/read_data.py
@@ -0,0 +1,581 @@
+import xarray as xr
+import numpy as np
+import numpy.typing as npt
+
+import ancillary_data as anc
+
+import os
+import logging
+from datetime import datetime as dt
+from typing import Dict
+from attrs import define, field, validators, Factory
+
+_DTR = np.pi/180.
+_RTD = 180./np.pi
+_bad_data = -999.0
+_datapath = '/ships19/hercules/pveglio/mvcm_test_data'
+
+logging.basicConfig(level=logging.INFO, format='%(name)s - %(levelname)s - %(message)s')
+# logging.basicConfig(level=logging.INFO, filename='logfile.log', 'filemode='w',
+# format='%(name)s %(levelname)s %(message)s')
+
+
+@define(kw_only=True, slots=True)
+class CollectInputs(object):
+ """Class that collects all input files
+
+ Parameters
+ ----------
+ file_name_geo: str
+ name of VIIRS geolocation file
+ file_name_l1b: str
+ name of VIIRS L1b file
+ ancillary_dir: str
+ path for the ancillary files. (this is necessary because the C functions
+ still require path and file name separately)
+ sst_file: str
+ file name for the Reynolds SST
+ ndvi_file: str
+ file name for the NDVI hdf4 file
+ geos_file_before: str
+ file name of the GEOS-5 containing atmospheric profiles before the VIIRS timestamp
+ geos_file_after: str
+ file name of the GEOS-5 containing atmospheric profiles after the VIIRS timestamp
+ geos_land: str
+ file name of the GEOS-5 file that has the land ice/snow information
+ geos_ocean: str
+ file name of the GEOS-5 file that has the ocean information
+ geos_constants: str
+ file name for the GEOS-5 constans
+ dims [optional]: str
+ name of the dimensions for the arrays in the xarray.Dataset output
+ """
+ file_name_geo: str = field(default=f'{_datapath}/VNP03MOD.A2022173.1312.001.2022174012746.uwssec.nc',
+ validator=[validators.instance_of(str), ])
+ file_name_l1b: str = field(default=f'{_datapath}/VNP02MOD.A2022173.1312.001.2022174011547.uwssec.nc',
+ validator=[validators.instance_of(str), ])
+ ancillary_dir: str = field(default=f'{_datapath}/ancillary',
+ validator=[validators.instance_of(str), ])
+ sst_file: str = field(default='oisst.20220622',
+ validator=[validators.instance_of(str), ])
+ eco_file: str = field(default='goge1_2_img.v1',
+ validator=[validators.instance_of(str), ])
+ ndvi_file: str = field(default='NDVI.FM.c004.v2.0.WS.00-04.177.hdf',
+ validator=[validators.instance_of(str), ])
+ geos_file_1: str = field(default='GEOS.fpit.asm.inst3_2d_asm_Nx.GEOS5124.20220622_1200.V01.nc4',
+ validator=[validators.instance_of(str), ])
+ geos_file_2: str = field(default='GEOS.fpit.asm.inst3_2d_asm_Nx.GEOS5124.20220622_1500.V01.nc4',
+ validator=[validators.instance_of(str), ])
+ geos_land: str = field(default='GEOS.fpit.asm.tavg1_2d_lnd_Nx.GEOS5124.20220622_1330.V01.nc4',
+ validator=[validators.instance_of(str), ])
+ geos_ocean: str = field(default='GEOS.fpit.asm.tavg1_2d_ocn_Nx.GEOS5124.20220622_1330.V01.nc4',
+ validator=[validators.instance_of(str), ])
+ geos_constants: str = field(default='GEOS.fp.asm.const_2d_asm_Nx.00000000_0000.V01.nc4',
+ validator=[validators.instance_of(str), ])
+
+ dims: tuple = field(default=('number_of_lines', 'number_of_pixels'),
+ validator=[validators.instance_of(tuple), ])
+
+
+@define(slots=True, kw_only=True)
+class ReadData(CollectInputs):
+ """Class that reads the L1b/geolocation data from VIIRS. Inherits file names from CollectInputs
+
+ Parameters
+ ----------
+ satellite: str
+ satellite name.
+ sensor: str
+ sensor name
+ """
+ satellite: str = field(validator=[validators.instance_of(str),
+ validators.in_(['snpp'])])
+ sensor: str = field(validator=[validators.instance_of(str),
+ validators.in_(['viirs'])])
+
+ logging.debug('Instance of ReadData created')
+
+ def read_viirs_geo(self) -> xr.Dataset:
+ """Read VIIRS geolocation data and generate additional angles
+
+ Parameters
+ ----------
+ self.file_name_geo: str
+ file name of the geolocation file
+
+ Returns
+ -------
+ geo_data xarray.Dataset
+ dataset containing all geolocation data
+ """
+ logging.debug(f'Reading {self.file_name_geo}')
+ geo_data = xr.open_dataset(self.file_name_geo, group='geolocation_data')
+
+ relazi = self.relative_azimuth_angle(geo_data.sensor_azimuth.values, geo_data.solar_azimuth.values)
+ sunglint = self.sun_glint_angle(geo_data.sensor_zenith.values, geo_data.solar_zenith.values, relazi)
+ scatt_angle = self.scattering_angle(geo_data.solar_zenith.values, geo_data.sensor_zenith.values,
+ relazi)
+
+ geo_data['relative_azimuth'] = (self.dims, relazi)
+ geo_data['sunglint_angle'] = (self.dims, sunglint)
+ geo_data['scattering_angle'] = (self.dims, scatt_angle)
+
+ logging.debug('Geolocation file read correctly')
+
+ return geo_data
+
+ def read_viirs_l1b(self,
+ solar_zenith: npt.NDArray[float]) -> xr.Dataset:
+ """Read VIIRS L1b data
+
+ Parameters
+ ----------
+ self.file_name_l1b: str
+ file name of the L1b file
+ solar_zenith: np.ndarray
+ solar zenith angle derived from the geolocation file
+ """
+ logging.debug(f'Reading {self.file_name_l1b}')
+ l1b_data = xr.open_dataset(self.file_name_l1b, group='observation_data', decode_cf=False)
+
+ rad_data = xr.Dataset()
+ for band in list(l1b_data.variables):
+ if 'reflectance' in l1b_data[band].long_name:
+ if hasattr(l1b_data[band], 'VCST_scale_factor'):
+ scale_factor = l1b_data[band].VCST_scale_factor * l1b_data[band].bias_correction
+ else:
+ scale_factor = l1b_data[band].scale_factor
+ rad_data[band] = (self.dims, l1b_data[band].values * scale_factor / np.cos(solar_zenith*_DTR))
+
+ elif 'radiance' in l1b_data[band].long_name:
+ bt_lut = f'{band}_brightness_temperature_lut'
+ rad_data[band] = (self.dims,
+ l1b_data[bt_lut].values[l1b_data[band].values])
+ else:
+ pass
+
+ logging.debug('L1b file read correctly')
+
+ return rad_data
+
+ def preprocess_viirs(self,
+ geo_data: xr.Dataset,
+ viirs: xr.Dataset) -> xr.Dataset:
+ """Create band combinations (e.g. ratios, differences) that are used by some spectral tests.
+
+ Parameters
+ ----------
+ geo_data: xarray.Dataset
+ dataset containing all geolocation data
+ viirs: xr.Dataset
+ VIIRS L1b data
+
+ Returns
+ -------
+ viirs: xr.Dataset
+ VIIRS L1b data plus band combinations for the spectral tests
+ """
+ if (('M05' in viirs) and ('M07' in viirs)):
+ m01 = viirs.M05.values
+ m02 = viirs.M07.values
+ r1 = 2.0 * (np.power(m02, 2.0) - np.power(m01, 2.0)) + (1.5 * m02) + (0.5 * m01)
+ r2 = m01 + m02 + 0.5
+ r3 = r1 / r2
+ gemi = r3 * (1.0 - 0.25*r3) - ((m01 - 0.125) / (1.0 - m01))
+ else:
+ gemi = np.full((viirs.M15.shape), _bad_data)
+
+ """
+ if 'M05' in viirs:
+ idx = np.nonzero((viirs.M05.values < -99) | (viirs.M05.values > 2))
+ viirs['M05'].values[idx] = _bad_data
+ else:
+ viirs['M05'] = (self.dims, np.full(viirs.M15.shape, _bad_data))
+ if 'M07' in viirs:
+ idx = np.nonzero((viirs.M07.values < -99) | (viirs.M07.values > 2))
+ viirs['M07'].values[idx] = _bad_data
+ else:
+ viirs['M07'] = (self.dims, np.full(viirs.M15.shape, _bad_data))
+
+ idx = np.nonzero((viirs.M12.values < 0) | (viirs.M12.values > 1000))
+ viirs['M12'].values[idx] = _bad_data
+ idx = np.nonzero((viirs.M13.values < 0) | (viirs.M13.values > 1000))
+ viirs['M13'].values[idx] = _bad_data
+ idx = np.nonzero((viirs.M14.values < 0) | (viirs.M14.values > 1000))
+ viirs['M14'].values[idx] = _bad_data
+ idx = np.nonzero((viirs.M15.values < 0) | (viirs.M15.values > 1000))
+ viirs['M15'].values[idx] = _bad_data
+ idx = np.nonzero((viirs.M16.values < 0) | (viirs.M16.values > 1000))
+ viirs['M16'].values[idx] = _bad_data
+ """
+
+ # Compute channel differences and ratios that are used in the tests
+ if (('M05' in viirs) and ('M07' in viirs)):
+ viirs['M07-M05ratio'] = (self.dims, viirs.M07.values / viirs.M05.values)
+ else:
+ viirs['M07-M05ratio'] = (self.dims, np.full(viirs.M15.shape, _bad_data))
+ viirs['M13-M16'] = (self.dims, viirs.M13.values - viirs.M16.values)
+ viirs['M14-M15'] = (self.dims, viirs.M14.values - viirs.M15.values)
+ viirs['M15-M12'] = (self.dims, viirs.M15.values - viirs.M12.values)
+ viirs['M15-M13'] = (self.dims, viirs.M15.values - viirs.M13.values)
+ viirs['M15-M16'] = (self.dims, viirs.M15.values - viirs.M16.values)
+ viirs['GEMI'] = (self.dims, gemi)
+
+ # temp value to force the code to work
+ viirs['M128'] = (self.dims, np.zeros(viirs.M15.shape))
+
+ viirs.update(geo_data)
+ viirs = viirs.set_coords(['latitude', 'longitude'])
+
+ logging.debug('Viirs preprocessing completed successfully.')
+ return viirs
+
+ def relative_azimuth_angle(self,
+ sensor_azimuth: npt.NDArray[float],
+ solar_azimuth: npt.NDArray[float]) -> npt.NDArray[float]:
+ """Computation of the relative azimuth angle
+
+ Parameters
+ ----------
+ sensor_azimuth: np.ndarray
+ sensor azimuth angle from the geolocation file
+ solar_azimuth: np.ndarray
+ solar azimuth angle from the geolocation file
+
+ Returns
+ -------
+ relative_azimuth: np.ndarray
+ """
+ rel_azimuth = np.abs(180. - np.abs(sensor_azimuth - solar_azimuth))
+
+ logging.debug('Relative azimuth calculated successfully.')
+
+ return rel_azimuth
+
+ def sun_glint_angle(self,
+ sensor_zenith: npt.NDArray[float],
+ solar_zenith: npt.NDArray[float],
+ rel_azimuth: npt.NDArray[float]) -> npt.NDArray[float]:
+ """Computation of the sun glint angle
+
+ Parameters
+ ----------
+ sensor_zenith: np.ndarray
+ sensor zenith angle from the geolocation file
+ solar_zenith: np.ndarray
+ solar zenith angle from the geolocation file
+ relative_azimuth: np.ndarray
+ relative azimuth computed from function relative_azimuth_angle()
+
+ Returns
+ -------
+ sunglint_angle: np.ndarray
+ """
+ cossna = (np.sin(sensor_zenith*_DTR) * np.sin(solar_zenith*_DTR) * np.cos(rel_azimuth*_DTR) +
+ np.cos(sensor_zenith*_DTR) * np.cos(solar_zenith*_DTR))
+ cossna[cossna > 1] = 1
+ sunglint_angle = np.arccos(cossna) * _RTD
+
+ logging.debug('Sunglint generated')
+
+ return sunglint_angle
+
+ def scattering_angle(self,
+ solar_zenith: npt.NDArray[float],
+ sensor_zenith: npt.NDArray[float],
+ relative_azimuth: npt.NDArray[float]) -> npt.NDArray[float]:
+ """Computation of the scattering angle
+
+ Parameters
+ ----------
+ solar_zenith: np.ndarray
+ solar zenith angle from the geolocation file
+ sensor_zenith: np.ndarray
+ sensor zenith angle angle from the geolocation file
+ relative_azimuth: np.ndarray
+ relative azimuth computed from function relative_azimuth_angle()
+
+ Returns
+ -------
+ scattering_angle: np.ndarray
+ """
+ cos_scatt_angle = -1. * (np.cos(solar_zenith*_DTR) * np.cos(sensor_zenith*_DTR) -
+ np.sin(solar_zenith*_DTR) * np.sin(sensor_zenith*_DTR) *
+ np.cos(relative_azimuth*_DTR))
+
+ scatt_angle = np.arccos(cos_scatt_angle) * _RTD
+
+ logging.debug('Scattering angle calculated successfully')
+
+ return scatt_angle
+
+
+@define(kw_only=True, slots=True)
+class ReadAncillary(CollectInputs):
+ """Class tha processes all the ancillary files and makes them available for the MVCM"
+
+ Parameters
+ ----------
+ latitude: npt.NDArray[float]
+ array of latitudes for the granule that is being processed
+ longitude: npt.NDArray[float]
+ array of longitudes for the granule that is being processed
+ resolution: int
+ flag to switch between MOD (1) and IMG (2) resolution
+ """
+ latitude: npt.NDArray[float] = field(validator=[validators.instance_of(np.ndarray), ])
+ longitude: npt.NDArray[float] = field(validator=[validators.instance_of(np.ndarray), ])
+ resolution: int = field(validator=[validators.instance_of(int),
+ validators.in_([1, 2]), ])
+
+ out_shape: tuple = field(init=False,
+ default=Factory(lambda self: self.latitude.shape, takes_self=True))
+
+ def get_granule_time(self):
+ """Get granule timestamp and format it for MVCM"""
+ vnp_time = '.'.join(os.path.basename(self.file_name_l1b).split('.')[1:3])
+ return dt.strftime(dt.strptime(vnp_time, 'A%Y%j.%H%M'), '%Y-%m-%d %H:%M:%S.000')
+
+ def get_sst(self) -> npt.NDArray[float]:
+ """Read Reynolds SST file
+
+ Parameters
+ ----------
+
+ Returns
+ -------
+ sst: npt.NDArray[float]
+ array containing the Reynolds SST interpolated at the sensor's resolution
+ """
+ if not os.path.isfile(os.path.join(self.ancillary_dir, self.sst_file)):
+ logging.error('SST file not found')
+ sst = np.empty(self.out_shape, dtype=np.float32).ravel()
+ sst = anc.py_get_Reynolds_SST(self.latitude.ravel(), self.longitude.ravel(), self.resolution,
+ self.ancillary_dir, self.sst_file, sst)
+ logging.debug('SST file read successfully')
+ return sst.reshape(self.out_shape)
+
+ def get_ndvi(self) -> npt.NDArray[float]:
+ """Read NDVI file
+
+ Parameters
+ ----------
+
+ Returns
+ -------
+ ndvi: npt.NDArray[float]
+ NDVI interpolated at the sensor's resolution
+ """
+ if not os.path.isfile(os.path.join(self.ancillary_dir, self.ndvi_file)):
+ logging.error('NDVI file not found')
+ ndvi = np.empty(self.out_shape, dtype=np.float32).ravel()
+ ndvi = anc.py_get_NDVI_background(self.latitude.ravel(), self.longitude.ravel(), self.resolution,
+ self.ancillary_dir, self.ndvi_file, ndvi)
+ logging.debug('NDVI file read successfully')
+ return ndvi.reshape(self.out_shape)
+
+ def get_eco(self) -> npt.NDArray[float]:
+ """Read ECO file
+
+ Parameters
+ ----------
+
+ Returns
+ -------
+ eco: npt.NDArray[float]
+ Olson ecosystem type interpolated at the sensor's resolution
+ """
+ eco = np.empty(self.out_shape, dtype=np.ubyte).ravel()
+ eco = anc.py_get_Olson_eco(self.latitude.ravel(), self.longitude.ravel(), self.resolution,
+ self.ancillary_dir, eco)
+ logging.debug('Olson ecosystem file read successfully')
+ return eco.reshape(self.out_shape)
+
+ def get_geos(self) -> Dict:
+ """Read GEOS-5 data and interpolate the fields to the sensor resolution
+
+ Parameters
+ ----------
+
+ Returns
+ -------
+ geos_data: Dict
+ dictionary containing all quantities required by MVCM (see geos_variables here below)
+ """
+ if not os.path.isfile(os.path.join(self.ancillary_dir, self.geos_file_1)):
+ logging.error('GEOS-5 file 1 not found')
+ if not os.path.isfile(os.path.join(self.ancillary_dir, self.geos_file_2)):
+ logging.error('GEOS-5 file 2 not found')
+ if not os.path.isfile(os.path.join(self.ancillary_dir, self.geos_land)):
+ logging.error('GEOS-5 land file not found')
+ if not os.path.isfile(os.path.join(self.ancillary_dir, self.geos_ocean)):
+ logging.error('GEOS-5 ocean file not found')
+ if not os.path.isfile(os.path.join(self.ancillary_dir, self.geos_constants)):
+ logging.error('GEOS-5 constants file not found')
+
+ geos_variables = ['tpw', 'snow_fraction', 'ice_fraction', 'ocean_fraction',
+ 'land_ice_fraction', 'surface_temperature']
+ geos_data = {var: np.empty(self.out_shape, dtype=np.float32).ravel() for var in geos_variables}
+
+ geos_data = anc.py_get_GEOS(self.latitude.ravel(), self.longitude.ravel(), self.resolution,
+ self.get_granule_time(), self.ancillary_dir, self.geos_file_1,
+ self.geos_file_2, self.geos_land, self.geos_ocean,
+ self.geos_constants, geos_data)
+
+ for var in list(geos_variables):
+ geos_data[var] = geos_data[var].reshape(self.out_shape)
+
+ logging.debug('GEOS data read successfully')
+ return geos_data
+
+ def pack_data(self) -> xr.Dataset:
+ """Pack all the ancillary data into a single dataset
+
+ Parameters
+ ----------
+
+ Returns
+ -------
+ ancillary_data: xr.Dataset
+ dataset containing all the ancillary data
+ """
+ ancillary_data = xr.Dataset()
+ ancillary_data['sst'] = (self.dims, self.get_sst())
+ ancillary_data['ecosystem'] = (self.dims, self.get_eco())
+ ancillary_data['ndvi'] = (self.dims, self.get_ndvi())
+
+ geos_tmp = self.get_geos()
+ for var in list(geos_tmp.keys()):
+ ancillary_data[f'geos_{var}'] = (self.dims, geos_tmp[var])
+
+ return ancillary_data
+
+
+"""
+ scene_flags = scn.find_scene(viirs, sunglint_angle)
+ scene = scn.scene_id(scene_flags)
+
+ scene_xr = xr.Dataset()
+ for s in scn._scene_list:
+ scene_xr[s] = (('number_of_lines', 'number_of_pixels'), scene[s])
+
+ scene['lat'] = viirs.latitude
+ scene['lon'] = viirs.longitude
+
+ data = xr.Dataset(viirs, coords=scene_xr)
+ data.drop_vars(['latitude', 'longitude'])
+
+
+
+def get_data(satellite: str,
+ sensor: str,
+ file_names: Dict[str, str],
+ sunglint_angle: float,
+ hires: bool = False) -> xr.Dataset:
+
+ mod02 = file_names['MOD02']
+ mod03 = file_names['MOD03']
+
+ if hires is True:
+ img02 = file_names['IMG02']
+ img03 = file_names['IMG03']
+
+ if hires is False:
+ viirs = read_viirs(sensor, f'{mod02}', f'{mod03}')
+ viirs = read_ancillary_data(file_names, viirs)
+
+ if (('M05' in viirs) and ('M07' in viirs)):
+ m01 = viirs.M05.values
+ m02 = viirs.M07.values
+ r1 = 2.0 * (np.power(m02, 2.0) - np.power(m01, 2.0)) + (1.5 * m02) + (0.5 * m01)
+ r2 = m02 + m01 + 0.5
+ r3 = r1 / r2
+ gemi = r3 * (1.0 - 0.25*r3) - ((m01 - 0.125) / (1.0 - m01))
+ else:
+ gemi = np.full((viirs.M15.shape), _bad_data)
+
+ if 'M05' in viirs:
+ idx = np.nonzero((viirs.M05.values < -99) | (viirs.M05.values > 2))
+ viirs['M05'].values[idx] = _bad_data
+ else:
+ viirs['M05'] = (('number_of_lines', 'number_of_pixels'),
+ np.full(viirs.M15.shape, _bad_data))
+ if 'M07' in viirs:
+ idx = np.nonzero((viirs.M07.values < -99) | (viirs.M07.values > 2))
+ viirs['M07'].values[idx] = _bad_data
+ else:
+ viirs['M07'] = (('number_of_lines', 'number_of_pixels'),
+ np.full(viirs.M15.shape, _bad_data))
+
+ idx = np.nonzero((viirs.M12.values < 0) | (viirs.M12.values > 1000))
+ viirs['M12'].values[idx] = _bad_data
+ idx = np.nonzero((viirs.M13.values < 0) | (viirs.M13.values > 1000))
+ viirs['M13'].values[idx] = _bad_data
+ idx = np.nonzero((viirs.M14.values < 0) | (viirs.M14.values > 1000))
+ viirs['M14'].values[idx] = _bad_data
+ idx = np.nonzero((viirs.M15.values < 0) | (viirs.M15.values > 1000))
+ viirs['M15'].values[idx] = _bad_data
+ idx = np.nonzero((viirs.M16.values < 0) | (viirs.M16.values > 1000))
+ viirs['M16'].values[idx] = _bad_data
+
+ # Compute channel differences and ratios that are used in the tests
+ viirs['M15-M13'] = (('number_of_lines', 'number_of_pixels'),
+ viirs.M15.values - viirs.M13.values)
+ viirs['M14-M15'] = (('number_of_lines', 'number_of_pixels'),
+ viirs.M14.values - viirs.M15.values)
+ viirs['M15-M16'] = (('number_of_lines', 'number_of_pixels'),
+ viirs.M15.values - viirs.M16.values)
+ viirs['M15-M12'] = (('number_of_lines', 'number_of_pixels'),
+ viirs.M15.values - viirs.M12.values)
+ viirs['M13-M16'] = (('number_of_lines', 'number_of_pixels'),
+ viirs.M13.values - viirs.M16.values)
+ viirs['M07-M05ratio'] = (('number_of_lines', 'number_of_pixels'),
+ viirs.M07.values / viirs.M05.values)
+ viirs['GEMI'] = (('number_of_lines', 'number_of_pixels'), gemi)
+
+ # temp value to force the code to work
+ viirs['M128'] = (('number_of_lines', 'number_of_pixels'), np.zeros(viirs.M15.shape))
+
+ else:
+ viirs = read_viirs('viirs', f'{img02}', f'{img03}')
+ viirs['M05'] = viirs.I01
+ viirs['M07'] = viirs.I02
+
+ idx = np.nonzero((viirs.M05.values < -99) | (viirs.M05.values > 2))
+ viirs['M05'].values[idx] = _bad_data
+ idx = np.nonzero((viirs.M07.values < -99) | (viirs.M07.values > 2))
+ viirs['M07'].values[idx] = _bad_data
+
+ idx = np.nonzero((viirs.I01.values < 0) | (viirs.I01.values > 1000))
+ viirs['I01'].values[idx] = _bad_data
+ idx = np.nonzero((viirs.I02.values < 0) | (viirs.I02.values > 1000))
+ viirs['I02'].values[idx] = _bad_data
+ idx = np.nonzero((viirs.I03.values < 0) | (viirs.I03.values > 1000))
+ viirs['I03'].values[idx] = _bad_data
+ idx = np.nonzero((viirs.I04.values < 0) | (viirs.I04.values > 1000))
+ viirs['I04'].values[idx] = _bad_data
+ idx = np.nonzero((viirs.I05.values < 0) | (viirs.I05.values > 1000))
+ viirs['I05'].values[idx] = _bad_data
+
+ viirs = read_ancillary_data(file_names, viirs, resolution=2)
+
+ viirs['I05-I04'] = (('number_of_lines_2', 'number_of_pixels_2'),
+ viirs.I05.values - viirs.I04.values)
+ viirs['I02-I01ratio'] = (('number_of_lines_2', 'number_of_pixels_2'),
+ viirs.I02.values / viirs.I01.values)
+
+ scene_flags = scn.find_scene(viirs, sunglint_angle)
+ scene = scn.scene_id(scene_flags)
+
+ scene_xr = xr.Dataset()
+ for s in scn._scene_list:
+ scene_xr[s] = (('number_of_lines', 'number_of_pixels'), scene[s])
+
+ scene['lat'] = viirs.latitude
+ scene['lon'] = viirs.longitude
+
+ data = xr.Dataset(viirs, coords=scene_xr)
+ data.drop_vars(['latitude', 'longitude'])
+
+ return data
+"""
diff --git a/mvcm/restoral.py b/mvcm/restoral.py
new file mode 100644
index 0000000..e89e3b5
--- /dev/null
+++ b/mvcm/restoral.py
@@ -0,0 +1,127 @@
+import numpy as np
+
+from numpy.lib.stride_tricks import sliding_window_view
+
+_bad_data = -999.0
+
+
+def spatial_var(rad, threshold):
+
+ test = sliding_window_view(np.pad(rad, [1, 1], mode='constant'), (3, 3)) - np.expand_dims(rad, (2, 3))
+ test = np.abs(test.reshape(rad.shape[0], rad.shape[1], 9))
+
+ var = np.ones(test.shape)
+ var[test < threshold] = 0
+
+ return var.sum(axis=2)
+
+
+def sunglint(viirs_data, threshold, bit, conf):
+
+ m09 = viirs_data.M02.values
+ m20 = viirs_data.M12.values
+ m31 = viirs_data.M15.values
+ m02 = viirs_data.M07.values
+ m01 = viirs_data.M05.values
+ irclr = np.ones(viirs_data.M02.shape)
+
+ idx = np.nonzero((m02 > 1.3) & (m01 <= 2.0))
+ m02[idx] = 1.3
+ idx = np.nonzero((m02 < -99) | (m02 > 1.3))
+ m02[idx] = _bad_data
+
+ irclr[bit < 3] = 0
+
+ var = spatial_var(m31, 0.4)
+ reg_var_mean = sliding_window_view(np.pad(m02, [1, 1], mode='constant'),
+ (3, 3)).reshape(m02.shape[0], m02.shape[1], 9).mean(axis=2)
+ reg_std = sliding_window_view(np.pad(m02, [1, 1], mode='constant'),
+ (3, 3)).reshape(m02.shape[0], m02.shape[1], 9).std(axis=2)
+ d37_11 = m20 - m31
+
+ idx = np.nonzero((var == 0) & (m09 != _bad_data) & (m20 != _bad_data) & (m31 != _bad_data) & (irclr == 1) &
+ (m02 != _bad_data) & (m09 < threshold['sngm09']) & (d37_11 >= threshold['sg_tbdfl']))
+ conf[idx] = 0.67
+
+ idx = np.nonzero((var == 0) & (m09 != _bad_data) & (m20 != _bad_data) & (m31 != _bad_data) & (irclr == 1) &
+ (m02 != _bad_data) & (m09 < threshold['sngm09']) & (d37_11 >= threshold['sg_tbdfl']) &
+ (reg_var_mean != _bad_data) & (reg_var_mean*reg_std < threshold['sngm02vm']))
+ conf[idx] = 0.96
+
+ return conf
+
+
+def spatial(viirs_data, threshold, scene, conf):
+
+ m02 = viirs_data.M07.values
+ m31 = viirs_data.M15.values
+
+ var_m31 = spatial_var(m31, 0.40)
+ var_m02 = spatial_var(m02, 0.0020)
+
+ idx = np.nonzero((conf > 0.95) & (scene['day'] == 1) & (var_m31 == 0) & (var_m02 == 0))
+ conf[idx] = 1
+
+ idx = np.nonzero((conf > 0.66) & (conf <= 0.95) & (scene['day'] == 1) & (var_m31 == 0) & (var_m02 == 0))
+ conf[idx] = 0.96
+
+ idx = np.nonzero((conf <= 0.66) & (scene['day'] == 1) & (var_m31 == 0))
+ conf[idx] = 0.67
+
+ return conf
+
+
+def land(viirs_data, threshold, scene, conf):
+ m04 = viirs_data.M04.values.ravel()
+ m05 = viirs_data.M08.values.ravel()
+ m20 = viirs_data.M12.values.ravel()
+ m22 = viirs_data.M13.values.ravel()
+ m31 = viirs_data.M15.values.ravel()
+ eco = viirs_data.eco.values.ravel()
+ desert = scene['desert'].ravel()
+ conf = conf.ravel()
+ tbadj = 0
+ ldsbt11bd = np.array(threshold['Land_Restoral']['ldsbt11bd'])
+ ldsbt11 = np.array(threshold['Land_Restoral']['ldsbt11bd'])
+
+ irclr = 1
+ hds11 = np.ones((eco.shape[0], 3)) * (ldsbt11 - tbadj)
+ hds11[eco == 8, :] = ldsbt11bd - tbadj
+
+ if irclr == 1:
+ conf[m31 > hds11[:, 2]] = 1
+ conf[(m31 > hds11[:, 1]) & (m31 <= hds11[:, 2])] = 0.96
+ conf[m31 <= hds11[:, 1]] = 0.5
+
+ m5_4_thr = np.full(eco.shape, threshold['Land_Restoral']['ldr5_4_thr'])
+ m5_4_thr[desert == 1] = threshold['Land_Restoral']['ldsr5_4_thr']
+
+ m5_4 = m05/m04
+ md1 = m20 - m22
+ md2 = m22 - m31
+
+ idx = np.nonzero((md1 < threshold['Land_Restoral']['ld20m22']) &
+ (md2 < threshold['Land_Restoral']['ld22m31']) &
+ (m5_4 > m5_4_thr) &
+ (conf <= 0.95))
+ conf[idx] = 0.96
+
+ conf = conf.reshape(viirs_data.M01.shape)
+ return conf
+
+
+def coast(viirs_data, threshold, scene, conf):
+
+ m01 = viirs_data.M05.values
+ m02 = viirs_data.M07.values
+ coast_ndvi = threshold['Coastal_NDVI_Thresholds']['coast_ndvi']
+
+ irclr = 1
+
+ if irclr == 1:
+ ndvi = (m02 - m01)/(m01 + m02)
+
+ idx = np.nonzero((ndvi <= coast_ndvi[0]) | (ndvi >= coast_ndvi[1]))
+ conf[idx] = 1
+
+ return conf
diff --git a/mvcm/scene.py b/mvcm/scene.py
new file mode 100644
index 0000000..5e03b43
--- /dev/null
+++ b/mvcm/scene.py
@@ -0,0 +1,446 @@
+import numpy as np
+try:
+ from ruamel import yaml as yml
+except ImportError:
+ import ruamel_yaml as yml
+
+# from glob import glob
+
+import read_data as rd
+import ancillary_data as anc
+
+# lsf: land sea flag
+_scene_list = ['Ocean_Day', 'Ocean_Night', 'Land_Day', 'Land_Night', 'Day_Snow', 'Night_Snow', 'Coast_Day',
+ 'Land_Day_Desert', 'Antarctic_Day', 'Polar_Day_Snow', 'Polar_Day_Desert', 'Polar_Day_Ocean',
+ 'Polar_Day_Desert_Coast', 'Polar_Day_Coast', 'Polar_Day_Land', 'Polar_Night_Snow',
+ 'Polar_Night_Land', 'Polar_Night_Ocean', 'Land_Day_Desert_Coast', 'Land_Day_Coast', 'Desert',
+ 'Australia']
+_flags = ['day', 'night', 'land', 'coast', 'sh_lake', 'sh_ocean', 'water', 'polar', 'sunglint',
+ 'greenland', 'high_elevation', 'antarctica', 'desert', 'visusd', 'vrused', 'map_snow', 'map_ice',
+ 'ndsi_snow', 'snow', 'ice', 'new_zealand', 'uniform', 'australia']
+
+# temp value, need to verify what the actual bad_data value is in the C code
+_bad_data = -999.0
+
+_dtr = np.pi/180.
+_rtd = 180./np.pi
+
+# I'm defining here the flags for difference scenes. Eventually I want to find a better way of doing this
+land = 1
+# coast = .2
+sh_lake = .3
+sh_ocean = .4
+water = 5
+polar = 60
+sunglint = 70
+day = 100
+night = 200
+
+# #################################################################### #
+# TEST CASE
+# data:
+#datapath = '/ships19/hercules/pveglio/mvcm_viirs_hires'
+#fname_mod02 = glob(f'{datapath}/VNP02MOD.A2022173.1448.001.*.uwssec*.nc')[0]
+#fname_mod03 = glob(f'{datapath}/VNP03MOD.A2022173.1448.001.*.uwssec.nc')[0]
+#fname_img02 = glob(f'{datapath}/VNP02IMG.A2022173.1448.001.*.uwssec*.nc')[0]
+#fname_img03 = glob(f'{datapath}/VNP03IMG.A2022173.1448.001.*.uwssec.nc')[0]
+
+# thresholds:
+#threshold_file = '/home/pveglio/mvcm_leo/thresholds/new_thresholds.mvcm.snpp.v1.0.0.yaml'
+
+# ancillary files:
+geos_atm_1 = 'GEOS.fpit.asm.inst3_2d_asm_Nx.GEOS5124.20220622_1200.V01.nc4'
+geos_atm_2 = 'GEOS.fpit.asm.inst3_2d_asm_Nx.GEOS5124.20220622_1500.V01.nc4'
+geos_land = 'GEOS.fpit.asm.tavg1_2d_lnd_Nx.GEOS5124.20220622_1430.V01.nc4'
+geos_ocean = 'GEOS.fpit.asm.tavg1_2d_ocn_Nx.GEOS5124.20220622_1430.V01.nc4'
+geos_constants = 'GEOS.fp.asm.const_2d_asm_Nx.00000000_0000.V01.nc4'
+ndvi_file = 'NDVI.FM.c004.v2.0.WS.00-04.177.hdf'
+sst_file = 'oisst.20220622'
+eco_file = 'goge1_2_img.v1'
+
+# #################################################################### #
+
+
+def test_scene():
+
+ ancillary_file_names = {'GEOS_atm_1': geos_atm_1,
+ 'GEOS_atm_2': geos_atm_2,
+ 'GEOS_land': geos_land,
+ 'GEOS_ocean': geos_ocean,
+ 'GEOS_constants': geos_constants,
+ 'NDVI': ndvi_file,
+ 'SST': sst_file,
+ 'ANC_DIR': f'{datapath}/ancillary'
+ }
+
+ viirs_data = rd.read_data('viirs', f'{fname_mod02}', f'{fname_mod03}')
+
+ viirs_data = rd.read_ancillary_data(ancillary_file_names, viirs_data)
+
+ with open(threshold_file) as f:
+ text = f.read()
+ thresholds = yml.safe_load(text)
+
+ sunglint_angle = thresholds['Sun_Glint']['bounds'][3]
+
+ scene = find_scene(viirs_data, sunglint_angle)
+
+ return scene
+
+
+def find_scene(data, sunglint_angle):
+ eco = np.array(data['ecosystem'].values, dtype=np.uint8)
+ snowfr = data['geos_snowfr'].values
+ icefr = data['geos_icefr'].values
+ lsf = np.array(data['land_water_mask'].values, dtype=np.uint8)
+ lat = data['latitude'].values
+ lon = data['longitude'].values
+ sza = data['solar_zenith'].values
+ vza = data['sensor_zenith'].values
+ raz = data['relative_azimuth'].values
+ b065 = data['M05'].values
+ b086 = data['M07'].values
+ elev = data['height'].values
+ ndvibk = data['ndvi'].values
+ sfctmp = data['geos_sfct'].values
+
+ dim1 = data.latitude.shape[0]
+ dim2 = data.latitude.shape[1]
+
+ day = np.zeros((dim1, dim2))
+ day[sza <= 85] = 1
+ cos_refang = np.sin(vza*_dtr) * np.sin(sza*_dtr) * np.cos(raz*_dtr) + np.cos(vza*_dtr) * np.cos(sza*_dtr)
+ refang = np.arccos(cos_refang) * _rtd
+
+ # tmp = np.ones((dim1, dim2))
+ # tmp[day == 1] = day
+ # tmp[day == 0] = night
+
+ scene_flag = {flg: np.zeros((dim1, dim2)) for flg in _flags}
+
+ scene_flag['day'][sza <= 85] = 1
+ scene_flag['visusd'][sza <= 85] = 1
+ scene_flag['night'][sza > 85] = 1
+
+ scene_flag['polar'][np.abs(lat) > 60] = 1
+ # ################# need to pass refang (once I figure out what it is) and sunglint_angle. The latter
+ # comes from the thresholds file. In the C code is snglnt_bounds[3]
+ idx = np.nonzero((scene_flag['day'] == 1) & (refang <= sunglint_angle))
+ scene_flag['sunglint'][idx] = 1
+
+ # Force consistency between lsf and ecosystem type for water
+ idx = np.nonzero((lsf == 0) | (lsf >= 5) & (lsf < 7))
+ eco[idx] = 14
+
+ # start by defining anythings as land
+ scene_flag['land'] = np.ones((dim1, dim2))
+ scene_flag['water'] = np.zeros((dim1, dim2))
+
+ # Fix-up for missing ecosystem data in eastern Greenland and north-eastern Siberia.
+ # Without this, these regions become completely "coast".
+ idx = np.nonzero((lsf != 255) & (lsf == 1) | (lsf == 4))
+ scene_flag['land'][idx] = 1
+
+ # idx = np.nonzero((lsf != 255) & (eco == 14))
+
+ idx = np.nonzero((lsf != 255) & (eco == 14) & (lsf == 1) | (lsf == 4) & (lat < 64.0))
+ scene_flag['coast'][idx] = 1
+
+ idx = np.nonzero((lsf != 255) & (eco == 14) & (lsf == 1) | (lsf == 4) &
+ (lat >= 67.5) & (lon < -40.0) & (lon > -168.6))
+ scene_flag['coast'][idx] = 1
+ idx = np.nonzero((lsf != 255) & (eco == 14) & (lsf == 1) | (lsf == 4) &
+ (lat >= 67.5) & (lon > -12.5))
+ scene_flag['coast'][idx] = 1
+
+ idx = np.nonzero((lsf != 255) & (eco == 14) & (lsf == 1) | (lsf == 4) &
+ (lat >= 64.0) & (lat < 67.5) & (lon < -40.0) & (lon > -168.5))
+ scene_flag['coast'][idx] = 1
+ idx = np.nonzero((lsf != 255) & (eco == 14) & (lsf == 1) | (lsf == 4) &
+ (lat >= 64.0) & (lat < 67.5) & (lon > -30.0))
+ scene_flag['coast'][idx] = 1
+
+ idx = np.nonzero(lsf == 2)
+ scene_flag['coast'][idx] = 1
+ scene_flag['land'][idx] = 1
+
+ idx = np.nonzero(lsf == 3)
+ scene_flag['land'][idx] = 1
+ scene_flag['sh_lake'][idx] = 1
+
+ idx = np.nonzero((lsf == 0) | (lsf >= 5) & (lsf <= 7))
+ scene_flag['water'][idx] = 1
+ scene_flag['land'][idx] = 0
+
+ scene_flag['sh_ocean'][lsf == 0] = 1
+
+ # Need shallow lakes to be processed as "coast" for day, but not night
+ idx = np.nonzero((lsf == 3) & (day == 1))
+ scene_flag['coast'][idx] = 1
+
+ # if land/sea flag is missing, then calculate visible ratio to determine if land or water.
+ idx = np.nonzero((lsf == 255) & (b065 != _bad_data) & (b086 != _bad_data) & (b086/b065 > 0.9))
+ scene_flag['land'][idx] = 1
+
+ idx = np.nonzero((lsf == 255) & (b065 != _bad_data) & (b086 != _bad_data) & (b086/b065 <= 0.9))
+ scene_flag['land'][idx] = 0
+ scene_flag['water'][idx] = 1
+
+ # Check surface elevation
+ # First, define "Greenland".
+ idx = np.nonzero((scene_flag['land'] == 1) &
+ (lat >= 60.0) & (lat < 67.0) & (lon >= -60.0) & (lon < -30.0))
+ scene_flag['greenland'][idx] = 1
+
+ idx = np.nonzero((scene_flag['land'] == 1) &
+ (lat >= 67.0) & (lat < 75.0) & (lon >= -60.0) & (lon < -10.0))
+ scene_flag['greenland'][idx] = 1
+
+ idx = np.nonzero((scene_flag['land'] == 1) &
+ (lat >= 75.0) & (lon >= -70.0) & (lon < -10.0))
+ scene_flag['greenland'][idx] = 1
+
+ scene_flag['high_elevation'][elev > 2000] = 1
+ idx = np.nonzero((elev > 200) & (scene_flag['greenland'] == 1) & (scene_flag['land'] == 1))
+ scene_flag['high_elevation'][idx] = 1
+
+ idx = np.nonzero((lat >= 75.0) & (lat <= 79.0) & (lon >= -73.0) & (lon <= -50.0) &
+ (scene_flag['land'] == 1))
+ scene_flag['high_elevation'][idx] = 1
+
+ scene_flag['antarctica'][lat < -60.0] = 1
+
+ ##################################
+ # somewhere here I need to add #
+ # the 11um elevation correction #
+ ##################################
+
+ # this is a temporary variable for the 11um elevation correction
+ elev_correction = elev/1000.0 * 5.0
+
+ ## Get surface temperature from NWP and SST fields
+ ## if it's land get it from GDAS/GEOS5
+ #sfctmp[scene_flag['land'] == 1] = sfct
+ ## otherwise use the ReynoldsSST
+ #sfctmp[scene_flag['land'] == 0] = reynSST
+
+ # Use background NDVI to define "desert"
+ idx = np.nonzero((scene_flag['land'] == 1) & (ndvibk < 0.3))
+ scene_flag['desert'][idx] = 1
+ idx = np.nonzero((scene_flag['land'] == 1) & (lat < -69.0))
+ scene_flag['desert'][idx] = 1
+
+ idx = np.nonzero((eco == 2) | (eco == 8) | (eco == 11) | (eco == 40) | (eco == 41) | (eco == 46) |
+ (eco == 51) | (eco == 52) | (eco == 59) | (eco == 71) | (eco == 50))
+ scene_flag['vrused'] = np.ones((dim1, dim2))
+ scene_flag['vrused'][idx] = 0
+
+ snow_fraction = data['geos_snowfr']
+ perm_ice_fraction = data['geos_landicefr']
+ ice_fraction = data['geos_icefr']
+
+ idx = tuple(np.nonzero((snow_fraction > 0.10) & (snow_fraction <= 1.0)))
+ scene_flag['map_snow'][idx] = 1
+
+ idx = tuple(np.nonzero((perm_ice_fraction > 0.10) & (perm_ice_fraction <= 1.0)))
+ scene_flag['map_snow'][idx] = 1
+
+ idx = tuple(np.nonzero((ice_fraction > 0.10) & (ice_fraction <= 1.0)))
+ scene_flag['map_ice'][idx] = 1
+
+ # need to define this function and write this block better
+ # if day == 1:
+ # # Run quick version of D. Hall's snow detection algorithm
+ # # Temporarily disabled because this function does not exist and I need to decide how to implement it
+ # # scene_flag['ndsi_snow'] = run_snow_mask()
+ # scene_flag['ndsi_snow'] = np.zeros((dim1, dim2))
+
+ idx = np.nonzero((day == 1) & (water == 1) & (lat >= -60.0) & (lat <= 25.0) &
+ (scene_flag['map_snow'] == 1) & (scene_flag['ndsi_snow'] == 1))
+ scene_flag['ice'][idx] = 1
+
+ idx = np.nonzero((day == 1) & (water == 1) & (lat < -60.0) &
+ (scene_flag['ndsi_snow'] == 1))
+ scene_flag['ice'][idx] = 1
+
+ idx = np.nonzero((day == 1) & (water == 1) & (lsf == 3) | (lsf == 5) &
+ (scene_flag['ndsi_snow'] == 1))
+ scene_flag['ice'][idx] = 1
+
+ idx = np.nonzero((day == 1) & (water == 1) &
+ (scene_flag['map_ice'] == 1) & (scene_flag['ndsi_snow'] == 1))
+ scene_flag['ice'][idx] = 1
+
+ # Define New Zealand region which receives snow but snow map does not show it.
+ idx = np.nonzero((day == 1) & (land == 1) &
+ (lat >= 48.0) & (lat <= -34.0) & (lon >= 165.0) & (lon <= 180.0))
+ scene_flag['new_zealand'][idx] = 1
+
+ idx = np.nonzero((day == 1) & (land == 1) & (lat >= -60.0) & (lat <= 25.0) &
+ (scene_flag['map_snow'] == 1) & (scene_flag['ndsi_snow'] == 1) |
+ (scene_flag['new_zealand'] == 1))
+ scene_flag['snow'][idx] = 1
+
+ idx = np.nonzero((day == 1) & (land == 1) & (lat < -60.0))
+ scene_flag['snow'][idx] = 1
+
+ idx = np.nonzero((day == 1) & (land == 1) & (scene_flag['ndsi_snow'] == 1))
+ scene_flag['snow'][idx] = 1
+
+ idx = np.nonzero((day == 0) & (scene_flag['map_snow'] == 1) &
+ (sfctmp > 280.0) & (elev < 500.0))
+ scene_flag['snow'][idx] = 0
+
+ idx = np.nonzero((day == 0) & (scene_flag['map_snow'] == 1) &
+ (sfctmp > 280.0) & (elev < 500.0))
+ scene_flag['ice'][idx] = 0
+
+ idx = np.nonzero((day == 0) & (lat > 86.0))
+ scene_flag['ice'][idx] = 1
+
+ idx = np.nonzero((lat <= -11) & (lat > -40) & (lon <= 155) & (lon > 110))
+ scene_flag['australia'][idx] = 1
+
+ # Check regional uniformity
+ # Check for data border pixels
+ scene_flag['uniform'] = anc.py_check_reg_uniformity(eco, eco, snowfr, icefr, lsf)['loc_uniform']
+ print(scene_flag['uniform'][:10, :10])
+
+ # TEMP VALUES FOR DEBUGGING
+ scene_flag['lat'] = lat
+ scene_flag['lon'] = lon
+
+ return scene_flag
+
+
+def scene_id(scene_flag):
+
+ dim1, dim2 = scene_flag['day'].shape[0], scene_flag['day'].shape[1]
+ scene = {scn: np.zeros((dim1, dim2)) for scn in _scene_list}
+
+ # Ocean Day
+ idx = np.nonzero((scene_flag['water'] == 1) & (scene_flag['day'] == 1) &
+ (scene_flag['ice'] == 0) & (scene_flag['snow'] == 0) &
+ (scene_flag['polar'] == 0) & (scene_flag['antarctica'] == 0) &
+ (scene_flag['coast'] == 0) & (scene_flag['desert'] == 0))
+ scene['Ocean_Day'][idx] = 1
+
+ # Ocean Night
+ idx = np.nonzero((scene_flag['water'] == 1) & (scene_flag['night'] == 1) &
+ (scene_flag['ice'] == 0) & (scene_flag['snow'] == 0) &
+ (scene_flag['polar'] == 0) & (scene_flag['antarctica'] == 0) &
+ (scene_flag['coast'] == 0) & (scene_flag['desert'] == 0))
+ scene['Ocean_Night'][idx] = 1
+
+ # Land Day
+ idx = np.nonzero((scene_flag['land'] == 1) & (scene_flag['day'] == 1) &
+ (scene_flag['ice'] == 0) & (scene_flag['snow'] == 0) &
+ (scene_flag['polar'] == 0) & (scene_flag['antarctica'] == 0) &
+ (scene_flag['coast'] == 0) & (scene_flag['desert'] == 0))
+ scene['Land_Day'][idx] = 1
+
+ # Land Night
+ idx = np.nonzero((scene_flag['land'] == 1) & (scene_flag['night'] == 1) &
+ (scene_flag['ice'] == 0) & (scene_flag['snow'] == 0) &
+ (scene_flag['polar'] == 0) & (scene_flag['antarctica'] == 0) &
+ (scene_flag['coast'] == 0))
+ scene['Land_Night'][idx] = 1
+
+ # Snow Day
+ idx = np.nonzero((scene_flag['day'] == 1) &
+ ((scene_flag['ice'] == 1) | (scene_flag['snow'] == 1)) &
+ (scene_flag['polar'] == 1) & (scene_flag['antarctica'] == 1))
+ scene['Day_Snow'][idx] = 1
+
+ # Snow Night
+ idx = np.nonzero((scene_flag['night'] == 1) &
+ ((scene_flag['ice'] == 1) | (scene_flag['snow'] == 1)) &
+ (scene_flag['polar'] == 1) & (scene_flag['antarctica'] == 1))
+ scene['Night_Snow'][idx] = 1
+
+ # Land Day Coast
+ idx = np.nonzero((scene_flag['land'] == 1) & (scene_flag['day'] == 1) &
+ (scene_flag['coast'] == 1) & (scene_flag['desert'] == 0) &
+ (scene_flag['polar'] == 0) & (scene_flag['antarctica'] == 0))
+ scene['Land_Day_Coast'][idx] = 1
+
+ # Land Day Desert
+ idx = np.nonzero((scene_flag['land'] == 1) & (scene_flag['day'] == 1) &
+ (scene_flag['desert'] == 1) & (scene_flag['coast'] == 0) &
+ (scene_flag['polar'] == 0) & (scene_flag['antarctica'] == 0))
+ scene['Land_Day_Desert'][idx] = 1
+
+ # Land Day Desert Coast
+ idx = np.nonzero((scene_flag['land'] == 1) & (scene_flag['day'] == 1) &
+ (scene_flag['desert'] == 1) & (scene_flag['coast'] == 1) &
+ (scene_flag['polar'] == 0) & (scene_flag['antarctica'] == 0))
+ scene['Land_Day_Desert_Coast'][idx] = 1
+
+ # Antarctic Day
+ idx = np.nonzero((scene_flag['polar'] == 1) & (scene_flag['day'] == 1) &
+ (scene_flag['antarctica'] == 1) & (scene_flag['land'] == 1))
+ scene['Antarctic_Day'][idx] = 1
+
+ # Polar Day Snow
+ idx = np.nonzero((scene_flag['polar'] == 1) & (scene_flag['day'] == 1) &
+ ((scene_flag['snow'] == 1) | (scene_flag['ice'] == 1)) &
+ (scene_flag['antarctica'] == 0))
+ scene['Polar_Day_Snow'][idx] = 1
+
+ # Polar Day Desert
+ idx = np.nonzero((scene_flag['polar'] == 1) & (scene_flag['day'] == 1) &
+ (scene_flag['land'] == 1) & (scene_flag['desert'] == 1) &
+ (scene_flag['coast'] == 0) & (scene_flag['antarctica'] == 0) &
+ (scene_flag['ice'] == 0) & (scene_flag['snow'] == 0))
+ scene['Polar_Day_Desert'][idx] = 1
+
+ # Polar Day Desert Coast
+ idx = np.nonzero((scene_flag['polar'] == 1) & (scene_flag['day'] == 1) &
+ (scene_flag['land'] == 1) & (scene_flag['desert'] == 1) &
+ (scene_flag['coast'] == 1) & (scene_flag['antarctica'] == 0) &
+ (scene_flag['ice'] == 0) & (scene_flag['snow'] == 0))
+ scene['Polar_Day_Desert_Coast'][idx] = 1
+
+ # Polar Day Coast
+ idx = np.nonzero((scene_flag['polar'] == 1) & (scene_flag['day'] == 1) &
+ (scene_flag['land'] == 1) & (scene_flag['coast'] == 1) &
+ (scene_flag['desert'] == 0) & (scene_flag['antarctica'] == 0) &
+ (scene_flag['ice'] == 0) & (scene_flag['snow'] == 0))
+ scene['Polar_Day_Coast'][idx] = 1
+
+ # Polar Day Land
+ idx = np.nonzero((scene_flag['polar'] == 1) & (scene_flag['day'] == 1) &
+ (scene_flag['land'] == 1) & (scene_flag['coast'] == 0) &
+ (scene_flag['desert'] == 0) & (scene_flag['antarctica'] == 0) &
+ (scene_flag['ice'] == 0) & (scene_flag['snow'] == 0))
+ scene['Polar_Day_Land'][idx] = 1
+
+ # Polar Day Ocean
+ idx = np.nonzero((scene_flag['polar'] == 1) & (scene_flag['day'] == 1) &
+ (scene_flag['water'] == 1) & (scene_flag['coast'] == 0) &
+ (scene_flag['desert'] == 0) & (scene_flag['antarctica'] == 0) &
+ (scene_flag['ice'] == 0) & (scene_flag['snow'] == 0))
+ scene['Polar_Day_Ocean'][idx] = 1
+
+ # Polar Night Snow
+ idx = np.nonzero((scene_flag['polar'] == 1) & (scene_flag['night'] == 1) &
+ ((scene_flag['snow'] == 1) | (scene_flag['ice'] == 1)))
+ scene['Polar_Night_Snow'][idx] = 1
+
+ # Polar Night Land
+ idx = np.nonzero((scene_flag['polar'] == 1) & (scene_flag['night'] == 1) &
+ (scene_flag['land'] == 1))
+ scene['Polar_Night_Land'][idx] = 1
+
+ # Polar Night Ocean
+ idx = np.nonzero((scene_flag['polar'] == 1) & (scene_flag['night'] == 1) &
+ (scene_flag['water'] == 1))
+ scene['Polar_Night_Ocean'][idx] = 1
+
+ idx = np.nonzero(scene_flag['desert'] == 1)
+ scene['Desert'][idx] = 1
+
+ idx = np.nonzero(scene_flag['australia'] == 1)
+ scene['Australia'][idx] = 1
+
+ return scene
diff --git a/mvcm/spectral_tests.py b/mvcm/spectral_tests.py
new file mode 100644
index 0000000..183efb5
--- /dev/null
+++ b/mvcm/spectral_tests.py
@@ -0,0 +1,826 @@
+import numpy as np
+import xarray as xr
+
+from numpy.lib.stride_tricks import sliding_window_view
+from typing import Dict
+
+import functools
+
+# import utils
+import conf
+import conf_xr
+import scene as scn
+import preprocess_thresholds as preproc
+import restoral
+
+import importlib
+
+_RTD = 180./np.pi
+_DTR = np.pi/180
+
+# this is used for testing, eventually we want to remove it
+importlib.reload(preproc)
+importlib.reload(conf)
+importlib.reload(restoral)
+
+
+class CloudTests(object):
+
+ def __init__(self,
+ data: xr.Dataset,
+ scene_name: str,
+ thresholds: Dict) -> None:
+ self.data = data
+ self.scene_name = scene_name
+ self.thresholds = thresholds
+ self.scene_idx = tuple(np.nonzero(data[scene_name] == 1))
+ if self.scene_idx[0].shape[0] == 0:
+ self.pixels_in_scene = False
+ else:
+ self.pixels_in_scene = True
+
+ def run_if_test_exists_for_scene(func):
+ @functools.wraps(func)
+ def wrapper(self, *args, test_name, **kwargs):
+ if test_name not in self.thresholds[self.scene_name]:
+ print('Not running test for this scene')
+ # returns cmin. This could be changed into a keyworded argument for readability
+ test_bit = np.zeros(args[-1].shape)
+ return args[-1], test_bit
+ return func(self, *args, test_name, **kwargs)
+ return wrapper
+
+ @run_if_test_exists_for_scene
+ def test_11um(self,
+ band: str,
+ cmin: np.ndarray,
+ test_name: str = '11um_Test') -> np.ndarray:
+
+ confidence = np.ones(self.data[band].shape)
+ qa_bit = np.zeros(self.data[band].shape)
+ test_bit = np.zeros(self.data[band].shape)
+ threshold = self.thresholds[self.scene_name][test_name]
+
+ if (threshold['perform'] is True and self.pixels_in_scene is True):
+ qa_bit[self.scene_idx] = 1
+ print(f'Testing "{self.scene_name}"\n')
+ rad = self.data[band].values[self.scene_idx]
+ idx = np.nonzero((self.data[band].values >= threshold['thr'][1]) &
+ (self.data[self.scene_name] == 1))
+ test_bit[idx] = 1
+ confidence[self.scene_idx] = conf.conf_test_new(rad, threshold['thr'])
+
+ cmin = np.fmin(cmin, confidence)
+
+ return cmin, test_bit
+
+ @run_if_test_exists_for_scene
+ def surface_temperature_test(self,
+ band: str,
+ viirs_data: xr.Dataset,
+ cmin: np.ndarray,
+ test_name: str = 'Surface_Temperature_Test') -> np.ndarray:
+
+ confidence = np.ones(self.data[band].shape)
+ qa_bit = np.zeros(self.data[band].shape)
+ test_bit = np.zeros(self.data[band].shape)
+ threshold = self.thresholds[self.scene_name][test_name]
+
+ if (threshold['perform'] is True and self.pixels_in_scene is True):
+ qa_bit[self.scene_idx] = 1
+ print(f'Testing "{self.scene_name}"\n')
+ rad = self.data[band].values[self.scene_idx]
+ sfcdif = viirs_data.geos_sfct.values[self.scene_idx] - rad
+ # need to write the test_bit here
+ thr = preproc.thresholds_surface_temperature(viirs_data, threshold, self.scene_idx)
+ confidence[self.scene_idx] = conf.conf_test_new(sfcdif, thr)
+
+ cmin = np.fmin(cmin, confidence)
+
+ return cmin, test_bit
+
+ @run_if_test_exists_for_scene
+ def sst_test(self,
+ band31: str,
+ band32: str,
+ cmin: np.ndarray,
+ test_name: str = 'SST_Test') -> np.ndarray:
+
+ confidence = np.ones(self.data[band31].shape)
+ qa_bit = np.zeros(self.data[band31].shape)
+ test_bit = np.zeros(self.data[band31].shape)
+ threshold = self.thresholds[self.scene_name][test_name]
+
+ if (threshold['perform'] is True and self.pixels_in_scene is True):
+ qa_bit[self.scene_idx] = 1
+ m31 = self.data[band31].values - 273.16
+ bt_diff = self.data[band31].values - self.data[band32].values
+ sst = self.data.sst.values - 273.16
+ cosvza = np.cos(self.data.sensor_zenith.values*_DTR)
+
+ c = threshold['coeffs']
+
+ modsst = 273.16 + c[0] + c[1]*m31 + c[2]*bt_diff*sst + c[3]*bt_diff*((1.0/cosvza) - 1.0)
+ sfcdif = self.data.sst.values - modsst
+
+ idx = np.nonzero((sfcdif < threshold['thr'][1]) &
+ (self.data[self.scene_name] == 1))
+ test_bit[idx] = 1
+ print(f'Testing "{self.scene_name}"\n')
+ confidence[self.scene_idx] = conf.conf_test_new(sfcdif[self.scene_idx], threshold['thr'])
+
+ cmin = np.fmin(cmin, confidence)
+
+ # return cmin, np.abs(1-test_bit)*qa_bit
+ return cmin, test_bit
+
+ @run_if_test_exists_for_scene
+ def bt_diff_86_11um(self,
+ band: str,
+ cmin: np.ndarray,
+ test_name: str = '8.6-11um_Test') -> np.ndarray:
+
+ confidence = np.ones(self.data[band].shape)
+ qa_bit = np.zeros(self.data[band].shape)
+ test_bit = np.zeros(self.data[band].shape)
+ threshold = self.thresholds[self.scene_name][test_name]
+
+ if (threshold['perform'] is True and self.pixels_in_scene is True):
+ qa_bit[self.scene_idx] = 1
+ print(f'Testing "{self.scene_name}"\n')
+ rad = self.data[band].values[self.scene_idx]
+ idx = np.nonzero((self.data[band].values < threshold['thr'][1]) &
+ (self.data[self.scene_name] == 1))
+ test_bit[idx] = 1
+ confidence[self.scene_idx] = conf.conf_test_new(rad, threshold['thr'])
+
+ cmin = np.fmin(cmin, confidence)
+
+ return cmin, test_bit # np.abs(1-test_bit)*qa_bit
+
+ @run_if_test_exists_for_scene
+ def test_11_12um_diff(self,
+ band: str,
+ cmin: np.ndarray,
+ test_name: str = '11-12um_Cirrus_Test') -> np.ndarray:
+
+ confidence = np.ones(self.data.M15.shape)
+ qa_bit = np.zeros(self.data[band].shape)
+ test_bit = np.zeros(self.data[band].shape)
+ threshold = self.thresholds[self.scene_name][test_name]
+
+ if (threshold['perform'] is True and self.pixels_in_scene is True):
+ qa_bit[self.scene_idx] = 1
+ thr = preproc.thresholds_11_12um(self.data, threshold, self.scene_name, self.scene_idx)
+ print(f'Testing "{self.scene_name}"\n')
+ rad = self.data[band].values[self.scene_idx]
+ idx = np.nonzero((rad <= thr[1, :]) &
+ (self.data[self.scene_name].values[self.scene_idx] == 1))
+ tmp_bit = test_bit[self.scene_idx]
+ tmp_bit[idx] = 1
+ test_bit[self.scene_idx] = tmp_bit
+ confidence[self.scene_idx] = conf.conf_test_new(rad, thr)
+
+ cmin = np.fmin(cmin, confidence)
+ test_bit = test_bit.reshape(self.data[band].shape)
+ # return cmin, np.abs(1-test_bit)*qa_bit
+ return cmin, test_bit
+
+ @run_if_test_exists_for_scene
+ def bt_difference_11_4um_test(self,
+ band: str,
+ cmin: np.ndarray,
+ test_name: str = '11-4um_BT_Difference_Test') -> np.ndarray:
+
+ confidence = np.ones(self.data.M15.shape)
+ qa_bit = np.zeros(self.data[band].shape)
+ test_bit = np.zeros(self.data[band].shape)
+ threshold = self.thresholds[self.scene_name][test_name]
+
+ if (threshold['perform'] is True and self.pixels_in_scene is True):
+ qa_bit[self.scene_idx] = 1
+ print(f'Testing "{self.scene_name}"\n')
+ thr = preproc.bt_diff_11_4um_thresholds(self.data, threshold, self.scene_idx)
+ rad = self.data[band].values[self.scene_idx]
+ test_bit[self.scene_idx] = 1 # THIS NEEDS TO BE CALCULATED PROPERLY
+ confidence[self.scene_idx] = conf.conf_test_dble(rad, thr)
+
+ cmin = np.fmin(cmin, confidence)
+
+ return cmin, test_bit
+
+ @run_if_test_exists_for_scene
+ def bt_difference_11_4um_test_land(self,
+ band1: str,
+ band2: str,
+ cmin: np.ndarray,
+ test_name: str = '11-4um_BT_Difference_Test') -> np.ndarray:
+
+ confidence = np.ones(self.data.M15.shape)
+ qa_bit = np.zeros(self.data[band1].shape)
+ test_bit = np.zeros(self.data[band1].shape)
+ threshold = self.thresholds[self.scene_name][test_name]
+
+ scene_flag = scn.find_scene(self.data, self.thresholds['Sun_Glint']['bounds'][3])
+
+ if (threshold['perform'] is True and self.pixels_in_scene is True):
+ qa_bit[self.scene_idx] = 1
+ print(f'Testing "{self.scene_name}"\n')
+ if self.scene_name == 'Land_Night':
+ australia = self.data.Australia.values
+ ndvi = self.data.ndvi.values
+ coast = scene_flag['coast']
+ sh_lake = scene_flag['sh_lake']
+ idx = np.where(((australia == 0) & (ndvi < threshold['ndvi']) & (coast == 0)) |
+ ((australia == 1) & (ndvi < threshold['ndvi_australia']) & (coast == 0)))
+ not_idx = np.ones(australia.shape, dtype=bool)
+ not_idx[idx] = False
+
+ rad11_12 = self.data[band1].values
+ rad11_4 = self.data[band2].values
+ thr = preproc.get_nl_thresholds(self.data, threshold, coast=False)
+ idx_lake = np.where((sh_lake == 1) | (rad11_12 < 0))
+ # test_bit[self.scene_idx] = 1 # THIS NEEDS TO BE CALCULATED PROPERLY
+ thr[:3, idx_lake] = thr[:3, idx_lake] + 2
+ temp_thr = np.squeeze(thr[:, idx[0]])
+ temp_rad = rad11_4[idx]
+ confidence[idx] = conf.conf_test_new(temp_rad, temp_thr)
+
+ thr = preproc.land_night_thresholds(self.data, threshold, coast=True)
+ temp_thr = np.squeeze(thr[:, not_idx])
+ temp_rad = rad11_4[not_idx]
+ thr[0:3, idx_lake[0], idx_lake[1]] = thr[0:3, idx_lake[0], idx_lake[1]] - 1
+ thr[3:6, idx_lake[0], idx_lake[1]] = thr[3:6, idx_lake[0], idx_lake[1]] + 1
+
+ confidence[not_idx] = conf.conf_test_dble(rad11_4[not_idx], thr[:, not_idx])
+
+ if self.scene_name in ['Polar_Night_Land', 'Polar_Day_Snow', 'Day_Snow']:
+ rad11_12 = self.data[band1].values[self.scene_idx]
+ rad11_4 = self.data[band2].values[self.scene_idx]
+ thr = preproc.polar_night_thresholds(self.data, threshold, self.scene_name,
+ test_name, self.scene_idx)
+ confidence[self.scene_idx] = conf.conf_test_new(rad11_4, thr)
+
+ if self.scene_name in ['Polar_Night_Snow']:
+ rad11_12 = self.data[band1].values
+ rad11_4 = self.data[band2].values
+ idx_top = np.nonzero(self.data.M15.values > threshold['bt_thr'])
+ thr_top = preproc.polar_night_thresholds(self.data, threshold['top'], self.scene_name,
+ test_name, idx_top)
+ idx_bottom = np.nonzero((self.data.M15.values > threshold['bt_thr']) &
+ (self.data.geos_tpw > threshold['tpw_thr']))
+ thr_bottom = preproc.polar_night_thresholds(self.data, threshold['bottom'], self.scene_name,
+ test_name, idx_top)
+
+ confidence[idx_top] = conf.conf_test_new(rad11_4[idx_top], thr_top)
+ confidence[idx_bottom] = conf.conf_test_new(rad11_4[idx_bottom], thr_bottom)
+
+ if self.scene_name in ['Night_Snow']:
+ rad11_12 = self.data[band1].values
+ rad11_4 = self.data[band2].values
+ idx_top = np.nonzero(self.data.M15.values > threshold['bt_thr'])
+ thr_top = threshold['thr']
+ idx_bottom = np.nonzero((self.data.M15.values > threshold['bt_thr']) &
+ (self.data.geos_tpw > threshold['tpw_thr']))
+ thr_bottom = preproc.polar_night_thresholds(self.data, threshold['bottom'], self.scene_name,
+ test_name, idx_top)
+
+ confidence[idx_top] = conf.conf_test_new(rad11_4[idx_top], thr_top)
+ confidence[idx_bottom] = conf.conf_test_new(rad11_4[idx_bottom], thr_bottom)
+
+ cmin = np.fmin(cmin, confidence)
+
+ return cmin, test_bit
+
+ # This test requires 7.3um band that is not available for VIIRS
+ @run_if_test_exists_for_scene
+ def midlevel_cloud_test():
+ pass
+
+ # This test requires 7.3um band that is not available for VIIRS
+ @run_if_test_exists_for_scene
+ def water_vapor_cloud_test():
+ pass
+
+ @run_if_test_exists_for_scene
+ def variability_11um_test(self,
+ band: str,
+ cmin: np.ndarray,
+ test_name: str = '11um_Variability_Test') -> np.ndarray:
+
+ confidence = np.ones(self.data.M15.shape)
+ qa_bit = np.zeros(self.data[band].shape)
+ test_bit = np.zeros(self.data[band].shape)
+ threshold = self.thresholds[self.scene_name][test_name]
+ var = np.zeros([self.data.M15.shape[0], self.data.M15.shape[1], 9])
+ radshape = self.data.M15.shape
+
+ if (threshold['perform'] is True and self.pixels_in_scene is True):
+ qa_bit[self.scene_idx] = 1
+ print(f'Testing "{self.scene_name}"\n')
+ rad = self.data[band].values
+
+ test = (sliding_window_view(np.pad(rad, [1, 1], mode='constant'), (3, 3)) -
+ np.expand_dims(rad, (2, 3)))
+ var[np.abs(test).reshape(radshape[0], radshape[1], 9) < threshold['variability']] = 1
+ var = var.sum(axis=2)
+
+ idx = np.where(var == 9)
+ test_bit[idx] = 1
+ thr = np.array(threshold['thr'])
+ confidence[idx] = conf.conf_test_new(rad[idx], thr)
+
+ cmin = np.fmin(cmin, confidence)
+
+ return cmin, test_bit
+
+ @run_if_test_exists_for_scene
+ def oceanic_stratus_11_4um_test(self,
+ band: str,
+ cmin: np.ndarray,
+ test_name: str = '11-4um_Oceanic_Stratus_Test') -> np.ndarray:
+
+ confidence = np.ones(self.data.M15.shape)
+ qa_bit = np.zeros(self.data[band].shape)
+ test_bit = np.zeros(self.data[band].shape)
+ threshold = self.thresholds[self.scene_name][test_name]
+
+ if (threshold['perform'] is True and self.pixels_in_scene is True):
+ qa_bit[self.scene_idx] = 1
+ print(f'Testing "{self.scene_name}"\n')
+ rad = self.data[band].values[self.scene_idx]
+ thr = threshold['thr']
+ if self.scene_name in ['Land_Day_Desert', 'Land_Day_Desert_Coast', 'Polar_Day_Desert',
+ 'Polar_Day_Desert_Coast']:
+ confidence[self.scene_idx] = conf.conf_test_dble(rad, thr)
+
+ # these scenes have not been implemented yet
+ # elif self.scene_name in ['Land_Night', 'Polar_Night_Land', 'Polar_Day_Snow', 'Polar_Night_Snow',
+ # 'Day_Snow', 'Night_Snow', 'Antarctic_Day']:
+ # pass
+ else:
+ scene_flag = scn.find_scene(self.data, self.thresholds['Sun_Glint']['bounds'][3])
+ idx = np.nonzero((self.data[band].values >= threshold['thr'][1]) &
+ (self.data[self.scene_name] == 1) &
+ (scene_flag['sunglint'] == 0))
+ test_bit[idx] = 1
+ idx = np.nonzero((self.data[self.scene_name] == 1) &
+ (scene_flag['sunglint'] == 0))
+ confidence[idx] = conf.conf_test_new(self.data[band].values[idx], thr)
+
+ cmin = np.fmin(cmin, confidence)
+
+ return cmin, test_bit
+
+ @run_if_test_exists_for_scene
+ def nir_reflectance_test(self,
+ band: str,
+ cmin: np.ndarray,
+ test_name: str = 'NIR_Reflectance_Test') -> np.ndarray:
+
+ confidence = np.ones(self.data.M01.shape)
+ qa_bit = np.zeros(self.data[band].shape)
+ test_bit = np.zeros(self.data[band].shape)
+ threshold = self.thresholds[self.scene_name][test_name]
+
+ if (threshold['perform'] is True and self.pixels_in_scene is True):
+ qa_bit[self.scene_idx] = 1
+ thr = preproc.thresholds_NIR(self.data, self.thresholds, self.scene_name,
+ test_name, self.scene_idx)
+ print(f'Testing "{self.scene_name}"\n')
+ rad = self.data[band].values[self.scene_idx]
+ idx = np.nonzero((rad <= thr[1, :]) &
+ (self.data[self.scene_name].values[self.scene_idx] == 1))
+ tmp_bit = test_bit[self.scene_idx]
+ tmp_bit[idx] = 1
+ test_bit[self.scene_idx] = tmp_bit
+ confidence[self.scene_idx] = conf.conf_test_new(rad, thr)
+
+ cmin = np.fmin(cmin, confidence)
+
+ return cmin, np.abs(1-test_bit)*qa_bit
+
+ @run_if_test_exists_for_scene
+ def vis_nir_ratio_test(self,
+ band: str,
+ cmin: np.ndarray,
+ test_name: str = 'Vis/NIR_Reflectance_Test') -> np.ndarray:
+
+ confidence = np.ones(self.data.M01.shape)
+ qa_bit = np.zeros(self.data[band].shape)
+ test_bit = np.zeros(self.data[band].shape)
+ threshold = self.thresholds[self.scene_name][test_name]
+
+ if (threshold['perform'] is True and self.pixels_in_scene is True):
+ # I'm not using the self.scene_idx here because it messes up with the indexing of the
+ # confidence array and I don't want to think of a solution at the moment. I will need
+ # to figure out the logic to make it work once I'm at the stage where I want to optimize
+ # the code
+ qa_bit[self.scene_idx] = 1
+ rad = self.data[band].values # [self.scene_idx]
+ solar_zenith = self.data.solar_zenith.values # [self.scene_idx]
+ sunglint = scn.find_scene(self.data, self.thresholds['Sun_Glint']['bounds'][3])['sunglint']
+
+ idx = np.nonzero((solar_zenith <= 85) & (sunglint == 1))
+
+ thr_no_sunglint = np.array([threshold['thr'][i] for i in range(8)])
+ thr_sunglint = np.array([self.thresholds['Sun_Glint']['snglnt'][i] for i in range(8)])
+ print(thr_no_sunglint)
+ print(thr_sunglint)
+ # tmp = self.thresholds['Sun_Glint']['snglnt']
+ # thr_sunglint = np.array([tmp[0], tmp[1], tmp[2], tmp[3], tmp[4], tmp[5], 1])
+ confidence[sunglint == 0] = conf.conf_test_dble(rad[sunglint == 0], thr_no_sunglint)
+ confidence[idx] = conf.conf_test_dble(rad[idx], thr_sunglint)
+
+ idx = np.nonzero(((rad < thr_no_sunglint[1]) | (rad > thr_no_sunglint[4])) &
+ (self.data[self.scene_name].values == 1) & (sunglint == 0))
+ test_bit[tuple(idx)] = 1
+
+ idx = np.nonzero(((rad < thr_sunglint[1]) | (rad > thr_sunglint[4])) &
+ ((self.data[self.scene_name].values == 1) &
+ (solar_zenith <= 85) & (sunglint == 1)))
+ test_bit[tuple(idx)] = 1
+
+ cmin = np.fmin(cmin, confidence)
+
+ return cmin, np.abs(1-test_bit)*qa_bit
+
+ @run_if_test_exists_for_scene
+ def test_16_21um_Reflectance(self,
+ band: str,
+ cmin: np.ndarray,
+ test_name: str = '1.6_2.1um_NIR_Reflectance_Test') -> np.ndarray:
+
+ confidence = np.ones(self.data.M01.shape)
+ qa_bit = np.zeros(self.data[band].shape)
+ test_bit = np.zeros(self.data[band].shape)
+ threshold = self.thresholds[self.scene_name][test_name]
+
+ if (threshold['perform'] is True and self.pixels_in_scene is True):
+ qa_bit[self.scene_idx] = 1
+ thr = preproc.thresholds_NIR(self.data, self.thresholds, self.scene_name,
+ test_name, self.scene_idx)
+ print(f'Testing "{self.scene_name}"\n')
+ rad = self.data[band].values[self.scene_idx]
+ idx = np.nonzero((rad <= thr[1, :]) &
+ (self.data[self.scene_name].values[self.scene_idx] == 1))
+ tmp_bit = test_bit[self.scene_idx]
+ tmp_bit[idx] = 1
+ test_bit[self.scene_idx] = tmp_bit
+ confidence[self.scene_idx] = conf.conf_test_new(rad, thr)
+
+ cmin = np.fmin(cmin, confidence)
+
+ return cmin, np.abs(1-test_bit)*qa_bit
+
+ @run_if_test_exists_for_scene
+ def visible_reflectance_test(self,
+ band: str,
+ cmin: np.ndarray,
+ test_name: str = 'Visible_Reflectance_Test') -> np.ndarray:
+
+ confidence = np.ones(self.data.M01.shape)
+ test_bit = np.zeros(self.data[band].shape)
+ threshold = self.thresholds[self.scene_name][test_name]
+
+ if (threshold['perform'] is True and self.pixels_in_scene is True):
+ print(f'Testing "{self.scene_name}"\n')
+ thr, rad = preproc.vis_refl_thresholds(self.data, self.thresholds, self.scene_name,
+ self.scene_idx)
+ confidence[self.scene_idx] = conf.conf_test_new(rad, thr)
+
+ cmin = np.fmin(cmin, confidence)
+
+ return cmin, test_bit
+
+ @run_if_test_exists_for_scene
+ def gemi_test(self,
+ band: str,
+ cmin: np.ndarray,
+ test_name: str = 'GEMI_Test') -> np.ndarray:
+
+ confidence = np.ones(self.data.M01.shape)
+ test_bit = np.zeros(self.data[band].shape)
+ threshold = self.thresholds[self.scene_name][test_name]
+
+ if (threshold['perform'] is True and self.pixels_in_scene is True):
+ thr = preproc.gemi_thresholds(self.data, threshold, self.scene_name,
+ self.scene_idx)
+ rad = self.data[band].values[self.scene_idx]
+ confidence[self.scene_idx] = conf.conf_test_new(rad, thr)
+
+ cmin = np.fmin(cmin, confidence)
+
+ return cmin, test_bit
+
+ @run_if_test_exists_for_scene
+ def test_1_38um_high_clouds(self,
+ band: str,
+ cmin: np.ndarray,
+ test_name: str = '1.38um_High_Cloud_Test') -> np.ndarray:
+
+ confidence = np.ones(self.data.M01.shape)
+ qa_bit = np.zeros(self.data[band].shape)
+ test_bit = np.zeros(self.data[band].shape)
+ threshold = self.thresholds[self.scene_name][test_name]
+
+ if (threshold['perform'] is True and self.pixels_in_scene is True):
+ qa_bit[self.scene_idx] = 1
+ if self.scene_name in ['Ocean_Day', 'Polar_Day_Ocean']:
+ thr = preproc.thresholds_1_38um_test(self.data, self.thresholds, self.scene_name,
+ self.scene_idx)
+ else:
+ return cmin, test_bit
+ thr = threshold['thr']
+
+ print(f'Testing "{self.scene_name}"\n')
+ rad = self.data[band].values[self.scene_idx]
+ idx = np.nonzero((rad <= thr[1, :]) &
+ (self.data[self.scene_name].values[self.scene_idx] == 1))
+ tmp_bit = test_bit[self.scene_idx]
+ tmp_bit[idx] = 1
+ test_bit[self.scene_idx] = tmp_bit
+ confidence[self.scene_idx] = conf.conf_test_new(rad, thr)
+
+ cmin = np.fmin(cmin, confidence)
+
+ return cmin, np.abs(1-test_bit)*qa_bit
+
+ @run_if_test_exists_for_scene
+ def thin_cirrus_4_12um_BTD_test(self,
+ band: str,
+ cmin: np.ndarray,
+ test_name: str = '4-12um_BTD_Thin_Cirrus_Test') -> np.ndarray:
+
+ confidence = np.ones(self.data.M01.shape)
+ test_bit = np.zeros(self.data[band].shape)
+ threshold = self.thresholds[self.scene_name][test_name]
+
+ if (threshold['perform'] is True and self.pixels_in_scene is True):
+ thr = preproc.polar_night_thresholds(self.data, self.thresholds, self.scene_name,
+ test_name, self.scene_idx)
+ rad = self.data[band].values[self.scene_idx]
+ confidence[self.scene_idx] = conf.conf_test_new(rad, thr)
+
+ cmin = np.fmin(cmin, confidence)
+
+ return cmin, test_bit
+
+ def single_threshold_test(self, test_name, band, cmin):
+
+ if band == 'bad_data':
+ return cmin
+ print(f'Running test "{test_name}" for "{self.scene_name}"')
+
+ # preproc_thresholds()
+ if 'thr' in self.thresholds[self.scene_name][test_name]:
+ thr = np.array(self.thresholds[self.scene_name][test_name]['thr'])
+ else:
+ thr = np.array(self.thresholds[self.scene_name][test_name])
+ thr_xr = xr.Dataset()
+
+ if test_name == '11-12um_Cirrus_Test':
+ thr_xr['threshold'] = pt.preproc(self.data, self.thresholds[self.scene_name], self.scene_name)
+ thr = np.ones((5,)) # This is only temporary to force the logic of the code
+ # I need to find a better solution at some point
+ elif test_name == 'SST_Test':
+ thr_xr['threshold'] = (('number_of_lines', 'number_of_pixels', 'z'),
+ np.ones((self.data[band].shape[0], self.data[band].shape[1], 5))*thr)
+ elif test_name == '7.3-11um_BTD_Mid_Level_Cloud_Test':
+ thr_xr['threshold'] = pt.get_pn_thresholds(self.data, self.thresholds, self.scene_name,
+ '7.3-11um_BTD_Mid_Level_Cloud_Test')
+ thr = np.ones((5,))
+ elif test_name == 'Surface_Temperature_Test':
+ thr_xr['threshold'] = pt.preproc_surf_temp(self.data, self.thresholds[self.scene_name])
+ thr = np.ones((5,))
+ elif (test_name == '11-4um_Oceanic_Stratus_Test' and
+ self.scene_name in ['Land_Day_Desert', 'Land_Day_Desert_Coast', 'Polar_Day_Desert',
+ 'Polar_Day_Desert_Coast']):
+ thr = np.array([self.thresholds[self.scene_name][test_name][i] for i in range(8)])
+ elif test_name == 'NIR_Reflectance_Test':
+ corr_thr = pt.preproc_nir(self.data, self.thresholds, self.scene_name)
+ thr_xr['threshold'] = (('number_of_lines', 'number_of_pixels', 'z'), corr_thr)
+ elif test_name == 'Visible_Reflectance_Test':
+ thr_xr['threshold'], self.data['M128'] = pt.vis_refl_thresholds(self.data,
+ self.thresholds,
+ self.scene_name)
+ elif test_name == '1.6_2.1um_NIR_Reflectance_Test':
+ corr_thr = pt.nir_refl(self.data, self.thresholds, self.scene_name)
+ thr_xr['threshold'] = (('number_of_lines', 'number_of_pixels', 'z'), corr_thr)
+ thr = np.ones((5,))
+ elif test_name == '4-12um_BTD_Thin_Cirrus_Test':
+ thr_xr['threshold'] = pt.get_pn_thresholds(self.data, self.thresholds, self.scene_name,
+ '4-12um_BTD_Thin_Cirrus_Test')
+ thr = np.ones((5,))
+ elif test_name == 'GEMI_Test':
+ thr_xr['threshold'] = pt.GEMI_test(self.data, self.thresholds, self.scene_name)
+ thr = np.ones((5,))
+ elif (test_name == '1.38um_High_Cloud_Test' and self.scene_name in ['Ocean_Day', 'Polar_Ocean_Day']):
+ thr_xr['threshold'] = pt.test_1_38um_preproc(self.data, self.thresholds, self.scene_name)
+ thr = np.ones((5,))
+ else:
+ thr_xr['threshold'] = (('number_of_lines', 'number_of_pixels', 'z'),
+ np.ones((self.data[band].shape[0], self.data[band].shape[1], 5))*thr)
+
+ data = xr.Dataset(self.data, coords=thr_xr)
+
+ if test_name == 'SST_Test':
+ data['sfcdif'] = (('number_of_lines', 'number_of_pixels'),
+ pt.preproc_sst(data, self.thresholds[self.scene_name][test_name]).values)
+ band = 'sfcdif'
+ if test_name == '11um_Variability_Test':
+ var = pt.var_11um(self.data, self.thresholds)
+ data['11um_var'] = data.M15
+ data['11um_var'].values[var != 9] = np.nan
+
+ if thr[4] == 1:
+ print('test running...')
+ confidence = conf_xr.conf_test(data, band)
+
+ cmin = np.fmin(cmin, confidence)
+
+ return cmin
+
+ def double_threshold_test(self, test_name, band, cmin):
+ data = self.data
+ if test_name == '11-4um_BT_Difference_Test':
+ thr = pt.bt11_4um_preproc(self.data, self.thresholds, self.scene_name)
+ print('test running...')
+ confidence = conf.conf_test_dble(data['M15-M13'].values, thr)
+ confidence = confidence.reshape(data.M01.shape)
+
+ if test_name == 'Vis/NIR_Ratio_Test':
+ print('test running...')
+ thr_no_sunglint = np.array([self.thresholds[self.scene_name][test_name][i] for i in range(8)])
+ thr_sunglint = np.array([self.thresholds['Sun_Glint']['snglnt'][i] for i in range(8)])
+ vrat = data.M07.values/data.M05.values
+ _dtr = np.pi/180.0
+ sza = data.sensor_zenith.values
+ raz = data.relative_azimuth.values
+ vza = data.sensor_zenith.values
+ cos_refang = np.sin(vza*_dtr) * np.sin(sza*_dtr) * np.cos(raz*_dtr) + \
+ np.cos(vza*_dtr) * np.cos(sza*_dtr)
+ refang = np.arccos(cos_refang) * 180./np.pi
+ idx = np.nonzero((data.solar_zenith <= 85) & (refang <= data.sunglint_angle))
+
+ confidence = conf.conf_test_dble(vrat, thr_no_sunglint)
+ confidence = confidence.reshape(data.M01.shape)
+ confidence[idx] = conf.conf_test_dble(vrat[idx], thr_sunglint)
+
+ confidence = confidence.reshape(data.M01.shape)
+
+ if (test_name == '11-4um_Oceanic_Stratus_Test' and
+ self.scene_name in ['Land_Day_Desert', 'Land_Day_Desert_Coast', 'Polar_Day_Desert',
+ 'Polar_Day_Desert_Coast']):
+ thr = np.array([self.thresholds[self.scene_name][test_name][i] for i in range(8)])
+
+ print('test running...')
+ confidence = conf.conf_test_dble(data['M15-M16'].values, thr)
+ confidence = confidence.reshape(data.M01.shape)
+
+ cmin = np.fmin(cmin, confidence)
+
+ return cmin
+
+
+class ComputeTests(CloudTests):
+
+ def __init__(self,
+ data: xr.Dataset,
+ scene_name: str,
+ thresholds: Dict) -> None:
+ super().__init__(data, scene_name, thresholds)
+
+ def run_tests(self):
+ cmin_G1 = np.ones(self.data.M01.shape)
+ cmin_G2 = np.ones(self.data.M01.shape)
+ cmin_G3 = np.ones(self.data.M01.shape)
+ cmin_G4 = np.ones(self.data.M01.shape)
+ cmin_G5 = np.ones(self.data.M01.shape)
+
+ # Group 1
+ cmin_G1 = self.test_11um('M15', cmin_G1, test_name='11um_Test')
+ cmin_G1 = self.sst_test('M15', 'M16', cmin_G1, test_name='SST_Test')
+
+ # Group 2
+ cmin_G2 = self.bt_diff_86_11um('M14-M15', cmin_G2, test_name='8.6-11um_Test')
+ cmin_G2 = self.test_11_12um_diff('M15-M16', cmin_G2, test_name='11-12um_Cirrus_Test')
+ cmin_G2 = self.oceanic_stratus_11_4um_test('M15-M13', cmin_G2,
+ test_name='11-4um_Oceanic_Stratus_Test')
+
+ # Group 3
+ cmin_G3 = self.nir_reflectance_test('M07', cmin_G3, test_name='NIR_Reflectance_Test')
+ cmin_G3 = self.vis_nir_ratio_test('M07-M05ratio', cmin_G3, test_name='Vis/NIR_Ratio_Test')
+ cmin_G3 = self.nir_reflectance_test('M10', cmin_G3, test_name='1.6_2.1um_NIR_Reflectance_Test')
+ cmin_G3 = self.visible_reflectance_test('M128', cmin_G3, test_name='Visible_Reflectance_Test')
+ cmin_G3 = self.gemi_test('GEMI', cmin_G3, test_name='GEMI_Test')
+
+ # Group 4
+ cmin_G4 = self.test_1_38um_high_clouds('M09', cmin_G4, test_name='1.38um_High_Cloud_Test')
+
+ # Group 5
+ cmin_G5 = self.thin_cirrus_4_12um_BTD_test('M13-M16', cmin_G5,
+ test_name='4-12um_BTD_Thin_Cirrus_Test')
+
+ cmin = cmin_G1 * cmin_G2 * cmin_G3 * cmin_G4 * cmin_G5
+
+ return cmin
+
+ def clear_sky_restoral(self,
+ cmin: np.ndarray) -> np.ndarray:
+
+ total_bit = np.full(self.data.M01.shape, 3)
+ sunglint_angle = self.thresholds['Sun_Glint']['bounds'][3]
+ scene_flags = scn.find_scene(self.data, sunglint_angle)
+
+ cmin_tmp = cmin + 0
+ idx = np.nonzero((scene_flags['water'] == 1) & (scene_flags['ice'] == 0) &
+ (scene_flags['uniform'] == 1) & (cmin <= 0.99) & (cmin >= 0.05))
+ cmin[idx] = restoral.spatial(self.data, self.thresholds['Sun_Glint'], scene_flags, cmin_tmp)[idx]
+
+ cmin_tmp = cmin + 0
+ idx = np.nonzero((scene_flags['water'] == 1) & (scene_flags['sunglint'] == 1) &
+ (scene_flags['uniform'] == 1) & (cmin <= 0.95))
+ cmin[idx] = restoral.sunglint(self.data, self.thresholds['Sun_Glint'], total_bit, cmin_tmp)[idx]
+
+ """
+ idx = np.nonzero((scene_flags['day'] == 1) & (scene_flags['land'] == 1) &
+ (scene_flags['snow'] == 0) & (scene_flags['ice'] == 0) &
+ (cmin <= 0.95))
+ cmin[idx] = restoral.land(self.data, self.thresholds, scene_flags, cmin)[idx]
+
+ idx = np.nonzero((scene_flags['day'] == 1) & (scene_flags['land'] == 1) &
+ (scene_flags['coast'] == 1) & (scene_flags['snow'] == 0) &
+ (scene_flags['ice'] == 0))
+ cmin[idx] = restoral.coast(self.data, self.thresholds, scene_flags, cmin)[idx]
+ """
+
+ return cmin
+
+
+def preproc_thresholds(thresholds, data):
+ thr = np.array(thresholds)
+ thr_xr = xr.Dataset()
+ thr_xr['tresholds'] = (('number_of_lines', 'number_of_pixels', 'z'),
+ np.ones((data['M01'].shape[0], data['M01'].shape[1], 5))*thr)
+
+ nl_sfct1 = thresholds['Land_Night']['Surface_Temperature_Test'][0]
+# nl_sfct2 = thresholds['Land_Night']['Surface_Temperature_Test'][1]
+# nlsfct_pfm = thresholds['Land_Night']['Surface_Temperature_Test'][2]
+ nl_df1 = thresholds['Land_Night']['Surface_Temperature_Test_difference'][0:2]
+ nl_df2 = thresholds['Land_Night']['Surface_Temperature_Test_difference'][2:]
+
+# df1 = data.M15 - data.M16
+# df2 = data.M15 - data.M13
+ thr_xr = thr_xr.where(data.desert != 1, nl_sfct1)
+ thr_xr = thr_xr.where((data['M15-M16'] > nl_df1[0]) |
+ ((data['M15-M16'] < nl_df1[0]) &
+ ((data['M15-M13'] <= nl_df2[0]) | (data['M15-M13'] >= nl_df2[1]))),
+ nl_sfct1[0])
+
+ data = xr.Dataset(data, coords=thr_xr)
+
+ return data
+
+
+def single_threshold_test(test, rad, threshold):
+
+ radshape = rad.shape
+ rad = rad.reshape(np.prod(radshape))
+
+ thr = np.array(threshold[test])
+ confidence = np.zeros(rad.shape)
+
+ if thr[4] == 1:
+ print(f"{test} test running")
+ # the C code has the line below that I don't quite understand the purpose of.
+ # It seems to be setting the bit to 0 if the BT value is greater than the midpoint
+ #
+ # if (m31 >= dobt11[1]) (void) set_bit(13, pxout.testbits);
+
+ # confidence = utils.conf_test(rad, thr)
+ confidence = conf.conf_test(rad, thr)
+
+ return confidence.reshape(radshape)
+
+
+def test():
+ rad = np.random.randint(50, size=[4, 8])
+ # coeffs = [5, 42, 20, 28, 15, 35, 1]
+ # coeffs = [20, 28, 5, 42, 15, 35, 1]
+ coeffs = [35, 15, 20, 1, 1]
+ # confidence = conf_test_dble(rad, coeffs)
+ confidence = test_11um(rad, coeffs)
+ print(rad)
+ print('\n')
+ print(confidence)
+
+
+if __name__ == "__main__":
+ test()
+
+
+
+
+
+
diff --git a/mvcm/utils.py b/mvcm/utils.py
new file mode 100644
index 0000000..6b45056
--- /dev/null
+++ b/mvcm/utils.py
@@ -0,0 +1,329 @@
+import numpy as np
+
+_test_rad = np.random.randint(25, size=[6, 8])
+# _test_thr = [15, 10, 5, 1, 1]
+_test_thr = [5, 10, 15, 1, 1]
+
+
+# this function creates a map of sunglint areas, based on the different angles set in the
+# threshold file. The goal is to create an array of indices that I can use to quickly assign
+# different coefficients depending on the angle interval. This will be mostly used in the
+# function get_sunglint_thresholds().
+# All of this is because we want to be able to process the whole array, instead of iterating
+# over all pixels one by one.
+def sunglint_scene(refang, sunglint_thr):
+ sunglint_flag = np.zeros(refang.shape)
+ sunglint_flag[(refang > sunglint_thr['bounds'][2]) & (refang <= sunglint_thr['bounds'][3])] = 1
+ sunglint_flag[(refang > sunglint_thr['bounds'][1]) & (refang <= sunglint_thr['bounds'][2])] = 2
+ sunglint_flag[refang <= sunglint_thr['bounds'][1]] = 3
+ return sunglint_flag
+
+
+def get_sunglint_thresholds(refang, thresholds, band_n, sunglint_flag, thr):
+
+ band = f'band{band_n}'
+ sunglint_thr = np.zeros((4, len(refang)))
+# if refang > thresholds['bounds'][3]:
+# sunglint = sunglint
+# # dosgref[2] = hicnf
+# # dosgref[0] = locnf
+# # dosgref[1] = mdcnf
+# # sunglint[3] = doref2[3]
+
+ # if refang <= thresholds['bounds'][1]:
+ if sunglint_flag == 3:
+ sunglint_thr = np.full((len(refang), 4), thresholds[f'{band}_0deg']).T
+ # sunglint_thr[:, :] = thresholds[f'{band}_0deg']
+
+ else:
+
+ # if (refang > thresholds['bounds'][1] and refang <= thresholds['bounds'][2]):
+ if sunglint_flag == 2:
+ lo_ang = thresholds['bounds'][1]
+ hi_ang = thresholds['bounds'][2]
+ lo_ang_val = thresholds[f'{band}_10deg'][0]
+ hi_ang_val = thresholds[f'{band}_10deg'][1]
+ power = thresholds[f'{band}_10deg'][3]
+ conf_range = thresholds[f'{band}_10deg'][2]
+
+ # elif (refang > thresholds['bounds'][2] and refang <= thresholds['bounds'][3]):
+ elif sunglint_flag == 1:
+ lo_ang = thresholds['bounds'][2]
+ hi_ang = thresholds['bounds'][3]
+ lo_ang_val = thresholds[f'{band}_20deg'][0]
+ hi_ang_val = thr[1]
+ power = thresholds[f'{band}_20deg'][3]
+ conf_range = thresholds[f'{band}_20deg'][2]
+ else:
+ raise ValueError("Wrong sunglint flag")
+
+ a = (refang - lo_ang) / (hi_ang - lo_ang)
+ midpt = lo_ang_val + a*(hi_ang_val - lo_ang_val)
+ sunglint_thr[1, :] = midpt
+ sunglint_thr[2, :] = midpt - conf_range
+ sunglint_thr[0, :] = midpt + conf_range
+ sunglint_thr[3, :] = power
+
+ return sunglint_thr
+
+
+def conf_test(rad=_test_rad, thr=_test_thr):
+ '''
+ Assuming a linear function between min and max confidence level, the plot below shows
+ how the confidence (y axis) is computed as function of radiance (x axis).
+ This case illustrates alpha < gamma, obviously in case alpha > gamma, the plot would be
+ flipped.
+ gamma
+ c 1 ________
+ o | /
+ n | /
+ f | /
+ i | beta /
+ d 1/2 |....../
+ e | /
+ n | /
+ c | /
+ e 0________/
+ | alpha
+ --------- radiance ---------->
+ '''
+
+ coeff = np.power(2, (thr[3] - 1))
+ hicut = thr[0]
+ beta = thr[1]
+ locut = thr[2]
+ power = thr[3]
+ confidence = np.zeros(rad.shape)
+
+ if hicut > locut:
+ gamma = thr[0]
+ alpha = thr[2]
+ flipped = False
+ else:
+ gamma = thr[2]
+ alpha = thr[0]
+ flipped = True
+
+ # Rad between alpha and beta
+ range_ = 2. * (beta - alpha)
+ s1 = (rad[rad <= beta] - alpha) / range_
+ if flipped is False:
+ confidence[rad <= beta] = coeff * np.power(s1, power)
+ if flipped is True:
+ confidence[rad <= beta] = 1. - (coeff * np.power(s1, power))
+
+ # Rad between beta and gamma
+ range_ = 2. * (beta - gamma)
+ s1 = (rad[rad > beta] - gamma) / range_
+ if flipped is False:
+ confidence[rad > beta] = 1. - (coeff * np.power(s1, power))
+ if flipped is True:
+ confidence[rad > beta] = coeff * np.power(s1, power)
+
+ # Rad outside alpha-gamma interval
+ if flipped is False:
+ confidence[rad > gamma] = 1
+ confidence[rad < alpha] = 0
+ if flipped is True:
+ confidence[rad > gamma] = 0
+ confidence[rad < alpha] = 1
+
+ confidence[confidence > 1] = 1
+ confidence[confidence < 0] = 0
+
+ return confidence
+
+
+def conf_test_dble(rad, coeffs):
+ # '''
+ # gamma1 gamma2
+ # c 1_______ ________
+ # o | \ /
+ # n | \ /
+ # f | \ /
+ # i | \ beta1 beta2 /
+ # d 1/2 \....| |...../
+ # e | \ /
+ # n | \ /
+ # c | \ /
+ # e 0 \_____________/
+ # | alpha1 alpha2
+ # --------------------- radiance ------------------------->
+ # '''
+
+ hicut = [coeffs[0], coeffs[1]]
+ locut = [coeffs[2], coeffs[3]]
+ midpt = [coeffs[4], coeffs[5]]
+ power = coeffs[6]
+
+ gamma1 = hicut[0]
+ gamma2 = hicut[1]
+ alpha1 = locut[0]
+ alpha2 = locut[1]
+ beta1 = midpt[0]
+ beta2 = midpt[1]
+
+ coeff = np.power(2, (power - 1))
+ radshape = rad.shape
+ rad = rad.reshape((rad.shape[0]*rad.shape[1]))
+ c = np.zeros(rad.shape)
+
+ # Find if interval between inner cutoffs passes or fails test
+ if (alpha1 - gamma1 > 0):
+
+ # Value is within range of lower set of limits
+ range_ = 2 * (beta1 - alpha1)
+ s1 = (rad[(rad <= alpha1) & (rad >= beta1)] - alpha1) / range_
+ c[(rad <= alpha1) & (rad >= beta1)] = coeff * np.power(s1, power)
+
+ range_ = 2 * (beta1 - gamma1)
+ s1 = (rad[(rad <= alpha1) & (rad < beta1)] - gamma1) / range_
+ c[(rad <= alpha1) & (rad < beta1)] = coeff * np.power(s1, power)
+
+ # Value is within range of upper set of limits
+ range_ = 2 * (beta2 - alpha2)
+ s1 = (rad[(rad > alpha1) & (rad <= beta2)] - alpha2) / range_
+ c[(rad > alpha1) & (rad <= beta2)] = coeff * np.power(s1, power)
+
+ range_ = 2 * (beta2 - gamma2)
+ s1 = (rad[(rad > alpha1) & (rad > beta2)] - gamma2) / range_
+ c[(rad > alpha1) & (rad > beta2)] = coeff * np.power(s1, power)
+
+ # Inner region fails test
+ # Check for value beyond function range
+ c[(rad > alpha1) & (rad < alpha2)] = 0
+ c[(rad < gamma1) | (rad > gamma2)] = 1
+
+ else:
+
+ # Value is withing range of lower set of limits
+ range_ = 2 * (beta1 - alpha1)
+ s1 = (rad[(rad <= gamma1) & (rad <= beta1)] - alpha1) / range_
+ c[(rad <= gamma1) & (rad <= beta1)] = coeff * np.power(s1, power)
+
+ range_ = 2 * (beta1 - gamma1)
+ s1 = (rad[(rad <= gamma1) & (rad > beta1)] - gamma1) / range_
+ c[(rad <= gamma1) & (rad > beta1)] = coeff * np.power(s1, power)
+
+ # Value is within range of upper set of limits
+ range_ = 2 * (beta2 - alpha2)
+ s1 = (rad[(rad > gamma1) & (rad >= beta2)] - alpha2) / range_
+ c[(rad > gamma1) & (rad >= beta2)] = coeff * np.power(s1, power)
+
+ range_ = 2 * (beta2 - gamma2)
+ s1 = (rad[(rad > gamma1) & (rad < beta2)] - gamma2) / range_
+ c[(rad > gamma1) & (rad < beta2)] = coeff * np.power(s1, power)
+
+ # Inner region passes test
+ # Check for value beyond function range
+ c[(rad > gamma1) & (rad < gamma2)] = 1
+ c[(rad < alpha1) | (rad > alpha2)] = 0
+
+ c[c > 1] = 1
+ c[c < 0] = 0
+
+ confidence = c.reshape(radshape)
+
+ return confidence
+
+
+def get_b1_threshold(thresholds, ndvi_thresholds):
+ # this needs to be read from the thresholds file
+ des_ndvi = 0.25
+ pxin_ndvibk = 0 # placeholder until I figure out how to pass this value
+ pxin_sctang = 0 # placeholder until I figure out how to pass this value
+ delta_ndvi_bin = 0.1 # defined as is in the original C code
+ thr = np.zeros((4, len(pxin_ndvibk)))
+ thr_adj = np.zeros((4, len(pxin_ndvibk)))
+
+ coeffs = np.full((10, 3, 4, len(pxin_ndvibk)))
+
+ indvi = np.zeros((len(pxin_ndvibk), ))
+ x = np.zeros((len(pxin_ndvibk), ))
+ y1 = np.zeros((len(pxin_ndvibk), ))
+ y2 = np.zeros((len(pxin_ndvibk), ))
+ interp = np.zeros((len(pxin_ndvibk), ))
+
+ # Check for special cases and fill values ('ndvibk'=32.767)
+ # idx = np.nonzero((pxin_ndvibk < ndvi_thresholds['fill_ndvi'][0]) &
+ # (pxin_ndvibk > ndvi_thresholds['fill_ndvi'][1]) &
+ # (pxin_ndvibk < ndvi_thresholds['ndvi_bnd1']))
+ idx = np.nonzero(pxin_ndvibk < ndvi_thresholds['ndvi_bnd1'])
+ # all gets set to zero, so no need to do anything for this idx
+
+ # idx = np.nonzero((pxin_ndvibk < ndvi_thresholds['fill_ndvi'][0]) &
+ # (pxin_ndvibk > ndvi_thresholds['fill_ndvi'][1]) &
+ # (pxin_ndvibk >= ndvi_thresholds['ndvi_bnd2']))
+ idx = np.nonzero(pxin_ndvibk >= ndvi_thresholds['ndvi_bnd2'])
+ indvi[idx] = 9
+
+ # else
+ # idx = np.nonzero((pxin_ndvibk < ndvi_thresholds['fill_ndvi'][0]) &
+ # (pxin_ndvibk > ndvi_thresholds['fill_ndvi'][1]) &
+ # (pxin_ndvibk >= ndvi_thresholds['ndvi_bnd1']) &
+ # (pxin_ndvibk < ndvi_thresholds['ndvi_bnd2']))
+ idx = np.nonzero((pxin_ndvibk >= ndvi_thresholds['ndvi_bnd1']) &
+ (pxin_ndvibk < ndvi_thresholds['ndvi_bnd2']))
+ interp[idx] = 1 # this is not really needed anymore, maybe
+ indvi = (pxin_ndvibk[idx] / delta_ndvi_bin) - 0.5
+ indvi[indvi < 0] = 0
+ x = (pxin_ndvibk - delta_ndvi_bin*indvi + delta_ndvi_bin/2) / delta_ndvi_bin
+ x[x < 0] = 0
+ x[x > 1] = 1
+
+ for i in range(3):
+ # idx = np.nonzero((pxin_ndvibk < ndvi_thresholds['fill_ndvi'][0]) &
+ # (pxin_ndvibk > ndvi_thresholds['fill_ndvi'][1]))
+
+ y1[:, i] = (coeffs[indvi, i, 0] +
+ coeffs[indvi, i, 1]*pxin_sctang +
+ coeffs[indvi, i, 2]*pxin_sctang**2 +
+ coeffs[indvi, i, 3]*pxin_sctang**3)
+
+ # idx = np.nonzero((pxin_ndvibk < ndvi_thresholds['fill_ndvi'][0]) &
+ # (pxin_ndvibk > ndvi_thresholds['fill_ndvi'][1]) & interp == 1)
+ y2[idx, i] = (coeffs[indvi[idx]+1, i, 0] +
+ coeffs[indvi[idx]+1, i, 1]*pxin_sctang[idx] +
+ coeffs[indvi[idx]+1, i, 2]*pxin_sctang[idx]**2 +
+ coeffs[indvi[idx]+1, i, 3]*pxin_sctang[idx]**3)
+
+ thr[:, i] = ((1.0 - x) * y1) + (x * y2)
+
+ thr_adj[pxin_ndvibk < des_ndvi, i] = thr[pxin_ndvibk < des_ndvi, 0]*ndvi_thresholds['adj_fac_desert']
+ thr_adj[pxin_ndvibk >= des_ndvi, i] = thr[pxin_ndvibk >= des_ndvi, 0]*ndvi_thresholds['adj_fac_land']
+
+ thr[:, 3] = 2.0
+
+ return thr
+
+
+def find_scene(data):
+ scene = {'ocean_day': 0,
+ 'ocean_night': 0,
+ 'land_day': 0,
+ 'land_night': 0,
+ 'snow_day': 0,
+ 'coast_day': 0,
+ 'desert_day': 0,
+ 'antarctic_day': 0,
+ 'polar_day_snow': 0,
+ 'polar_day_desert': 0,
+ 'polar_day_desert_coast': 0,
+ 'polar_day_coast': 0,
+ 'polar_day_land': 0,
+ 'polar_night_snow': 0,
+ 'polar_night_land': 0,
+ 'polar_ocean_night': 0,
+ }
+
+ return scene
+
+
+def snow_mask(viirs_data):
+ # (I1 - I3) / (I1 + I3)
+ # ndsi = (b04 - b10) / (b04 + b10)
+ # ndvi = (b07 - b05) / (b05 + b07)
+ # pred_ndvi = prd_ndvi_const[0] + 3.0*ndsi
+
+ pass
+
diff --git a/pyproject.toml b/pyproject.toml
index 1884af7..b4c84ce 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,24 +1,48 @@
-[build-system]
-requires = ['setuptools>=61.0', 'wheel', 'Cython', 'numpy', 'xarray']
-build-backend = 'setuptools.build_meta'
-
[project]
name = 'MVCM'
-version = '0.0.3'
-authors = [{name='Paolo Veglio', email='paolo.veglio@ssec.wisc.edu'}]
-description = 'High-Resolution MODIS-VIIRS Cloud Mask Algorithm'
+description = 'MODIS-VIIRS Cloud Mask algorithm for LEO/GEO'
readme = 'README.md'
-requires-python = '>=3.9'
-classifiers = ['Programming Language :: Python :: 3',
- 'Programming Language :: C',
-# 'License :: OSI Approved :: MIT License',
- 'Operating System :: OS Independent',
- ]
+requires-python = '>=3.9,<3.12'
+# license = ''
+authors = [
+ { name = 'Paolo Veglio', email = 'paolo.veglio@ssec.wisc.edu'},
+ ]
+classifiers = [
+ 'Development Status :: 2 - Pre-Alpha',
+ # 'License :: OSI Approved :: Apache Software License',
+ 'Programming Language :: Python :: 3.9',
+ 'Programming Language :: Python :: 3.10',
+ 'Programming Language :: Python :: 3.11',
+ 'Programming Language :: Python :: Implementation :: CPython',
+ 'Programming Language :: Cython',
+ 'Programming Language :: C',
+ ]
+dependencies = [
+ 'Cython',
+ 'wheel',
+ 'setuptools',
+ 'numpy',
+ 'xarray',
+ 'attrs',
+ ]
+dynamic = ['version']
+
+[tool.hatch.version]
+path = 'mvcm/__init__.py'
+
-[project.urls]
-'Homepage' = 'https://gitlab.ssec.wisc.edu/pveglio/mvcm'
+[tool.hatch.envs.test]
+dependencies = [
+ 'pytest',
+ ]
+
+[build-system]
+requires = ['hatchling']
+build-backend = 'hatchling.build'
[tool.pytest.ini_options]
addopts = [
'--import-mode=importlib',
-]
+ ]
+
+
diff --git a/setup.py b/setup.py
index 18f61ec..137d269 100644
--- a/setup.py
+++ b/setup.py
@@ -1,31 +1,32 @@
-from distutils.core import setup
-from distutils.extension import Extension
+# from distutils.core import setup
+# from distutils.extension import Extension
+from setuptools import Extension, setup, find_packages
from Cython.Build import cythonize
import numpy
-sourcefiles = ['src/get_Reynolds_SST.c', # SST
- 'src/bilinearInterpSST.c', # _|
- 'src/swap_bytes.c', # _|
- 'src/get_NDVI_background.c', # NDVI
- 'src/get_Olson_eco.c', # Olson eco type
- 'src/getcoord.c', # _|
- 'src/get_GEOS.c', # GEOS
- 'src/get_geos_times.c', # _|
- 'src/get_granule_times.c', # _|
- 'src/get_ti_vars.c', # _|
- 'src/get_ti_weights.c', # _|
- 'src/read_GEOS.c', # _|
- 'src/read_GEOS_constants.c', # _|
- 'src/read_GEOS_lndocn.c', # _|
- 'src/assign_geos_vals.c', # _|
- 'src/cithr.c', # Not sure
- 'src/check_reg_uniformity.c', # reg. uniformity for restoral
- 'ancillary.pyx',
+sourcefiles = ['mvcm/ancillary.pyx',
+ 'mvcm/c_tools/get_Reynolds_SST.c', # SST
+ 'mvcm/c_tools/bilinearInterpSST.c', # _|
+ 'mvcm/c_tools/swap_bytes.c', # _|
+ 'mvcm/c_tools/get_NDVI_background.c', # NDVI
+ 'mvcm/c_tools/get_Olson_eco.c', # Olson eco type
+ 'mvcm/c_tools/getcoord.c', # _|
+ 'mvcm/c_tools/get_GEOS.c', # GEOS
+ 'mvcm/c_tools/get_geos_times.c', # _|
+ 'mvcm/c_tools/get_granule_times.c', # _|
+ 'mvcm/c_tools/get_ti_vars.c', # _|
+ 'mvcm/c_tools/get_ti_weights.c', # _|
+ 'mvcm/c_tools/read_GEOS.c', # _|
+ 'mvcm/c_tools/read_GEOS_constants.c', # _|
+ 'mvcm/c_tools/read_GEOS_lndocn.c', # _|
+ 'mvcm/c_tools/assign_geos_vals.c', # _|
+ 'mvcm/c_tools/cithr.c', # Not sure
+ 'mvcm/c_tools/check_reg_uniformity.c', # reg. uniformity for restoral
]
-include_dirs = ['include',
+include_dirs = ['mvcm/c_tools/include',
'/opt/hdf4/4.2.14-gcc-8.3/include',
'/opt/hdfeos2/2.20-gcc-8.3/include',
'/opt/netcdf4/4.7.0-gcc-8.3/include',
@@ -41,5 +42,6 @@ extensions = [Extension('ancillary_data', sourcefiles,
library_dirs=library_dirs,
libraries=['netcdf']), ]
-setup(ext_modules=cythonize(extensions,
+setup(packages=find_packages(),
+ ext_modules=cythonize(extensions,
compiler_directives={'language_level': '3'}, ), )
diff --git a/tests/test_read_data.py b/tests/test_read_data.py
index 7541221..7f99a65 100644
--- a/tests/test_read_data.py
+++ b/tests/test_read_data.py
@@ -4,7 +4,7 @@ import os
import pytest
-import read_data as rd
+import mvcm.read_data as rd
@pytest.fixture
@@ -13,13 +13,13 @@ def fixturepath():
@pytest.fixture
-def l1b_file():
- return 'VNP02MOD.A2022173.1312.001.2022174011547.uwssec.nc'
+def l1b_file(fixturepath):
+ return os.path.join(fixturepath, 'VNP02MOD.A2022173.1312.001.2022174011547.uwssec.nc')
@pytest.fixture
-def geo_file():
- return 'VNP03MOD.A2022173.1312.001.2022174012746.uwssec.nc'
+def geo_file(fixturepath):
+ return os.path.join(fixturepath, 'VNP03MOD.A2022173.1312.001.2022174012746.uwssec.nc')
# @pytest.fixture
@@ -74,8 +74,12 @@ def test_l1b(fixturepath, l1b_file, geo_file):
file_name_l1b=l1b_file,
file_name_geo=geo_file)
geo = viirs.read_viirs_geo()
- l1b = viirs.read_l1b(geo.solar_zenith.values)
+ l1b = viirs.read_viirs_l1b(geo.solar_zenith.values)
+ geo_vars = ['latitude', 'longitude', 'height', 'range', 'solar_zenith', 'solar_azimuth',
+ 'sensor_zenith', 'sensor_azimuth', 'land_water_mask', 'quality_flag',
+ 'relative_azimuth', 'sunglint_angle', 'scattering_angle']
+ assert list(geo).sort() == geo_vars.sort()
assert 'M11' in l1b.data_vars
diff --git a/thresholds/thresholds.mvcm.snpp.v0.0.1.yaml b/thresholds/thresholds.mvcm.snpp.v0.0.1.yaml
new file mode 100644
index 0000000..801c795
--- /dev/null
+++ b/thresholds/thresholds.mvcm.snpp.v0.0.1.yaml
@@ -0,0 +1,706 @@
+# MODIS-VIIRS Continuity Cloud Mask (MVCCM) Thresholds
+
+# Version string
+thresholds_file_ver: v3.3.1_NPP
+
+Land_Day:
+ 11-12um_Cirrus_Test:
+ coeffs: [3.0, 1.0]
+ cmult: 0.3
+ adj: 1.25
+ perform: True
+ dl_ref3_tpw : -10.00
+ 11-4um_Oceanic_Stratus_Test:
+ thr: [-16.0, -14.0, -12.0, 1.0, 1.0]
+ perform: True
+ Visible_Reflectance_Test:
+ thr: [0.207, 0.169, 0.132, 1.0, 1.0] # used to be dlref1
+ adj: 0.94 # used to be dl_b1bias_adj
+ perform: True
+ 1.38um_High_Cloud_Test:
+ thr: [0.0375, 0.0250, 0.0125, 1.0, 1.0]
+ perform: True
+ dlvrat : [1.80, 1.85, 1.90, 1.0, 1.0]
+# dl11_12lcmult : 0.3
+# dl11_12hcadj : 1.25
+ CO2_High_Clouds_Test: [222.0, 224.0, 226.0, 1.0, 1.0]
+ Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
+
+Land_Night:
+ 11-12um_Cirrus_Test:
+ coeffs: [3.0, 1.0]
+ cmult: 0.0
+ adj: 0
+ bt1: 270.0 # WILL FIND A BETTER NAME AT SOME POINT, MAYBE
+ lat: 30.0
+ perform: True
+ 4-12um_BTD_Thin_Cirrus_Test:
+ thr: [15.0, 10.0, 5.00, 1.0, 1.0]
+ perform: True
+ 7.3-11um_BTD_Mid_Level_Cloud_Test:
+ thr: [-8.0, -10.0, -11.0, 1.0, 1.0] # nl7_11s
+ 11-4um_BT_Difference_Test:
+ thr_low: [0.0, -0.5, -1.0, 1.0]
+ thr_mid: [-0.5, 6.0, 0.5, 1.0]
+ thr_hi: [6.5, 6.0, 5.5, 1.0]
+ coeffs: [-0.3476, 0.5307, -0.1930]
+ int_adj: 0
+ ndvi: 0.25
+ ndvi_australia: 0.4
+ bt_diff_bounds : [1.0, -1.0]
+ perform: True
+ Surface_Temperature_Test:
+ desert_thr: 20.0
+ regular_thr: 12.0
+ channel_diff_11-12um_thr: [-0.2, 1.0]
+ channel_diff_11-4um_thr: [-0.5, 1.0]
+ perform: True
+ # Surface_Temperature_Test_df1: [-0.2, 1.0]
+ # Surface_Temperature_Test_df2: [-0.5, 1.0]
+ # Surface_Temperature_Test_difference: [-0.2, 1.0, -0.5, 1.0] # <- this merges the previous two arrays
+ # Surface_Temperature_Test_1: 20.0 # | might be worth figuring out if we can
+ # Surface_Temperature_Test_2: 12.0 # | merge these three coefficients
+ # Surface_Temperature_Test_pfm: 1.0 # __|
+ # Surface_Temperature_Test: [20.0, 12.0, 1.0] # <- First attempt to merge the three values above
+ nlbt1 : 270.0
+ nl_lat : 30.0
+ # nl_ndvi : 0.25
+ # nl_ndvi_Aust : 0.40
+ # nl_intadj : 0.0
+ nl_btd1 : -2.0
+ # nl11_4coef : [-0.3476, 0.5307, -0.1930]
+ CO2_High_Clouds_Test: [222.0, 224.0, 226.0, 1.0, 1.0]
+ Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
+
+Land_Day_Coast:
+ 11-12um_Cirrus_Test:
+ coeffs: [3.0, 1.0]
+ cmult: 0.3
+ adj: 1.25
+ perform: True
+ dl_ref3_tpw_t2 : -10.00
+ 11-4um_Oceanic_Stratus_Test:
+ thr: [-16.0, -14.0, -12.0, 1.0, 1.0]
+ perform: True
+ CO2_High_Clouds_Test: [222.0, 224.0, 226.0, 1.0, 1.0]
+ Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
+ Visible_Reflectance_Test:
+ thr: [0.207, 0.169, 0.132, 1.0, 1.0]
+ adj: 0.94
+ perform: True
+ 1.38um_High_Cloud_Test:
+ thr: [0.0375, 0.0250, 0.0125, 1.0, 1.0]
+ perform: True
+# dl11_12lcmult_t2 : 0.3
+# dl11_12hcadj_t2 : 1.25
+
+Land_Day_Desert:
+# lds11_12hi : [3.5, 1.0]
+ 11-12um_Cirrus_Test:
+ coeffs: [3.5, 1.0]
+ cmult: 0.3
+ adj: 1.25
+ perform: True
+ 11-4um_Oceanic_Stratus_Test:
+ thr: [-28.0, -26.0, -24.0, -2.0, 0.0, 2.0, 1.0, 1.0] # this replaces lds11_4hi and lds11_4lo. The values
+ perform: True # are sorted left to right so that they follow the
+ # scheme: Lo-Mid-Hi--Hi-Mid-Lo This is the opposite
+ # of Ocean_Day
+ # lds11_4hi : [2.0, 0.0, -2.0, 1.00, 1.0]
+ # lds11_4lo : [-28.0, -26.0, -24.0, 1.00]
+ CO2_High_Clouds_Test: [222.0, 224.0, 226.0, 1.0, 1.0]
+ Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
+ Visible_Reflectance_Test:
+ thr: [0.326, 0.288, 0.250, 1.00, 1.0]
+ adj: 0.94
+ ndvi_thr: 0.25 # this used to be lds_ndvi
+ perform: True
+ 1.38um_High_Cloud_Test:
+ thr: [0.0375, 0.0250, 0.0125, 1.0, 1.0]
+ perform: True
+ GEMI_Test:
+ gemi0: [0.085, 0.095, 0.115, 1.00, 1.0]
+ gemi1: [0.145, 0.170, 0.220, 1.00]
+ gemi2: [0.310, 0.335, 0.360, 1.00]
+ perform: True
+ lds_ref3_tpw : -10.00
+ ldsbt1 : 320.0
+# lds11_12lcmult : 0.3
+# lds11_12hcadj : 1.25
+
+Land_Day_Desert_Coast:
+# lds11_12hi_c : [3.5, 1.0]
+ 11-12um_Cirrus_Test:
+ coeffs: [3.5, 1.0]
+ cmult: 0.3
+ adj: 1.25
+ perform: True
+ 11-4um_Oceanic_Stratus_Test:
+ thr: [-23.0, -21.0, -19.0, -2.0, 0.0, 2.0, 1.0, 1.0]
+ perform: True
+ CO2_High_Clouds_Test: [222.0, 224.0, 226.0, 1.0, 1.0]
+ Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
+ Visible_Reflectance_Test:
+ thr: [0.326, 0.288, 0.250, 1.0, 1.0]
+ adj: 0.94
+ ndvi_thr: 0.25
+ perform: True
+ 1.38um_High_Cloud_Test:
+ thr: [0.0375, 0.0250, 0.0125, 1.0, 1.0]
+ perform: True
+ lds_ref3_tpw_c : -10.00
+ ldsbt1_c : 320.0
+# lds11_12lcmult_c : 0.3
+# lds11_12hcadj_c : 1.25
+
+Ocean_Day:
+ 11um_Test:
+ thr: [267.0, 270.0, 273.0, 1.0, 1.0]
+ perform: True
+ SST_Test:
+ thr: [3.000, 2.500, 2.000, 1.0, 1.0]
+ coeffs: [1.8860, 0.9380, 0.1280, 1.094]
+ perform: True
+ 8.6-11um_Test:
+ thr: [-0.50, -1.00, -1.50, 1.0, 1.0]
+ perform: True
+ 11-12um_Cirrus_Test:
+ coeffs: [3.0, 1.0]
+ cmult: 0.3
+ adj: 1.25
+ perform: True
+ 11-4um_Oceanic_Stratus_Test:
+ thr: [-11.0, -9.0, -7.0, 1.0, 1.0]
+ perform: True
+ NIR_Reflectance_Test:
+ thr: [0.062, 0.043, 0.029, 1.0, 1.0]
+ coeffs: [1.7291, 0.0715, -0.0026, 0.000025889]
+ adj: 0.0050
+ bias: 0.96
+ perform: True
+ midpt_coeff: 0.0200
+ locut_coeff: 0.0100
+ Vis/NIR_Ratio_Test:
+ thr: [0.837, 0.889, 0.934, 1.001, 1.154, 1.205, 1.0, 1.0] # This replace dovrathi and dovratlo
+ perform: True # The values are sorted left to right
+ # so that they follow the scheme:
+ # Hi-Mid-Lo--Lo-Mid-Hi
+ # Last two values are power and switch
+ # to turn the test on/off
+ 1.6_2.1um_NIR_Reflectance_Test:
+ thr: [0.0, 0.0, 0.0, 1.0, 1.0] # This is a placeholder to force consistency with NIR_Reflectance_Test.
+ # In both cases the only value used is thr[3] AFAIK
+ coeffs: [1.01339303876915, -0.00277128813739, 0.00027804834484, -0.00001549681141, 0.00000016623006, 0.00000000000000]
+ bias: 0.98 # this corresponds to do_b6bias_adj
+ adj: 0.0125
+ locut_coeff: 0.0100
+ midpt_coeff: 0.0070
+ perform: True
+ CO2_High_Clouds_Test: [222.0, 224.0, 226.0, 1.0, 1.0]
+ Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
+ # vnir_ratio_hi : [1.001, 1.154, 1.205, 1.0, 1.0]
+ # vnir_ratio_lo : [0.934, 0.889, 0.837, 1.0]
+ 1.38um_High_Cloud_Test:
+ coeffs: [-0.2419, 0.0455, -0.0024, 0.000059029, -0.00000069964, 0.000000003204]
+ adj: 0.0010
+ szafac: 4.0
+ perform: True
+ do_b6bias_adj : 0.98
+ do_b7bias_adj : 0.97
+
+Ocean_Night:
+ 11um_Test:
+ thr: [267.0, 270.0, 273.0, 2.0, 1.0]
+ perform: True
+ SST_Test:
+ thr: [3.000, 2.500, 2.000, 1.0, 1.0]
+ coeffs: [1.8860, 0.9380, 0.1280, 1.094]
+ perform: True
+ 8.6-11um_Test:
+ thr: [-0.5, -1.0, -1.5, 1.0, 1.0]
+ perform: True
+ 11-12um_Cirrus_Test:
+ coeffs: [3.0, 1.0]
+ cmult: 0.3
+ adj: 1.0
+ perform: True
+ 11-4um_Oceanic_Stratus_Test:
+ thr: [1.25, 1.0, 0.25, 1.0, 1.0]
+ perform: True
+ Water_Vapor_Cloud_Test:
+ thr: [14.0, 15.0, 16.0, 1.0, 1.0]
+ perform: True
+ 11um_Variability_Test:
+ thr: [3.0, 6.0, 7.0, 1.0, 1.0]
+ variability: 0.40
+ perform: True
+ 11-4um_BT_Difference_Test:
+ coeffs: [-0.7093, 0.1128, -0.1567]
+ corr: 0.0 # no_intadj
+ locut_coeff: [-4.0, 4.0] # no11_4load
+ midpt_coeff: [-2.5, 2.5] # no11_4mdad
+ hicut_coeff: [-1.5, 1.5] # no11_4hiad
+ perform: True # no11_4_pfm
+ CO2_High_Clouds_Test: [222.0, 224.0, 226.0, 1.0, 1.0]
+ Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
+
+Polar_Day_Land:
+ 11-12um_Cirrus_Test:
+ coeffs: [3.0, 1.0]
+ cmult: 0.3
+ adj: 0.3
+ perform: True
+ pdl_ref3_tpw : -10.00
+ 11-4um_Oceanic_Stratus_Test:
+ thr: [-16.0, -14.0, -12.0, 1.0, 1.0]
+ perform: True
+ pdlh20 : [215.0, 220.0, 225.0, 1.0, 1.0]
+ Visible_Reflectance_Test:
+ thr: [0.207, 0.169, 0.132, 1.0, 1.0]
+ adj: 0.94
+ perform: True
+ 1.38um_High_Cloud_Test:
+ thr: [0.0375, 0.0250, 0.0125, 1.0, 1.0]
+ perform: True
+
+Polar_Night_Land:
+ Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
+ 11-12um_Cirrus_Test:
+ coeffs: [3.0, 1.0]
+ cmult: 0
+ adj: 0 # I NEED TO WORK ON THIS
+ bt1: 270.0
+ perform: True
+ pnlbt2 : 270.0
+ Surface_Temperature_Test:
+ desert_thr: 20.0
+ regular_thr: 12.0
+ channel_diff_11-12um_thr: [0, 1.0]
+ channel_diff_11-4um_thr: [-0.5, 1.0]
+ perform: True
+ # Surface_Temperature_Test_df1: [0.0, 1.0]
+ # Surface_Temperature_Test_df2: [-0.5, 1.0]
+ # Surface_Temperature_Test_difference: [-0.2, 1.0, -0.5, 1.0] # <- this merges the previous two arrays
+ # Surface_Temperature_Test_1: 20.0 # | might be worth figuring out if we can
+ # Surface_Temperature_Test_2: 12.0 # | merge these three coefficients
+ # Surface_Temperature_Test_pfm: 1.0 # __|
+ # Surface_Temperature_Test: [20.0, 12.0, 1.0] # <- First attempt to merge the three values above
+ 11-4um_BT_Difference_Test:
+ low: [2.00, 1.70, 1.40, 1.0]
+ mid1: [1.70, 0.70, 0.3, 1.0]
+ mid2: [0.00, 0.00, 0.00, 0.0]
+ mid3: [0.00, 0.00, 0.00, 0.0]
+ high: [1.00, 0.70, 0.40, 1.0]
+ bt11_bounds: [235, 0, 0, 265]
+ perform: True
+ 7.3-11um_BTD_Mid_Level_Cloud_Test:
+ low: [-1.00, 0.00, 1.00, 1.0, 1.0] # pn_7_11l
+ mid1: [0.00, -4.50, -1.00, 1.0] # pn_7_11m1
+ mid2: [-4.50, -10.50, -1.00, 1.0] # pn_7_11m2
+ mid3: [-10.50, -20.0, -1.00, 1.0] # pn_7_11m3
+ high: [-21.0, -20.0, -19.0,1.0] # pn_7_11h
+ bt11_bounds: [220.0, 245.0, 255.0, 265.0] # bt11_bnds2
+ 4-12um_BTD_Thin_Cirrus_Test:
+ low: [2.00, 1.70, 1.40, 1.0]
+ mid1: [1.70, 0.70, 0.300, 1.0]
+ mid2: [0.00, 0.00, 0.00, 0.0]
+ mid3: [0.00, 0.00, 0.00, 0.0]
+ high: [1.00, 0.70, 0.40, 1.0]
+ bt11_bounds: [235.0, 000.0, 000.0, 265.0]
+ perform: True
+# pnl_11_4_pfm : 1.0
+# pnl_7_11_pfm : 1.0
+# pnl_4_12_pfm : 1.0
+
+Polar_Day_Coast:
+ 11-12um_Cirrus_Test:
+ coeffs: [3.0, 1.0]
+ cmult: 0 # I NEED TO WORK ON THIS
+ adj: 0 # I NEED TO WORK ON THIS
+ perform: True
+ pdl_ref3_tpw_t2 : -10.00
+ 11-4um_Oceanic_Stratus_Test:
+ thr: [-16.0, -14.0, -12.0, 1.0, 1.0]
+ perform: True
+ Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
+ Visible_Reflectance_Test:
+ thr: [0.207, 0.169, 0.132, 1.0, 1.0]
+ adj: 0.94
+ perform: True
+ 1.38um_High_Cloud_Test:
+ thr: [0.0375, 0.0250, 0.0125, 1.0, 1.0]
+ perform: True
+
+Polar_Day_Desert:
+ 11-12um_Cirrus_Test:
+ coeffs: [3.5, 1.0]
+ cmult: 0 # I NEED TO WORK ON THIS
+ adj: 0 # I NEED TO WORK ON THIS
+ perform: True
+ 11-4um_Oceanic_Stratus_Test:
+ thr: [-22.0, -20.0, -18.0, -2.0, 0.0, 2.0, 1.0, 1.0]
+ perform: True
+ Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
+ Visible_Reflectance_Test:
+ thr: [0.326, 0.288, 0.250, 1.00, 1.0]
+ adj: 0.94
+ perform: True
+ 1.38um_High_Cloud_Test:
+ thr: [0.0375, 0.0250, 0.0125, 1.00, 1.0]
+ perform: True
+ GEMI_Test:
+ gemi0: [0.085, 0.110, 0.135, 1.00, 1.0]
+ gemi1: [0.170, 0.220, 0.270, 1.00]
+ gemi2: [0.310, 0.335, 0.360, 1.00]
+ perform: True
+ pds_ref3_tpw : -10.00
+ pdsbt1 : 320.0
+
+Polar_Day_Desert_Coast:
+ 11-12um_Cirrus_Test:
+ coeffs: [3.5, 1.0]
+ cmult: 0 # I NEED TO WORK ON THIS
+ adj: 0 # I NEED TO WORK ON THIS
+ perform: True
+ 11-4um_Oceanic_Stratus_Test:
+ thr: [-23.0, -21.0, -19.0, -2.0, 0.0, 2.0, 1.0, 1.0]
+ perform: True
+ Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
+ Visible_Reflectance_Test:
+ thr: [0.326, 0.288, 0.250, 1.00, 1.0]
+ adj: 0.94
+ perform: True
+ 1.38um_High_Cloud_Test:
+ thr: [0.0375, 0.0250, 0.0125, 1.00, 1.0]
+ perform: True
+ pds_ref3_tpw_c : -10.00
+ pdsbt1_c : 320.0
+
+Polar_Day_Snow:
+ 11-12um_Cirrus_Test:
+ coeffs: [3.0, 1.0]
+ cmult: 0 # I NEED TO WORK ON THIS
+ adj: 0 # I NEED TO WORK ON THIS
+ perform: True
+ dps_ref3_tpw : 0.75
+ dpsbt1 : 230.0
+ Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
+ 1.38um_High_Cloud_Test:
+ thr: [0.060, 0.0525, 0.045, 1.0, 1.0]
+ perform: True
+ 11-4um_BT_Difference_Test:
+ low: [20.0, 18.0, 16.0, 1.0]
+ mid1: [18.0, 16.0, 2.0, 1.0]
+ mid2: [0.0, 0.0, 0.0, 0.0]
+ mid3: [0.0, 0.0, 0.0, 0.0]
+ high: [18.0, 16.0, 14.0, 1.0]
+ bt_11_bounds: [230.0, 0.0, 0.0, 245.0]
+ perform: True
+ dps11_12adj : 0.8
+
+Polar_Night_Snow:
+ 11-12um_Cirrus_Test:
+ coeffs: [3.0, 1.0]
+ cmult: 0
+ adj: 0 # I NEED TO WORK ON THIS
+ bt1: 270.0
+ perform: True
+ 4-12um_BTD_Thin_Cirrus_Test:
+ low: [4.50, 4.00, 3.50, 1.0] # pn_4_12l
+ mid1: [4.00, 2.00, 0.500, 1.0] # pn_4_12m1
+ mid2: [0.00, 0.00, 0.00, 0.0] # pn_4_12m2
+ mid3: [0.00, 0.00, 0.00, 0.0] # pn_4_12m3
+ high: [2.50, 2.00, 1.50, 1.0] # pn_4_12h
+ bt11_bounds: [235.0, 0.0, 0.0, 265.0] # bt_11_bounds
+ perform: True
+ 7.3-11um_BTD_Mid_Level_Cloud_Test:
+ low: [-1.00, 0.00, 1.00, 1.0, 1.0] # pn_7_11l
+ mid1: [0.00, -4.50, -1.00, 1.0] # pn_7_11m1
+ mid2: [-4.50, -10.50, -1.00, 1.0] # pn_7_11m2
+ mid3: [-10.50, -20.0, -1.00, 1.0] # pn_7_11m3
+ high: [-21.0, -20.0, -19.0,1.0] # pn_7_11h
+ low_ice: [0.00, 1.00, 2.00, 1.0] # pn_7_11lw
+ mid1_ice: [1.00, -7.00, -1.00, 1.0] # pn_7_11m1w
+ mid2_ice: [-7.00, -16.50, -1.00, 1.0] # pn_7_11m2w
+ mid3_ice: [-16.50, -20.0, -1.00, 1.0] # pn_7_11m3w
+ high_ice: [-21.0 -20.0, -19.0, 1.0] # pn_7_11hw
+ bt11_bounds: [220.0, 245.0, 255.0, 265.0] # bt11_bnds2
+ Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
+ 11-4um_BT_Difference_Test:
+ top:
+ low: [2.00, 1.70, 1.40, 1.0]
+ mid1: [1.70, 0.30, 0.300, 1.0]
+ mid2: [0.00, 0.00, 0.00, 0.0]
+ mid3: [0.00, 0.00, 0.00, 0.0]
+ high: [0.60, 0.30, 0.00, 1.0]
+ bottom:
+ low: [-3.00, -2.50, -2.00, 1.0]
+ mid1: [-2.50, -2.50, -0.50, 1.0]
+ mid2: [0.00, 0.00, 0.00, 0.0]
+ mid3: [0.00, 0.00, 0.00, 0.0]
+ high: [-3.00, -2.50, -2.00, 1.0]
+ bt_thr: 230.0 # pnbt3
+ tpw_thr: 0.2 # pntpw
+ perform: True
+ pn11_12adj : 0.8
+ pnbt2 : 270.0
+
+Polar_Day_Ocean:
+ 11um_Test:
+ thr: [267.0, 270.0, 273.0, 1.0, 1.0]
+ perform: True
+ SST_Test:
+ thr: [3.000, 2.500, 2.000, 1.0, 1.0]
+ coeffs: [1.886, 0.938, 0.128, 1.094]
+ perform: True
+ 8.6-11um_Test:
+ thr: [-0.50, -1.00, -1.50, 1.0, 1.0]
+ perform: True
+ 11-12um_Cirrus_Test:
+ coeffs: [3.0, 1.0]
+ cmult: 0.3
+ adj: 1.25
+ perform: True
+ NIR_Reflectance_Test:
+ thr: [0.062, 0.043, 0.029, 1.0, 1.0]
+ coeffs: [1.7291, 0.0715, -0.0026, 0.000025889]
+ adj: 0.0050
+ bias: 0.96
+ midpt_coeff: 0.0200
+ locut_coeff: 0.0100
+ perform: True
+ Vis/NIR_Ratio_Test:
+ thr: [0.837, 0.889, 0.934, 1.001, 1.154, 1.205, 1.0, 1.0]
+ perform: True
+ 1.6_2.1um_NIR_Reflectance_Test:
+ thr: [0.0, 0.0, 0.0, 1.0, 1.0]
+ coeffs: [1.01339303876915, -0.00277128813739, 0.00027804834484, -0.00001549681141, 0.00000016623006, 0.00000000000000]
+ bias: 0.98
+ adj: 0.0125
+ locut_coeff: 0.0100
+ midpt_coeff: 0.0070
+ perform: True
+ 11-4um_Oceanic_Stratus_Test:
+ thr: [-11.0, -9.0, -7.0, 1.0, 1.0]
+ perform: True
+ Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
+ pdo_b26pfm : 1.0
+ pdo_b2bias_adj : 0.96
+ pdo_b7bias_adj : 0.97
+ 1.38um_High_Cloud_Test:
+ coeffs: [-0.2419, 0.0455, -0.0024, 0.000059029, -0.00000069964, 0.000000003204]
+ adj: 0.0010
+ szafac: 4.0
+ perform: True
+
+Polar_Night_Ocean:
+ 11um_Test:
+ thr: [267.0, 270.0, 273.0, 1.0, 1.0]
+ perform: True
+ SST_Test:
+ thr: [3.000, 2.500, 2.000, 1.0, 1.0]
+ coeffs: [1.8860, 0.9380, 0.1280, 1.094]
+ perform: True
+ 8.6-11um_Test:
+ thr: [-0.5, -1.0, -1.5, 1.0, 1.0]
+ perform: True
+ 11-12um_Cirrus_Test:
+ coeffs: [3.0, 1.0]
+ cmult: 0.3
+ adj: 1.0
+ perform: True
+ pnobt1 : 280.0
+ 11-4um_Oceanic_Stratus_Test:
+ thr: [1.25, 1.0, 0.25, 1.0, 1.0]
+ perform: True
+ Water_Vapor_Cloud_Test:
+ thr: [14.0, 15.0, 16.0, 1.0, 1.0]
+ perform: True
+ 11um_Variability_Test:
+ thr: [3.0, 6.0, 7.0, 1.0, 1.0]
+ variability: 040
+ perform: True
+ 11-4um_BT_Difference_Test:
+ coeffs: [-0.7093, 0.1128, -0.1567]
+ corr: 0.0 # pno_intadj
+ locut_coeff: [-4.0, 4.0] # pno11_4load
+ midpt_coeff: [-2.5, 2.5] # pno11_4mdad
+ hicut_coeff: [-1.5, 1.5] # pno11_4hiad
+ perform: True # pno11_4_pfm
+ Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
+
+
+Day_Snow:
+ 11-12um_Cirrus_Test:
+ coeffs: [3.0, 1.0]
+ cmult: 0.3
+ adj: 0 # I NEED TO WORK ON THIS
+ perform: True
+ ds_ref3_tpw : 0.75
+ CO2_High_Clouds_Test: [222.0, 224.0, 226.0, 1.0, 1.0]
+ Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
+ 1.38um_High_Cloud_Test:
+ thr: [0.060, 0.0525, 0.045, 1.0, 1.0]
+ perform: True
+ 11-4um_BT_Difference_Test:
+ low: [20.0, 18.0, 16.0, 1.0]
+ mid1: [18.0, 16.0, 2.0, 1.0]
+ mid2: [0.0, 0.0, 0.0, 0.0]
+ mid3: [0.0, 0.0, 0.0, 0.0]
+ high: [18.0, 16.0, 14.0, 1.0]
+ bt_11_bounds: [230.0, 0.0, 0.0, 245.0]
+ perform: True
+ ds11_12adj : 0.8
+ ds11_12lcmult : 0.3
+ ds11_12hcmult : 0.3
+
+Night_Snow:
+ 11-12um_Cirrus_Test:
+ coeffs: [3.0, 1.0]
+ cmult: 0
+ adj: 0 # I NEED TO WORK ON THIS
+ bt1: 270.0
+ perform: True
+ nsbt2 : 270.0
+ 4-12um_BTD_Thin_Cirrus_Test:
+ thr: [4.50, 4.00, 3.50, 1.0, 1.0]
+ perform: True
+ 11-4um_BT_Difference_Test:
+ thr: [0.70, 0.60, 0.50, 1.0, 1.0]
+ bottom:
+ low: [20.0, 18.0, 16.0, 1.0]
+ mid1: [18.0, 16.0, 2.0, 1.0]
+ mid2: [0.0, 0.0, 0.0, 0.0]
+ mid3: [0.0, 0.0, 0.0, 0.0]
+ high: [18.0, 16.0, 14.0, 1.0]
+ bt_11_bounds: [230.0, 0.0, 0.0, 245.0]
+ bt_thr: 230.0
+ tpw_thr: 0.2
+ perform: True
+ 7.3-11um_BTD_Mid_Level_Cloud_Test:
+ low: [-1.00, 0.00, 1.00, 1.0, 1.0] # pn_7_11l
+ mid1: [0.00, -4.50, -1.00, 1.0] # pn_7_11m1
+ mid2: [-4.50, -10.50, -1.00, 1.0] # pn_7_11m2
+ mid3: [-10.50, -20.0, -1.00, 1.0] # pn_7_11m3
+ high: [-21.0, -20.0, -19.0,1.0] # pn_7_11h
+ low_ice: [0.00, 1.00, 2.00, 1.0] # pn_7_11lw
+ mid1_ice: [1.00, -7.00, -1.00, 1.0] # pn_7_11m1w
+ mid2_ice: [-7.00, -16.50, -1.00, 1.0] # pn_7_11m2w
+ mid3_ice: [-16.50, -20.0, -1.00, 1.0] # pn_7_11m3w
+ high_ice: [-21.0 -20.0, -19.0, 1.0] # pn_7_11hw
+ bt11_bounds: [220.0, 245.0, 255.0, 265.0] # bt11_bnds2
+ perform: False
+ CO2_High_Clouds_Test: [222.0, 224.0, 226.0, 1.0, 1.0]
+ Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
+ ns11_12adj : 0.8
+
+Antarctic_Day:
+ Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
+ ant4_11l : [26.0 23.0, 20.0, 1.0]
+ ant4_11h : [18.0, 15.0, 12.0, 1.0]
+ ant4_11m1 : [23.0, 15.0, 3.0 1.0]
+ ant4_11m2 : [0.0, 0.0, 0.0, 0.0]
+ ant4_11m3 : [0.0, 0.0, 0.0, 0.0]
+ bt_11_bnds4 : [240.0, 0.0, 0.0, 255.0]
+ antbt1 : 230.0
+ ant4_11_pfm : 1.0
+
+Daytime_Ocean_Spatial_Variability:
+ dovar11 : 0.40
+ dovar86 : 0.0020
+
+Sun_Glint:
+ # snglntv : [1.02, 1.17]
+ # snglntvch : [0.97, 1.28]
+ # snglntvcl : [1.07, 1.12]
+ snglnt : [0.97, 1.02, 1.07, 1.12, 1.17, 1.28, 1.0, 1.0] # This replaces snglntv, snglntvch snglntvcl
+ # The values are sorted left to right such that
+ # Hi-Mid-Lo--Lo-Mid-Hi
+ # I also added the two 1.0 at the end to
+ # maintain consistency with the other
+ # thresholds for conf_test_dble
+ sg_tbdfl : 13.0
+ sngm09 : 0.568
+ sngm02vm : 0.001
+ band2_0deg : [.1296, .1152, .1008, 1.0] # snglnt0_b2
+ band2_10deg : [.1152, .0624, .0144, 1.0] # snglnt10_b2
+ band2_20deg : [.0624, .0432, .0144, 1.0] # snglnt20_b2
+ band7_0deg : [.1176, .1078, .0980, 1.0] # snglnt0_b7
+ band7_10deg : [.1078, .0686, .0098, 1.0] # snglnt10_b7
+ band7_20deg : [.0686, .0196, .0098, 1.0] # snglnt20_b7
+ bounds : [0.0, 10.0, 25.0, 40.0] # snglnt_bounds
+
+Land_Restoral:
+ ldsr5_4_thr : 1.96
+ ldr5_4_thr : 2.94
+ ld20m22 : 11.0
+ ld22m31 : 15.0
+ ldsbt11 : [295.0, 300.0, 305.0]
+ ldsbt11bd : [290.0, 295.0, 305.0]
+ lnbt11 : [287.5, 292.5, 297.5]
+
+Night_Snow_Inversion:
+ pn65_11 : [10.0, 1.0]
+ pn13_11 : [3.0, 1.0]
+ pn7_11 : [5.0, 1.0]
+ n65_11 : [10.0, 1.0]
+
+Snow_Mask:
+ sm_bt11 : 280.0
+ sm_ndsi : 0.31
+ Grnlnd_ndsi : 0.66
+ sm_ref2 : 0.1056
+ sm85_11 : 0.0
+ sm85_11hel : 1.5
+ sm37_11 : 11.0
+ sm37_11hel : 16.0
+ sm_mnir : 0.196
+ sm_lsfcdif : 20.0
+ sm_wsfcdif : 20.0
+ sm_bt1 : 273.0
+ prd_ndvi_const : -0.2015
+
+Coastal_NDVI_Thresholds:
+ coast_ndvi : [-0.18, 0.40]
+
+# Ocean NDVI and shallow (turbid) water test thresholds.
+Ocean_NDVI_and_Shallow_Water:
+ swc_ndvi : [-0.35, 0.40]
+ swc_124std : 0.0023
+
+Thin_Cirrus_Tests:
+ tci_ref3_tpw : 0.75
+ dltci : [0.0250, 0.0125]
+ dstci : [0.0350, 0.0300]
+
+# Power cosine(VZA) is raised to in VZA-correction codes
+VZA_correction:
+ vzcpow : [0.75, 0.75, 0.25]
+
+# Thresholds used in calculating 0.66 and 0.413 um cloud test thresholds
+Misc:
+ des_ndvi : 0.25
+ fill_ndvi : [32.000, 0.00] # replaces fill_ndvi1 and fill_ndvi2
+ ndvi_bnd1 : 0.05
+ ndvi_bnd2 : 0.95
+ adj_fac_desert : 0.10
+ adj_fac_land : 0.00
+
+# Coefficients for band 1 land thresholds
+Coeffs_Band1_land_thresh:
+ - [32.00000000, 0.00000000, 0.00000000, 0.00000000, 42.00000000, 0.00000000, 0.00000000, 0.00000000, 52.00000000, 0.00000000, 0.00000000, 0.00000000]
+ - [24.00000000, 0.00000000, 0.00000000, 0.00000000, 28.00000000, 0.00000000, 0.00000000, 0.00000000, 32.00000000, 0.00000000, 0.00000000, 0.00000000]
+ - [99.13076923, -2.00907925, 0.01492075, -0.00003531, 122.19090909, -2.32652292, 0.01659848, -0.00003681, 142.66293706, -2.57860528, 0.01773252, -0.00003685]
+ - [85.07902098, -1.59413364, 0.01123310, -0.00002556, 144.56573427, -2.81054779, 0.01967366, -0.00004324, 204.35454545, -4.03411810, 0.02816667, -0.00006103]
+ - [85.03846154, -1.50831391, 0.01006760, -0.00002199, 165.15314685, -3.24716783, 0.02255594, -0.00004965, 242.06363636, -4.90912587, 0.03445455, -0.00007587]
+ - [81.00979021, -1.37731935, 0.00881294, -0.00001859, 220.36783217, -4.44111888, 0.03087762, -0.00006888, 359.72587413, -7.50491841, 0.05294231, -0.00011917]
+ - [76.94055944, -1.35441725, 0.00896096, -0.00001952, 172.36783217, -3.33144911, 0.02242308, -0.00004810, 267.90909091, -5.31620047, 0.03597727, -0.00007698]
+ - [85.83006993, -1.55480575, 0.01025932, -0.00002216, 160.73706294, -3.07291375, 0.02041900, -0.00004330, 237.37622378, -4.63444833, 0.03091317, -0.00006525]
+ - [105.02447552, -1.98017094, 0.01319522, -0.00002877, 135.50699301, -2.59097902, 0.01749301, -0.00003811, 165.33006993, -3.18872183, 0.02171387, -0.00004734]
+ - [105.02447552, -1.98017094, 0.01319522, -0.00002877, 135.50699301, -2.59097902, 0.01749301, -0.00003811, 165.33006993, -3.18872183, 0.02171387, -0.00004734]
+
+# Coefficients for band 8 land thresholds
+Coeffs_Band8_land_thresh:
+ - [282.74916084, -5.42869658, 0.03660781, -0.00008092, 344.87048951, -6.80660839, 0.04770163, -0.00010991, 407.83734266, -8.20392385, 0.05894114, -0.00013926]
+ - [229.50727273, -4.31606061, 0.02868182, -0.00006212, 316.38517483, -6.41910256, 0.04603089, -0.00010787, 403.78188811, -8.52743395, 0.06335781, -0.00015340]
+ - [239.32391608, -4.49928127, 0.02977214, -0.00006436, 270.31741259, -5.33625680, 0.03757168, -0.00008609, 301.76013986, -6.18557498, 0.04546562, -0.00010806]
diff --git a/unit_tests.py b/unit_tests.py
deleted file mode 100644
index 0e6b762..0000000
--- a/unit_tests.py
+++ /dev/null
@@ -1,97 +0,0 @@
-from glob import glob
-
-import ruamel_yaml as yml
-import numpy as np
-import xarray as xr
-
-from tests import CloudTests
-import read_data as rd
-import scene as scn
-
-# import pytest
-
-# #################################################################### #
-# TEST CASE
-
-# data:
-_datapath = '/ships19/hercules/pveglio/mvcm_viirs_hires'
-_fname_mod02 = glob(f'{_datapath}/VNP02MOD.A2022173.1454.001.*.uwssec_bowtie_restored.nc')[0]
-_fname_mod03 = glob(f'{_datapath}/VNP03MOD.A2022173.1454.001.*.uwssec.nc')[0]
-_fname_img02 = glob(f'{_datapath}/VNP02IMG.A2022173.1454.001.*.uwssec_bowtie_restored.nc')[0]
-_fname_img03 = glob(f'{_datapath}/VNP03IMG.A2022173.1454.001.*.uwssec.nc')[0]
-
-# thresholds:
-_threshold_file = '/home/pveglio/mvcm_leo/thresholds/new_thresholds.mvcm.snpp.v1.0.0.yaml'
-
-# ancillary files:
-_geos_atm_1 = 'GEOS.fpit.asm.inst3_2d_asm_Nx.GEOS5124.20220622_1200.V01.nc4'
-_geos_atm_2 = 'GEOS.fpit.asm.inst3_2d_asm_Nx.GEOS5124.20220622_1500.V01.nc4'
-_geos_land = 'GEOS.fpit.asm.tavg1_2d_lnd_Nx.GEOS5124.20220622_1430.V01.nc4'
-_geos_ocean = 'GEOS.fpit.asm.tavg1_2d_ocn_Nx.GEOS5124.20220622_1430.V01.nc4'
-_geos_constants = 'GEOS.fp.asm.const_2d_asm_Nx.00000000_0000.V01.nc4'
-_ndvi_file = 'NDVI.FM.c004.v2.0.WS.00-04.177.hdf'
-_sst_file = 'oisst.20220622'
-_eco_file = 'goge1_2_img.v1'
-
-# #################################################################### #
-
-
-def load_paths():
-
- file_names = {'MOD02': f'{_fname_mod02}',
- 'MOD03': f'{_fname_mod03}',
- 'IMG02': f'{_fname_img02}',
- 'IMG03': f'{_fname_img03}',
- 'GEOS_atm_1': f'{_geos_atm_1}',
- 'GEOS_atm_2': f'{_geos_atm_2}',
- 'GEOS_land': f'{_geos_land}',
- 'GEOS_ocean': f'{_geos_ocean}',
- 'GEOS_constants': f'{_geos_constants}',
- 'NDVI': f'{_ndvi_file}',
- 'SST': f'{_sst_file}',
- 'ANC_DIR': f'{_datapath}/ancillary'
- }
-
- return file_names
-
-
-def test_read_data(file_names):
-
- with open(_threshold_file) as f:
- text = f.read()
- thresholds = yml.safe_load(text)
-
- sunglint_angle = thresholds['Sun_Glint']['bounds'][3]
-
- viirs_xr = rd.get_data(file_names, sunglint_angle)
-
- return viirs_xr
-
-
-def test_scene(viirs_xr):
-
- scene_xr = xr.Dataset()
-
- for s in scn._scene_list:
- scene_xr[s] = (('number_of_lines', 'number_of_pixels'), scn.scene_id[s])
- scene_xr['latitude'] = viirs_xr.latitude
- scene_xr['longitude'] = viirs_xr.longitude
-
- viirs_data = xr.Dataset(viirs_xr, coords=scene_xr)
- viirs_data.drop_vars(['latitude', 'longitude'])
-
- return viirs_data
-
-
-def test_test(viirs_data):
-
- with open(_threshold_file) as f:
- text = f.read()
- thresholds = yml.safe_load(text)
-
- cmin = np.zeros(viirs_data.M01.shape)
-
- Ocean_Day = CloudTests(viirs_data, 'Ocean_Day', thresholds)
- cmin = Ocean_Day.single_threshold_test('11BT_Test', viirs_data.M15.values, cmin)
-
- return cmin
--
GitLab