snapshot...

modules/deeplearning/cloud_fraction_fcn.py

 import glob
 import tensorflow as tf
 
-import util.util
+from util.plot_cm import confusion_matrix_values
 from util.setup import logdir, modeldir, now, ancillary_path
 from util.util import EarlyStop, normalize, denormalize, get_grid_values_all
 import os, datetime
@@ -811,74 +811,232 @@ def run_evaluate_static(in_file, out_file, ckpt_dir):
         return cld_frac_out, bt, refl_avg, cp
 
 
-def analyze2(nda_m, nda_i):
-    n_imgs = nda_m.shape[0]
-    nda_m = np.where(nda_m < 0.5, 0, 1)
-    nda_i = np.where(nda_i < 0.5, 0, 1)
+def analyze_3cat(file):
 
-    cf_m = np.zeros((n_imgs, 64, 64))
-    cf_i = np.zeros((n_imgs, 64, 64))
+    tup = np.load(file, allow_pickle=True)
+    lbls = tup[0]
+    pred = tup[1]
+    prob_0 = tup[2]
+    prob_1 = tup[3]
+    prob_2 = tup[4]
 
-    for k in range(n_imgs):
-        for j in range(1, 65):
-            for i in range(1, 65):
-                sub_3x3 = nda_m[k, j-1:j+2, i-1:i+2]
-                cf_m[k, j-1, i-1] = np.sum(sub_3x3)
+    lbls = lbls.flatten()
+    pred = pred.flatten()
+    print(np.sum(lbls == 0), np.sum(lbls == 1), np.sum(lbls == 2))
 
-                sub_4x4 = nda_i[k, j*2-1:j*2+3, i*2-1:i*2+3]
-                cf_i[k, j-1, i-1] = np.sum(sub_4x4)
+    msk_0_1 = lbls != 2
+    msk_1_2 = lbls != 0
+    msk_0_2 = lbls != 1
 
-    for k in range(n_imgs):
-        cat_0 = (cf_m[k, ] == 0)
-        cat_1 = (cf_m[k, ] > 0) & (cf_m[k, ] < 9)
-        cat_2 = cf_m[k, ] == 9
+    lbls_0_1 = lbls[msk_0_1]
 
-        cf_m[k, cat_0] = 0
-        cf_m[k, cat_1] = 1
-        cf_m[k, cat_2] = 2
+    pred_0_1 = pred[msk_0_1]
+    pred_0_1 = np.where(pred_0_1 == 2, 1, pred_0_1)
 
-        cat_0 = (cf_i[k, ] == 0)
-        cat_1 = (cf_i[k, ] > 0) & (cf_i[k, ] < 16)
-        cat_2 = cf_i[k, ] == 16
+    # ----
+    lbls_1_2 = lbls[msk_1_2]
+    lbls_1_2 = np.where(lbls_1_2 == 1, 0, lbls_1_2)
+    lbls_1_2 = np.where(lbls_1_2 == 2, 1, lbls_1_2)
 
-        cf_i[k, cat_0] = 0
-        cf_i[k, cat_1] = 1
-        cf_i[k, cat_2] = 2
+    pred_1_2 = pred[msk_1_2]
+    pred_1_2 = np.where(pred_1_2 == 0, -9, pred_1_2)
+    pred_1_2 = np.where(pred_1_2 == 1, 0, pred_1_2)
+    pred_1_2 = np.where(pred_1_2 == 2, 1, pred_1_2)
+    pred_1_2 = np.where(pred_1_2 == -9, 1, pred_1_2)
 
-    return cf_m, cf_i
+    # ----
+    lbls_0_2 = lbls[msk_0_2]
+    lbls_0_2 = np.where(lbls_0_2 == 2, 1, lbls_0_2)
 
+    pred_0_2 = pred[msk_0_2]
+    pred_0_2 = np.where(pred_0_2 == 2, 1, pred_0_2)
 
-def helper(lbls, pred, file='/Users/tomrink/clavrx_surfrad_viirs_cld_prob_valid.npy'):
-    nda = np.load(file, allow_pickle=True)
+    cm_0_1 = confusion_matrix_values(lbls_0_1, pred_0_1)
+    cm_1_2 = confusion_matrix_values(lbls_1_2, pred_1_2)
+    cm_0_2 = confusion_matrix_values(lbls_0_2, pred_0_2)
 
-    bt = nda[:, 0, :, :]
-    refl = nda[:, 1, :, :]
-    cp = nda[:, 2, :, :]
+    true_0_1 = (lbls_0_1 == 0) & (pred_0_1 == 0)
+    false_0_1 = (lbls_0_1 == 1) & (pred_0_1 == 0)
 
-    bt = get_grid_cell_mean(bt)
-    bt = bt[:, 0:66, 0:66]
+    true_no_0_1 = (lbls_0_1 == 1) & (pred_0_1 == 1)
+    false_no_0_1 = (lbls_0_1 == 0) & (pred_0_1 == 1)
 
-    lo, hi, std, avg = get_min_max_std(refl)
+    true_0_2 = (lbls_0_2 == 0) & (pred_0_2 == 0)
+    false_0_2 = (lbls_0_2 == 1) & (pred_0_2 == 0)
 
-    cp = np.where(np.isnan(cp), 0, cp)
-    cp = get_grid_cell_mean(cp)
-    cp = np.where(np.isnan(cp), 0, cp)
-    cp = cp[:, 1:65, 1:65]
-    cp = cp.flatten()
+    true_no_0_2 = (lbls_0_2 == 1) & (pred_0_2 == 1)
+    false_no_0_2 = (lbls_0_2 == 0) & (pred_0_2 == 1)
+
+    true_1_2 = (lbls_1_2 == 0) & (pred_1_2 == 0)
+    false_1_2 = (lbls_1_2 == 1) & (pred_1_2 == 0)
+
+    true_no_1_2 = (lbls_1_2 == 1) & (pred_1_2 == 1)
+    false_no_1_2 = (lbls_1_2 == 0) & (pred_1_2 == 1)
+
+    tp_0 = np.sum(true_0_1).astype(np.float64)
+    tp_1 = np.sum(true_1_2).astype(np.float64)
+    tp_2 = np.sum(true_0_2).astype(np.float64)
+
+    tn_0 = np.sum(true_no_0_1).astype(np.float64)
+    tn_1 = np.sum(true_no_1_2).astype(np.float64)
+    tn_2 = np.sum(true_no_0_2).astype(np.float64)
+
+    fp_0 = np.sum(false_0_1).astype(np.float64)
+    fp_1 = np.sum(false_1_2).astype(np.float64)
+    fp_2 = np.sum(false_0_2).astype(np.float64)
+
+    fn_0 = np.sum(false_no_0_1).astype(np.float64)
+    fn_1 = np.sum(false_no_1_2).astype(np.float64)
+    fn_2 = np.sum(false_no_0_2).astype(np.float64)
+
+    recall_0 = tp_0 / (tp_0 + fn_0)
+    recall_1 = tp_1 / (tp_1 + fn_1)
+    recall_2 = tp_2 / (tp_2 + fn_2)
+
+    precision_0 = tp_0 / (tp_0 + fp_0)
+    precision_1 = tp_1 / (tp_1 + fp_1)
+    precision_2 = tp_2 / (tp_2 + fp_2)
+
+    mcc_0 = ((tp_0 * tn_0) - (fp_0 * fn_0)) / np.sqrt((tp_0 + fp_0) * (tp_0 + fn_0) * (tn_0 + fp_0) * (tn_0 + fn_0))
+    mcc_1 = ((tp_1 * tn_1) - (fp_1 * fn_1)) / np.sqrt((tp_1 + fp_1) * (tp_1 + fn_1) * (tn_1 + fp_1) * (tn_1 + fn_1))
+    mcc_2 = ((tp_2 * tn_2) - (fp_2 * fn_2)) / np.sqrt((tp_2 + fp_2) * (tp_2 + fn_2) * (tn_2 + fp_2) * (tn_2 + fn_2))
+
+    acc_0 = np.sum(lbls_0_1 == pred_0_1)/pred_0_1.size
+    acc_1 = np.sum(lbls_1_2 == pred_1_2)/pred_1_2.size
+    acc_2 = np.sum(lbls_0_2 == pred_0_2)/pred_0_2.size
+
+    print(acc_0, recall_0, precision_0, mcc_0)
+    print(acc_1, recall_1, precision_1, mcc_1)
+    print(acc_2, recall_2, precision_2, mcc_2)
+
+    return cm_0_1, cm_1_2, cm_0_2, [acc_0, acc_1, acc_2], [recall_0, recall_1, recall_2],\
+        [precision_0, precision_1, precision_2], [mcc_0, mcc_1, mcc_2]
+
+
+def analyze_5cat(file='/Users/tomrink/cld_opd_frac.npy'):
+
+    tup = np.load(file, allow_pickle=True)
+    lbls = tup[0]
+    pred = tup[1]
+    # prob_0 = tup[2]
+    # prob_1 = tup[3]
+    # prob_2 = tup[4]
 
     lbls = lbls.flatten()
     pred = pred.flatten()
-    print(lbls.shape, pred.shape, cp.shape)
+    np.histogram(lbls, bins=5)
+    np.histogram(pred, bins=5)
+
+    new_lbls = np.zeros(lbls.size, dtype=np.int32)
+    new_pred = np.zeros(pred.size, dtype=np.int32)
+
+    new_lbls[lbls == 0] = 0
+    new_lbls[lbls == 1] = 1
+    new_lbls[lbls == 2] = 1
+    new_lbls[lbls == 3] = 1
+    new_lbls[lbls == 4] = 2
+
+    new_pred[pred == 0] = 0
+    new_pred[pred == 1] = 1
+    new_pred[pred == 2] = 1
+    new_pred[pred == 3] = 1
+    new_pred[pred == 4] = 2
+
+    np.histogram(new_lbls, bins=3)
+    np.histogram(new_pred, bins=3)
+
+    lbls = new_lbls
+    pred = new_pred
+
+    print(np.sum(lbls == 0), np.sum(lbls == 1), np.sum(lbls == 2))
+
+    msk_0_1 = lbls != 2
+    msk_1_2 = lbls != 0
+    msk_0_2 = lbls != 1
+
+    lbls_0_1 = lbls[msk_0_1]
+
+    pred_0_1 = pred[msk_0_1]
+    pred_0_1 = np.where(pred_0_1 == 2, 1, pred_0_1)
+
+    # ----
+    lbls_1_2 = lbls[msk_1_2]
+    lbls_1_2 = np.where(lbls_1_2 == 1, 0, lbls_1_2)
+    lbls_1_2 = np.where(lbls_1_2 == 2, 1, lbls_1_2)
+
+    pred_1_2 = pred[msk_1_2]
+    pred_1_2 = np.where(pred_1_2 == 0, -9, pred_1_2)
+    pred_1_2 = np.where(pred_1_2 == 1, 0, pred_1_2)
+    pred_1_2 = np.where(pred_1_2 == 2, 1, pred_1_2)
+    pred_1_2 = np.where(pred_1_2 == -9, 1, pred_1_2)
+
+    # ----
+    lbls_0_2 = lbls[msk_0_2]
+    lbls_0_2 = np.where(lbls_0_2 == 2, 1, lbls_0_2)
+
+    pred_0_2 = pred[msk_0_2]
+    pred_0_2 = np.where(pred_0_2 == 2, 1, pred_0_2)
+
+    cm_0_1 = confusion_matrix_values(lbls_0_1, pred_0_1)
+    cm_1_2 = confusion_matrix_values(lbls_1_2, pred_1_2)
+    cm_0_2 = confusion_matrix_values(lbls_0_2, pred_0_2)
+
+    true_0_1 = (lbls_0_1 == 0) & (pred_0_1 == 0)
+    false_0_1 = (lbls_0_1 == 1) & (pred_0_1 == 0)
+
+    true_no_0_1 = (lbls_0_1 == 1) & (pred_0_1 == 1)
+    false_no_0_1 = (lbls_0_1 == 0) & (pred_0_1 == 1)
+
+    true_0_2 = (lbls_0_2 == 0) & (pred_0_2 == 0)
+    false_0_2 = (lbls_0_2 == 1) & (pred_0_2 == 0)
+
+    true_no_0_2 = (lbls_0_2 == 1) & (pred_0_2 == 1)
+    false_no_0_2 = (lbls_0_2 == 0) & (pred_0_2 == 1)
+
+    true_1_2 = (lbls_1_2 == 0) & (pred_1_2 == 0)
+    false_1_2 = (lbls_1_2 == 1) & (pred_1_2 == 0)
+
+    true_no_1_2 = (lbls_1_2 == 1) & (pred_1_2 == 1)
+    false_no_1_2 = (lbls_1_2 == 0) & (pred_1_2 == 1)
+
+    tp_0 = np.sum(true_0_1).astype(np.float64)
+    tp_1 = np.sum(true_1_2).astype(np.float64)
+    tp_2 = np.sum(true_0_2).astype(np.float64)
+
+    tn_0 = np.sum(true_no_0_1).astype(np.float64)
+    tn_1 = np.sum(true_no_1_2).astype(np.float64)
+    tn_2 = np.sum(true_no_0_2).astype(np.float64)
+
+    fp_0 = np.sum(false_0_1).astype(np.float64)
+    fp_1 = np.sum(false_1_2).astype(np.float64)
+    fp_2 = np.sum(false_0_2).astype(np.float64)
+
+    fn_0 = np.sum(false_no_0_1).astype(np.float64)
+    fn_1 = np.sum(false_no_1_2).astype(np.float64)
+    fn_2 = np.sum(false_no_0_2).astype(np.float64)
+
+    recall_0 = tp_0 / (tp_0 + fn_0)
+    recall_1 = tp_1 / (tp_1 + fn_1)
+    recall_2 = tp_2 / (tp_2 + fn_2)
+
+    precision_0 = tp_0 / (tp_0 + fp_0)
+    precision_1 = tp_1 / (tp_1 + fp_1)
+    precision_2 = tp_2 / (tp_2 + fp_2)
+
+    mcc_0 = ((tp_0 * tn_0) - (fp_0 * fn_0)) / np.sqrt((tp_0 + fp_0) * (tp_0 + fn_0) * (tn_0 + fp_0) * (tn_0 + fn_0))
+    mcc_1 = ((tp_1 * tn_1) - (fp_1 * fn_1)) / np.sqrt((tp_1 + fp_1) * (tp_1 + fn_1) * (tn_1 + fp_1) * (tn_1 + fn_1))
+    mcc_2 = ((tp_2 * tn_2) - (fp_2 * fn_2)) / np.sqrt((tp_2 + fp_2) * (tp_2 + fn_2) * (tn_2 + fp_2) * (tn_2 + fn_2))
 
-    cp_cm = np.zeros((5, 5))
+    acc_0 = np.sum(lbls_0_1 == pred_0_1)/pred_0_1.size
+    acc_1 = np.sum(lbls_1_2 == pred_1_2)/pred_1_2.size
+    acc_2 = np.sum(lbls_0_2 == pred_0_2)/pred_0_2.size
 
-    for j in range(5):
-        for i in range(5):
-            keep = (lbls == j) & (pred == i)
-            cp_avg = np.sum(cp[keep])/ np.sum(keep)
-            cp_cm[j, i] = cp_avg
+    print(acc_0, recall_0, precision_0, mcc_0)
+    print(acc_1, recall_1, precision_1, mcc_1)
+    print(acc_2, recall_2, precision_2, mcc_2)
 
-    return cp_cm
+    return cm_0_1, cm_1_2, cm_0_2, [acc_0, acc_1, acc_2], [recall_0, recall_1, recall_2],\
+        [precision_0, precision_1, precision_2], [mcc_0, mcc_1, mcc_2], lbls, pred
 
 
 if __name__ == "__main__":