snapshot...

modules/util/viirs_l1b_l2.py

@@ -9,10 +9,13 @@ emis_params = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13
                'temp_6_7um_nom']
 l2_params = ['cloud_fraction', 'cld_temp_acha', 'cld_press_acha']
 
+data_params = emis_params
+label_params = l2_params
+
 
 def run_all(directory):
 
-    cnt = 0
+    cnt = 10
     for p in os.scandir(directory):
         if not p.is_dir():
             continue
@@ -47,11 +50,11 @@ def run_all(directory):
                 #     l1b_h5f.close()
                 #     continue
 
-                l1b_tiles = []
-                l2_tiles = []
+                data_tiles = []
+                label_tiles = []
 
                 try:
-                    run(l1b_h5f, None, l1b_tiles, l2_tiles, factor=1)
+                    run(l1b_h5f, None, data_tiles, label_tiles, factor=1)
                 except Exception as e:
                     print(e)
                     l1b_h5f.close()
@@ -61,15 +64,19 @@ def run_all(directory):
                 l1b_h5f.close()
                 # l2_h5f.close()
 
-                if len(l1b_tiles) == 0 or len(l2_tiles) == 0:
+                if len(data_tiles) == 0 or len(label_tiles) == 0:
+                    continue
+
+                if len(data_tiles) != len(label_tiles):
+                    print('weirdness: ', l1b_f)
                     continue
 
-                num = len(l1b_tiles)
+                num = len(data_tiles)
 
-                [label_valid_tiles.append(l2_tiles[k]) for k in range(10)]
-                [label_train_tiles.append(l2_tiles[k]) for k in range(10, num)]
-                [data_valid_tiles.append(l1b_tiles[k]) for k in range(10)]
-                [data_train_tiles.append(l1b_tiles[k]) for k in range(10, num)]
+                [label_valid_tiles.append(label_tiles[k]) for k in range(10)]
+                [label_train_tiles.append(label_tiles[k]) for k in range(10, num)]
+                [data_valid_tiles.append(data_tiles[k]) for k in range(10)]
+                [data_train_tiles.append(data_tiles[k]) for k in range(10, num)]
 
             if f_cnt == 10:
                 cnt += 1
@@ -95,6 +102,9 @@ def run_all(directory):
 
         if len(label_train_tiles) == 0 or len(data_train_tiles) == 0:
             continue
+        if len(label_train_tiles) != len(data_train_tiles):
+            print('weirdness')
+            continue
 
         label_valid = np.stack(label_valid_tiles)
         label_train = np.stack(label_train_tiles)
@@ -147,9 +157,9 @@ def run(l1b_h5f, l2_h5f, l1b_tiles, l2_tiles, factor=2):
     mod_data = np.stack(l1b_grd_s)
     img_data = np.stack(l2_grd_s)
 
-    num_keep_x_tiles = 5
+    num_keep_x_tiles = 3
     i_skip = 3 * mod_tile_width
-    i_start = int(mod_num_pixels / 2) - int((num_keep_x_tiles * 3 * 64) / 2)
+    i_start = int(mod_num_pixels / 2) - int((num_keep_x_tiles * 3 * mod_tile_width) / 2)
 
     num_keep_y_tiles = 16
     j_skip = 3 * mod_tile_width
@@ -170,27 +180,6 @@ def run(l1b_h5f, l2_h5f, l1b_tiles, l2_tiles, factor=2):
             l2_tiles.append(nda)
 
 
-# def hack():
-#     l1b_nda = np.load('/Users/tomrink/data/viirs_clavrx/l1b_all.npy')
-#     l2_nda = np.load('/Users/tomrink/data/viirs_clavrx/l2_all.npy')
-#
-#     num_samples = l1b_nda.shape[0]
-#     idxs = np.arange(num_samples)
-#     vld_idxs = idxs[::10]
-#     trn_idxs = idxs[np.in1d(idxs, vld_idxs, invert=True)]
-#
-#     l1b_trn = l1b_nda[trn_idxs]
-#     l2_trn = l2_nda[trn_idxs]
-#     l1b_vld = l1b_nda[vld_idxs]
-#     l2_vld = l2_nda[vld_idxs]
-#
-#     print(l1b_trn.shape, l2_trn.shape, l1b_vld.shape, l2_vld.shape)
-#
-#     np.save('/Users/tomrink/data/viirs_clavrx/l1b_all_trn.npy', l1b_trn)
-#     np.save('/Users/tomrink/data/viirs_clavrx/l1b_all_vld.npy', l1b_vld)
-#     np.save('/Users/tomrink/data/viirs_clavrx/l2_all_trn.npy', l2_trn)
-#     np.save('/Users/tomrink/data/viirs_clavrx/l2_all_vld.npy', l2_vld)
-
 # def run_mean_std(directory):
 #
 #     data_dct = {name: [] for name in mod_res_params}