deal with bigger data

modules/util/viirs_l1b_l2.py

@@ -16,11 +16,14 @@ def run_all(directory):
     for p in os.scandir(directory):
         if not p.is_dir():
             continue
-        cnt += 1
+
         l1b_files = glob.glob(directory+p.name+'/'+'clavrx_snpp_viirs*.uwssec*.nc')
 
-        l1b_tiles = []
-        l2_tiles = []
+        label_valid_tiles = []
+        label_train_tiles = []
+        data_valid_tiles = []
+        data_train_tiles = []
+        f_cnt = 0
 
         for idx, l1b_f in enumerate(l1b_files):
             # if idx % 8 == 0:
@@ -44,6 +47,9 @@ def run_all(directory):
                 #     l1b_h5f.close()
                 #     continue
 
+                l1b_tiles = []
+                l2_tiles = []
+
                 try:
                     run(l1b_h5f, None, l1b_tiles, l2_tiles, factor=1)
                 except Exception as e:
@@ -55,14 +61,49 @@ def run_all(directory):
                 l1b_h5f.close()
                 # l2_h5f.close()
 
-        if len(l1b_tiles) == 0 or len(l2_tiles) == 0:
+                if len(l1b_tiles) == 0 or len(l2_tiles) == 0:
+                    continue
+
+                num = len(l1b_tiles)
+
+                [label_valid_tiles.append(l2_tiles[k]) for k in range(6)]
+                [label_train_tiles.append(l2_tiles[k]) for k in range(6, num)]
+                [data_valid_tiles.append(l1b_tiles[k]) for k in range(6)]
+                [data_train_tiles.append(l1b_tiles[k]) for k in range(6, num)]
+
+            if f_cnt == 10:
+                cnt += 1
+
+                label_valid = np.stack(label_valid_tiles)
+                label_train = np.stack(label_train_tiles)
+                data_valid = np.stack(data_valid_tiles)
+                data_train = np.stack(data_train_tiles)
+
+                np.save('/data/Personal/rink/viirs_clavrx/data_train_' + str(cnt), data_train)
+                np.save('/data/Personal/rink/viirs_clavrx/data_valid_' + str(cnt), data_valid)
+                np.save('/data/Personal/rink/viirs_clavrx/label_train_' + str(cnt), label_train)
+                np.save('/data/Personal/rink/viirs_clavrx/label_valid_' + str(cnt), label_valid)
+
+                label_valid_tiles = []
+                label_train_tiles = []
+                data_valid_tiles = []
+                data_train_tiles = []
+
+                f_cnt = 0
+
+        if len(label_train_tiles) == 0 or len(data_train_tiles) == 0:
             continue
 
-        l1b_nda = np.stack(l1b_tiles)
-        l2_nda = np.stack(l2_tiles)
+        label_valid = np.stack(label_valid_tiles)
+        label_train = np.stack(label_train_tiles)
+        data_valid = np.stack(data_valid_tiles)
+        data_train = np.stack(data_train_tiles)
 
-        np.save('/data/Personal/rink/viirs_clavrx/l1b_'+str(cnt), l1b_nda)
-        np.save('/data/Personal/rink/viirs_clavrx/l2_'+str(cnt), l2_nda)
+        cnt += 1
+        np.save('/data/Personal/rink/viirs_clavrx/data_train_' + str(cnt), data_train)
+        np.save('/data/Personal/rink/viirs_clavrx/data_valid_' + str(cnt), data_valid)
+        np.save('/data/Personal/rink/viirs_clavrx/label_train_' + str(cnt), label_train)
+        np.save('/data/Personal/rink/viirs_clavrx/label_valid_' + str(cnt), label_valid)
 
 
 def run(l1b_h5f, l2_h5f, l1b_tiles, l2_tiles, factor=2):