initial commit...

modules/util/viirs_surfrad.py

@@ -101,14 +101,12 @@ def process_cld_opd_(grd_k):
     return grd_k
 
 
-def run_all(directory, out_directory, day_night='ANY', start=10):
+def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', start=10):
     cnt = start
     total_num_train_samples = 0
     total_num_valid_samples = 0
     num_keep_x_tiles = 8
 
-    pattern = 'clavrx_VNP02MOD*.highres.nc.level2.nc'
-    pattern = 'clavrx_*.nc'
     path = directory + '**' + '/' + pattern
 
     data_files = glob.glob(path, recursive=True)
@@ -120,7 +118,6 @@ def run_all(directory, out_directory, day_night='ANY', start=10):
     f_cnt = 0
 
     num_files = len(data_files)
-
     print('Start, number of files: ', num_files)
 
     for idx, data_f in enumerate(data_files):
@@ -138,13 +135,11 @@ def run_all(directory, out_directory, day_night='ANY', start=10):
             except Exception as e:
                 print(e)
                 h5f.close()
-                # label_h5f.close()
                 continue
             print(data_f)
             f_cnt += 1
 
             h5f.close()
-            # label_h5f.close()
 
             if len(data_train_tiles) == 0:
                 continue
@@ -152,16 +147,16 @@ def run_all(directory, out_directory, day_night='ANY', start=10):
             if (f_cnt % 5) == 0:
                 num_valid_samples = 0
                 if len(data_valid_tiles) > 0:
-                    # label_valid = np.stack(label_valid_tiles)
+                    label_valid = np.stack(label_valid_tiles)
                     data_valid = np.stack(data_valid_tiles)
                     np.save(out_directory + 'data_valid_' + str(cnt), data_valid)
-                    # np.save(out_directory+'label_valid_' + str(cnt), label_valid)
+                    np.save(out_directory + 'label_valid_' + str(cnt), label_valid)
                     num_valid_samples = data_valid.shape[0]
 
-                # label_train = np.stack(label_train_tiles)
-                # np.save(out_directory+'label_train_' + str(cnt), label_train)
+                label_train = np.stack(label_train_tiles)
                 data_train = np.stack(data_train_tiles)
-                np.save(out_directory+'data_train_' + str(cnt), data_train)
+                np.save(out_directory + 'label_train_' + str(cnt), label_train)
+                np.save(out_directory + 'data_train_' + str(cnt), data_train)
                 num_train_samples = data_train.shape[0]
 
                 label_valid_tiles = []