snapshot...

modules/deeplearning/icing.py

@@ -4,14 +4,12 @@ import subprocess
 
 import os, datetime
 import numpy as np
-import xarray as xr
 import pickle
 import h5py
 
 from deeplearning.amv_raob import get_bounding_gfs_files, convert_file, get_images, get_interpolated_profile, get_time_tuple_utc, get_profile
 
-from icing.pirep_goes import split_data
-from icing.pirep_goes import train_params_day
+from icing.pirep_goes import split_data, normalize
 
 LOG_DEVICE_PLACEMENT = False
 
@@ -49,8 +47,10 @@ img_width = 16
 #img_width = 12
 #img_width = 6
 
-NUM_VERT_LEVELS = 26
-NUM_VERT_PARAMS = 2
+
+train_params_day = ['cld_height_acha', 'cld_geo_thick', 'supercooled_cloud_fraction', 'cld_temp_acha', 'cld_press_acha',
+                    'cld_reff_dcomp', 'cld_opd_dcomp', 'cld_cwp_dcomp', 'iwc_dcomp', 'lwc_dcomp']
+                    #'cloud_phase']
 
 
 def build_residual_block(input, drop_rate, num_neurons, activation, block_name, doDropout=True, doBatchNorm=True):
@@ -116,6 +116,7 @@ class IcingIntensityNN:
         self.in_mem_batch = None
         self.filename = None
         self.h5f = None
+        self.h5f_l1b = None
 
         self.logits = None
 
@@ -164,7 +165,7 @@ class IcingIntensityNN:
             n_chans *= 3
         self.X_img = tf.keras.Input(shape=(img_width, img_width, n_chans))
         #self.X_img = tf.keras.Input(shape=NUM_PARAMS)
-        self.X_prof = tf.keras.Input(shape=(NUM_VERT_LEVELS, NUM_VERT_PARAMS))
+        #self.X_prof = tf.keras.Input(shape=(NUM_VERT_LEVELS, NUM_VERT_PARAMS))
         self.X_sfc = tf.keras.Input(shape=2)
 
         self.inputs.append(self.X_img)
@@ -201,7 +202,7 @@ class IcingIntensityNN:
         data = []
         for param in train_params_day:
             nda = self.h5f[param][nd_keys, ]
-            # nda = do_normalize(nda)
+            # nda = normalize(nda, param)
             data.append(nda)
         data = np.stack(data)
         data = data.astype(np.float32)
@@ -224,7 +225,6 @@ class IcingIntensityNN:
         #             label.append(tup[2])
         #             continue
         #
-        #
         #     if CACHE_DATA_IN_MEM:
         #         self.in_mem_data_cache[key] = (nda, ndb, ndc)
 
@@ -576,7 +576,7 @@ class IcingIntensityNN:
                 self.predict(mini_batch_test)
         print('loss, acc: ', self.test_loss.result(), self.test_accuracy.result())
 
-    def run(self, filename, train_dict=None, valid_dict=None):
+    def run(self, filename, filename_l1b=None, train_dict=None, valid_dict=None):
         with tf.device('/device:GPU:'+str(self.gpu_device)):
             self.setup_pipeline(filename, train_idxs=train_dict, test_idxs=valid_dict)
             self.build_model()