From 658192e77aa9717dbb44b72e88138b6da86c2b8e Mon Sep 17 00:00:00 2001
From: Geoff Cureton <geoff.cureton@ssec.wisc.edu>
Date: Mon, 9 Sep 2019 15:16:34 +0000
Subject: [PATCH] Added proper reporting of the VIIRS and CrIS input versions.

---
 source/flo/__init__.py | 100 ++++++++++++++++++++++++++++-------------
 1 file changed, 69 insertions(+), 31 deletions(-)

diff --git a/source/flo/__init__.py b/source/flo/__init__.py
index 4336822..6512aba 100644
--- a/source/flo/__init__.py
+++ b/source/flo/__init__.py
@@ -91,6 +91,8 @@ class CFConversionFailed(Exception):
     exit_code = 6003
 class CFConversionFailedToProductNC(Exception):
     exit_code = 6004
+class NCL2MetadataFailed(Exception):
+    exit_code = 6005
 
 class FUSION_MATLAB(Computation):
 
@@ -296,24 +298,50 @@ class FUSION_MATLAB(Computation):
             LOG.debug('VIIRS L1B granule {}: {} -> {}'.format(idx, vl1b_file.begin_time, vl1b_file.end_time))
             task.input('l1b_{}'.format(idx),  vl1b_file)
 
+    def get_cris_l1b_version(self, product, context):
+        '''
+        Return the correct CrIS version and input name depending on the target granule, as the
+        required version and input name may differ from that given in the product definition for a
+        given granule.
+        '''
+
+        satellite = context['satellite']
+        granule = context['granule']
+
+        input_name = 'CL1B'
+        version = product.input('cris_l1').version
+
+        if satellite == 'snpp':
+
+            while True:
+                # Use CL1B_nsr (2.0.15) before dt_switch_nsr
+                dt_switch_nsr = datetime(2015, 12, 1, 0)
+                if (granule - dt_switch_nsr).total_seconds() <= 0.:
+                    input_name = 'CL1B_nsr'
+                    version = '2.0.15'
+                    break
+
+                # Use CL1B (2.0.15) before dt_switch_side2
+                dt_switch_side2 = datetime(2019, 6, 24, 18, 48)
+                if (granule - dt_switch_side2).total_seconds() <= 0.:
+                    input_name = 'CL1B'
+                    version = '2.0.15'
+                    break
+
+                # We're after the special cases, use what's in the product definition
+                break
+
+        return input_name, version
+
     def _add_cris_l1b_input(self, product, context, task):
         satellite = context['satellite']
         granule = context['granule']
         granule_length = timedelta(minutes=6)
 
-        cris_l1 = product.input('cris_l1')
-
         interval = TimeInterval(granule-granule_length, granule+granule_length)
 
-        version = cris_l1.version
-
-        # SNPP CrIS hardware switch-over requires two different cris_l1 versions
-        if satellite == 'snpp':
-            dt_switch = datetime(2019, 6, 24)
-            if (granule - dt_switch).total_seconds() <= 0.:
-                version = '2.0.15'
+        input_name, version = self.get_cris_l1b_version(product, context)
 
-        input_name = 'CL1B'
         LOG.debug("Ingesting input {} ({}) for FSNRAD_L2_VIIRS_CRIS version {}".format(input_name, version, product.version))
         cris = dawg_catalog.files(satellite, input_name, interval, version=version)
 
@@ -376,7 +404,7 @@ class FUSION_MATLAB(Computation):
         cmd = '{} {} {} {}'.format(py_exe, viirsmend_exe, out_fn, geo)
 
         dummy_bowtie_rest_file = sorted(glob(pjoin('/data/geoffc/fusion_matlab/work/local_processing',
-                                                   'snpp_test_case/outputs/tmptkFpCs',
+                                                   'snpp_fusion_output/outputs',
                                                    'VNP02MOD*bowtie*.nc')))
 
         if not dummy:
@@ -388,7 +416,9 @@ class FUSION_MATLAB(Computation):
 
         return out_fn
 
-    def cris_viirs_collocation(self, product, geo, sounder, dummy=False):
+    def cris_viirs_collocation(self, product, context, geo, sounder, dummy=False):
+
+        satellite = context['satellite']
 
         LOG.debug("geo = \n\t{}".format('\n\t'.join(geo)))
         LOG.debug("sounder = \n\t{}".format('\n\t'.join(sounder)))
@@ -398,13 +428,16 @@ class FUSION_MATLAB(Computation):
         crisviirs_exe = pjoin(crisviirs.path,'bin/crisviirs')
 
         dummy_collo_file = sorted(glob(pjoin('/data/geoffc/fusion_matlab/work/local_processing',
-                                                   'snpp_test_case/outputs/tmptkFpCs',
+                                                   'snpp_fusion_output/outputs',
                                                    'colloc.cris_snpp.viirs_m_snpp.*.nc')))
 
         if not dummy:
             for cris, vgeo in zip(sounder, geo):
+                collo_dt = datetime.strptime(basename(cris).split('.')[3],'%Y%m%dT%H%M')
+                collo_log = 'colloc.cris_{0:}.viirs_m_{0:}.{1:%Y%m%dT%H%M%S}_{1:%H%M%S}'.format(
+                        satellite, collo_dt)
                 #cmd = '{} {} {} > /dev/null'.format(crisviirs_exe, cris, vgeo)
-                cmd = '{} {} {} > {}.log'.format(crisviirs_exe, cris, vgeo, basename(cris))
+                cmd = '{} {} {} > {}.log'.format(crisviirs_exe, cris, vgeo, collo_log)
 
                 runscript(cmd, requirements=[])
         else:
@@ -466,8 +499,7 @@ class FUSION_MATLAB(Computation):
         dummy = kwargs['dummy']
         if dummy:
             dummy_matlab_file = pjoin('/data/geoffc/fusion_matlab/work/local_processing',
-                                                    'snpp_fusion_output',
-                                                    'fusion_output.mat')
+                                                    'snpp_fusion_output/outputs/fusion_output.mat')
 
         rc_fusion = 0
 
@@ -506,7 +538,6 @@ class FUSION_MATLAB(Computation):
             rc_fusion = err.returncode
             LOG.error("Matlab binary {} returned a value of {}".format(fusion_binary, rc_fusion))
             raise FusionProcessFailed('Matlab binary {} failed or was killed, returning a value of {}'.format(fusion_binary, rc_fusion))
-            #return rc_fusion, None
 
         # Move matlab file to the fused outputs directory
         matlab_file = glob(matlab_file_glob)
@@ -521,8 +552,6 @@ class FUSION_MATLAB(Computation):
         else:
             LOG.error('There are no Matlab files "{}" to convert, aborting'.format(matlab_file_glob))
             raise FusionFailedToProduceMat('Matlab binary {} failed to produce the file "{}"'.format(fusion_binary, matlab_file_glob))
-            #rc_fusion = 1
-            #return rc_fusion, None
 
         return rc_fusion, matlab_file
 
@@ -608,7 +637,7 @@ class FUSION_MATLAB(Computation):
         else:
 
             if satellite=='snpp':
-                dummy_fusion_file = '/data/geoffc/fusion_matlab/work/local_processing/snpp_fusion_output/FSNRAD_L2_VIIRS_CRIS_SNPP.A2018283.1206.001.2019218152028.nc'
+                dummy_fusion_file = '/data/geoffc/fusion_matlab/work/local_processing/snpp_fusion_output/FSNRAD_L2_VIIRS_CRIS_SNPP.A2019218.1824.001.2019220192142.nc'
             if satellite=='noaa20':
                 dummy_fusion_file = '/data/geoffc/fusion_matlab/work/local_processing/snpp_fusion_output/VNP02FSN.A2018033.1836.001.2018058173216.nc'
 
@@ -681,14 +710,9 @@ class FUSION_MATLAB(Computation):
         '''
         Construct various metadata strings and write them to the level2 file
         '''
+        LOG.debug('Updating the metadata in {}'.format(l2_file))
 
-        #context = {}
-        #context['satellite'] = satellite
         context['nrt'] = True
-        #context['collection'] = 0
-
-        #product = sipsprod.lookup_product_recurse('FSNRAD_L2_VIIRS_CRIS', version=version)
-        #product.options['collection'] = context['collection']
 
         satname = 'SNPP' if context['satellite']=='snpp' else 'NOAA20'
         esdt = product.name + ('_SNPP' if context['satellite']=='snpp' else '_NOAA20')
@@ -698,7 +722,9 @@ class FUSION_MATLAB(Computation):
         viirs_input_fns, viirs_lut_version, viirs_lut_created = get_viirs_l1_luts(viirs_l1b_file, geo_fn=viirs_geo_file)
         ancillary_fns = []
         viirs_l1_version = product.input('viirs_l1').version
-        cris_l1_version = product.input('cris_l1').version
+
+        # Make any input version mods that aren't in the product definition...
+        cris_input_name, cris_l1_version = self.get_cris_l1b_version(product, context)
 
         set_official_product_metadata(
             esdt,
@@ -726,16 +752,26 @@ class FUSION_MATLAB(Computation):
             nc_l2.delncattr('l1_lut_version')
             nc_l2.delncattr('l1_lut_created')
             nc_l2.delncattr('input_files')
-        except:
-            pass
+            source = nc_l2.getncattr('source')
+            nc_l2.delncattr('source')
+        except Exception as err:
+            LOG.warning('There was a problem removing nc_l2 attrs in {}'.format(l2_file))
+            LOG.debug(traceback.format_exc())
+            nc_l2.close()
+            raise NCL2MetadataFailed('NetCDF4 global metadata update failed')
 
         nc_l2.close()
 
         nc_remove_unlimited_dims(basename(l2_file))
 
+        cris_l1_version += ' (NSR)' if 'nsr' in cris_input_name else ''
+        source = ', '.join(
+                ['cris_l1 {}'.format(cris_l1_version) if 'cris_l1' in x else x for x in source.split(',')]
+                ).replace('  ',' ')
+
         FIX_ATTRS = [
             ('title', '{0:} VIIRS+CrIS Fusion ({1:})'.format(satname, esdt)),
-            ('platform', {'snpp':'Suomi NPP', 'noaa20':'NOAA-20'}[context['satellite']]),
+            ('platform', {'snpp':'SUOMI-NPP', 'noaa20':'NOAA-20'}[context['satellite']]),
             ('instrument', 'VIIRS+CrIS'),
             ('conventions', 'CF-1.6, ACDD-1.3'),
             ('AlgorithmType', 'OPS'),
@@ -746,6 +782,7 @@ class FUSION_MATLAB(Computation):
             ('viirs_l1_version', viirs_l1_version),
             ('viirs_lut_version', viirs_lut_version),
             ('viirs_lut_created', str(viirs_lut_created.isoformat())),
+            ('source', source),
             ('cris_l1_version', cris_l1_version),
             ('input_files', ', '.join([basename(x) for x in [viirs_geo_file, viirs_l1b_file, cris_l1b_file]])),
             ('xmlmetadata', xmlmetadata),
@@ -858,7 +895,7 @@ class FUSION_MATLAB(Computation):
                     # Mend bowtie pixels of NASA VIIRS l1b file...
                     l1b[idx] = self.mend_viirs_l1b(product, geo[idx], l1b[idx], dummy=dummy)
 
-            collo = self.cris_viirs_collocation(product, geo, sounder, dummy=dummy)
+            collo = self.cris_viirs_collocation(product, context, geo, sounder, dummy=dummy)
 
         # Generate the AIRS/MODIS collocation
         if satellite == 'aqua':
@@ -928,6 +965,7 @@ class FUSION_MATLAB(Computation):
             raise RuntimeError('Output fusion file fusion_output.mat not created.')
 
         #kwargs['dummy'] = True # dummy
+        #kwargs['dummy'] = False # dummy
 
         # Now that we've computed the Matlab file, convert to a NetCDF file...
         if satellite=='snpp' or satellite=='noaa20':
-- 
GitLab