From 10c2a779ec57ef20170a30bdfbec5f73b2f92ce4 Mon Sep 17 00:00:00 2001 From: Geoff Cureton <geoff.cureton@ssec.wisc.edu> Date: Wed, 14 Aug 2019 22:58:22 +0000 Subject: [PATCH] Now can handle different product versions on a per-satellite basis. Uses updated CF-converter package which now generates a candidate fusion nc4 files from a CDL file. --- example_local_prepare.py | 6 +- source/flo/__init__.py | 423 +++++++++++++++++++++------------------ 2 files changed, 230 insertions(+), 199 deletions(-) diff --git a/example_local_prepare.py b/example_local_prepare.py index 619095a..7c02a95 100644 --- a/example_local_prepare.py +++ b/example_local_prepare.py @@ -21,10 +21,8 @@ comp = FUSION_MATLAB() #granule = datetime(2015, 4, 17, 14) #interval = TimeInterval(granule, granule+timedelta(minutes=0)) -satellite = 'snpp' -#satellite = 'noaa20' -#satellite = 'aqua' -version = '1.0dev10' # base VIIRS level-1b +satellite, version = 'snpp', '1.0.0dev1' +satellite, version = 'noaa20', '1.0.1dev1' def local_execute_example(interval, satellite, version, skip_prepare=False, skip_execute=False, verbosity=2): diff --git a/source/flo/__init__.py b/source/flo/__init__.py index 92c0fd0..4b2b28f 100644 --- a/source/flo/__init__.py +++ b/source/flo/__init__.py @@ -53,7 +53,7 @@ import traceback from glob import glob from itertools import chain from subprocess import check_output, check_call, CalledProcessError -from netCDF4 import Dataset +import netCDF4 from pyhdf.SD import SD, SDC from flo.computation import Computation @@ -70,13 +70,12 @@ from glutil import ( runscript, get_viirs_l1_luts, #prepare_env, - #nc_gen, - nc_compress, hdf_compress, reraise_as, set_official_product_metadata, FileNotFound ) +from glutil.nc import (nc_gen, nc_compress, nc_remove_unlimited_dims, nc_setattrs, nc_getattrs) from utils import create_dir from detect_bad_fusion import SFX, detect @@ -225,7 +224,7 @@ class FUSION_MATLAB(Computation): input_name = sipsprod.satellite_esdt('V03MOD', satellite) - LOG.debug("Ingesting input {} ({}) for V02FSN version {}".format(input_name, version, product.version)) + LOG.debug("Ingesting input {} ({}) for FSNRAD_L2_VIIRS_CRIS version {}".format(input_name, version, product.version)) vgeom = dawg_catalog.files(satellite, input_name, interval, version=version) if vgeom == []: @@ -256,7 +255,7 @@ class FUSION_MATLAB(Computation): if viirs_l1: version = viirs_l1.version input_name = sipsprod.satellite_esdt('V02MOD', satellite) - LOG.debug("Ingesting input {} ({}) for V02FSN version {}".format(input_name, version, product.version)) + LOG.debug("Ingesting input {} ({}) for FSNRAD_L2_VIIRS_CRIS version {}".format(input_name, version, product.version)) vl1b = dawg_catalog.files(satellite, input_name, interval, version=version) if vl1b == []: raise WorkflowNotReady('FUSION_MATLAB: Missing {} inputs for version {} and interval {}'.format( @@ -266,7 +265,7 @@ class FUSION_MATLAB(Computation): elif v02mod_bt_sc: version = v02mod_bt_sc.version input_name = sipsprod.satellite_esdt('V02MOD-bt-sc', satellite) - LOG.debug("Ingesting input {} ({}) for V02FSN version {}".format(input_name, version, product.version)) + LOG.debug("Ingesting input {} ({}) for FSNRAD_L2_VIIRS_CRIS version {}".format(input_name, version, product.version)) from flo.sw.v02mod_bt_sc import v02mod_bt_sc vl1b = v02mod_bt_sc().dataset('out').product( {'granule': context['granule'], 'satellite': context['satellite'], @@ -298,8 +297,15 @@ class FUSION_MATLAB(Computation): interval = TimeInterval(granule-granule_length, granule+granule_length) version = cris_l1.version + + # SNPP CrIS hardware switch-over requires two different cris_l1 versions + if satellite == 'snpp': + dt_switch = datetime(2019, 6, 24) + if (granule - dt_switch).total_seconds() <= 0.: + version = '2.0.15' + input_name = 'CL1B' - LOG.debug("Ingesting input {} ({}) for V02FSN version {}".format(input_name, version, product.version)) + LOG.debug("Ingesting input {} ({}) for FSNRAD_L2_VIIRS_CRIS version {}".format(input_name, version, product.version)) cris = dawg_catalog.files(satellite, input_name, interval, version=version) if cris == []: @@ -320,11 +326,11 @@ class FUSION_MATLAB(Computation): ''' Build up a set of inputs for a single context ''' - LOG.debug("Ingesting inputs for V02FSN version {} ...".format(context['version'])) + LOG.debug("Ingesting inputs for FSNRAD_L2_VIIRS_CRIS version {} ...".format(context['version'])) - # Get the product definition for 'V02FSN'. Different versions may use regular or bias + # Get the product definition for 'FSNRAD_L2_VIIRS_CRIS'. Different versions may use regular or bias # corrected VIIRS level-1b files. - product_name = 'V02FSN' + product_name = 'FSNRAD_L2_VIIRS_CRIS' product = sipsprod.lookup_product_recurse(product_name, version=context['version']) # Ingest the required inputs, defined in the VNP02 product definition for context['version'] @@ -335,7 +341,7 @@ class FUSION_MATLAB(Computation): # Make the product definition available to build_task() task.option('product', product) - @reraise_as(WorkflowNotReady, FileNotFound, prefix='V02FSN') + @reraise_as(WorkflowNotReady, FileNotFound, prefix='FSNRAD_L2_VIIRS_CRIS') def build_task(self, context, task): satellite = context['satellite'] @@ -389,6 +395,7 @@ class FUSION_MATLAB(Computation): if not dummy: for cris, vgeo in zip(sounder, geo): cmd = '{} {} {} > /dev/null'.format(crisviirs_exe, cris, vgeo) + #cmd = '{} {} {} > {}.log'.format(crisviirs_exe, cris, vgeo, basename(cris)) # DEBUG runscript(cmd, requirements=[]) else: @@ -450,17 +457,18 @@ class FUSION_MATLAB(Computation): dummy = kwargs['dummy'] if dummy: dummy_matlab_file = pjoin('/data/geoffc/fusion_matlab/work/local_processing', - 'snpp_test_case/outputs/tmptkFpCs', - 'fused_outputs/fusion_output.mat') + 'snpp_fusion_output', + 'fusion_output.mat') rc_fusion = 0 # Get the matlab runtim version that we require - matlab_version = '2015b' + #matlab_version = '2015b' + matlab_version = '2018b' #matlab_version = product.input('fusion_matlab').options['matlab_version'] #run matlab - cmd = '{}/{} {} {} {} {} {} {}'.format( + cmd = '{}/{} {}/ {} {} {} {} {}'.format( bin_dir, fusion_binary, support_software.lookup('matlab', matlab_version).path, @@ -507,14 +515,14 @@ class FUSION_MATLAB(Computation): return rc_fusion, matlab_file - - def convert_matlab_to_netcdf(self, product, matlab_file, l1b_file, **kwargs): + def convert_matlab_to_viirs_netcdf(self, product, viirs_l1b_file, cris_l1b_file, matlab_file, **kwargs): ''' - Transcode the Matlab *.mat file into a CF-compliant HDF4 (aqua) or NetCDF4 (snpp) file. + Transcode the Matlab *.mat file into a CF-compliant NetCDF4 file. ''' py_interp = kwargs['py_interp'] bin_dir = kwargs['bin_dir'] + cdl_dir = kwargs['cdl_dir'] fused_output_dir = kwargs['fused_output_dir'] conversion_bin = kwargs['conversion_bin'] env = kwargs['env'] @@ -523,14 +531,6 @@ class FUSION_MATLAB(Computation): dummy = kwargs['dummy'] - if dummy: - if satellite=='snpp': - dummy_fusion_file = '/mnt/sdata/geoffc/fusion_matlab/work/snpp_temp_outputs/outputs/tmp5PwThb/VNP02FSN.A2018033.1836.001.2018058173216.nc' - if satellite=='noaa20': - dummy_fusion_file = '/mnt/sdata/geoffc/fusion_matlab/work/snpp_temp_outputs/outputs/tmp5PwThb/VNP02FSN.A2018033.1836.001.2018058173216.nc' - if satellite=='aqua': - dummy_fusion_file = '/mnt/sdata/geoffc/fusion_matlab/work/aqua_temp_outputs/outputs/tmpMUoHF3/MYD02FSN.A2015107.1755.006.2018058170733.hdf' - rc_fusion = 0 dt_create = datetime.utcnow() @@ -539,98 +539,106 @@ class FUSION_MATLAB(Computation): LOG.debug('tmp_work_dir (CWD): "{}"'.format(tmp_work_dir)) LOG.debug('fused_output_dir: "{}"'.format(fused_output_dir)) - # Copy the un-fused level-1b file to the work directory as a template... - unfused_l1b_file = pjoin(tmp_work_dir, 'unfused', basename(l1b_file)) - unfused_l1b_dir = dirname(unfused_l1b_file) - create_dir(unfused_l1b_dir) - - if exists(unfused_l1b_file): - LOG.debug('{} exists, removing...'.format(unfused_l1b_file)) - os.remove(unfused_l1b_file) - - LOG.debug('Copying {} to {}'.format(l1b_file, unfused_l1b_file)) - shutil.copy(l1b_file, unfused_l1b_file) - - # Removing the fused file if it exists - fused_l1b_file = pjoin(fused_output_dir, basename(unfused_l1b_file)) - LOG.debug('Checking for existing fused output file "{}"'.format(fused_l1b_file)) - if exists(fused_l1b_file): - LOG.debug('{} exists, removing...'.format(fused_l1b_file)) - os.remove(fused_l1b_file) - - # Convert the Matlab file to the desired format... - cmd = '{} {} {} {} {}'.format( - py_interp, - conversion_bin, - unfused_l1b_file, - matlab_file, - fused_output_dir - ) - try: - LOG.debug("cmd = \\\n\t{}".format(cmd.replace(' ',' \\\n\t'))) - rc_fusion = 0 - if not dummy: + # Construct the candidate filename for the new level-2 file + esdt = 'FSNRAD_L2_VIIRS_CRIS' + ('_SNPP' if satellite=='snpp' else '_NOAA20') + collection = product.options['collection'] + viirs_fused_l1b_file = sipsprod.product_filename(esdt, collection, granule, created=dt_create) + viirs_fused_l1b_file = pjoin(fused_output_dir, basename(viirs_fused_l1b_file)) + + # Remove the output level-2 template file if it exists + LOG.debug('Checking for existing fused output file "{}"'.format(viirs_fused_l1b_file)) + if exists(viirs_fused_l1b_file): + LOG.debug('{} exists, removing...'.format(viirs_fused_l1b_file)) + os.remove(viirs_fused_l1b_file) + + if not dummy: + # Create the new level2 template file and add a dimension from the level1 file + cdl_file = pjoin(cdl_dir, 'fusion_cris_viirs.cdl') + LOG.info('Creating template file {} from CDL file {}'.format(viirs_fused_l1b_file, cdl_file)) + + try: + nc_gen(cdl_file, viirs_fused_l1b_file) + except CalledProcessError as err: + rc = err.returncode + LOG.error("ncgen returned a value of {}".format(rc)) + return rc, [] + + # Open the l1b file and get the 'number_of_LUT_values' dimension + nc_l1b = netCDF4.Dataset(viirs_l1b_file, 'r') + dim = nc_l1b.dimensions['number_of_LUT_values'] + + # Copy the 'number_of_LUT_values' dim to the level-2 template file + nc_l2 = netCDF4.Dataset(viirs_fused_l1b_file, 'r+') + nc_l2.createDimension(dim.name, size=dim.size) + + # Close the level1 and level2 files + nc_l2.close() + nc_l1b.close() + + # Copy the Matlab file data to the level-2 tamplate file... + cmd = '{} {} {} {} {} {}'.format( + py_interp, + conversion_bin, + viirs_l1b_file, + cris_l1b_file, + matlab_file, + viirs_fused_l1b_file) + + try: + LOG.debug("cmd = \\\n\t{}".format(cmd.replace(' ',' \\\n\t'))) + rc_fusion = 0 runscript(cmd, requirements=[], env=env) - else: - LOG.debug('dummy cmd = "cp {} {}"'.format(dummy_fusion_file, - pjoin(fused_output_dir, basename(l1b_file)))) - shutil.copy(dummy_fusion_file, pjoin(fused_output_dir, basename(l1b_file))) - except CalledProcessError as err: - rc_fusion = err.returncode - LOG.error("CF converter {} returned a value of {}".format(conversion_bin, rc_fusion)) - return rc_fusion, None, {} + except CalledProcessError as err: + rc_fusion = err.returncode + LOG.error("CF converter {} returned a value of {}".format(conversion_bin, rc_fusion)) + return rc_fusion, None, {} + + else: + + if satellite=='snpp': + dummy_fusion_file = '/data/geoffc/fusion_matlab/work/local_processing/snpp_fusion_output/FSNRAD_L2_VIIRS_CRIS_SNPP.A2018283.1206.001.2019218152028.nc' + if satellite=='noaa20': + dummy_fusion_file = '/data/geoffc/fusion_matlab/work/local_processing/snpp_fusion_output/VNP02FSN.A2018033.1836.001.2018058173216.nc' + + LOG.debug('dummy cmd = "cp {} {}"'.format(dummy_fusion_file, fused_output_dir)) + shutil.copy(dummy_fusion_file, fused_output_dir) + viirs_fused_l1b_file = pjoin(fused_output_dir, basename(dummy_fusion_file)) + # Determine success... - LOG.debug('Looking for fused output file "{}"'.format(fused_l1b_file)) - fused_l1b_file = glob(fused_l1b_file) - if len(fused_l1b_file) != 0: - fused_l1b_file = fused_l1b_file[0] - LOG.debug('Found final fused output file "{}"'.format(fused_l1b_file)) + LOG.debug('Looking for fused output file "{}"'.format(viirs_fused_l1b_file)) + viirs_fused_l1b_file = glob(viirs_fused_l1b_file) + if len(viirs_fused_l1b_file) != 0: + viirs_fused_l1b_file = viirs_fused_l1b_file[0] + LOG.debug('Found final fused output file "{}"'.format(viirs_fused_l1b_file)) else: - LOG.error('There is no fused file {}, aborting'.format(fused_l1b_file)) + LOG.error('There is no fused file {}, aborting'.format(viirs_fused_l1b_file)) rc_fusion = 1 return rc_fusion, None, {} - # Remove the unfused dir... - #LOG.debug('Removing the unfused level-1b dir {} ...'.format(unfused_l1b_dir)) - #shutil.rmtree(unfused_l1b_dir) - # Determine the name of the output fused file - if satellite=='snpp' or satellite=='noaa20': - esdt = sipsprod.satellite_esdt('V02FSN', satellite) - product.options['collection'] = int(basename(l1b_file).split('.')[3]) - fused_l1b_file_new = sipsprod.product_filename(esdt, product.options['collection'], - granule, dt_create) - if satellite=='aqua': - esdt = sipsprod.satellite_esdt('M02FSN', satellite) - product.options['collection'] = int(basename(l1b_file).split('.')[3]) - fused_l1b_file_new = sipsprod.product_filename(esdt, product.options['collection'], - granule, dt_create) - fused_l1b_file_new = fused_l1b_file_new.replace('.nc', '.hdf') - - # Move the HDF4/NetCDF4 file to its new filename - LOG.debug('Moving "{}" to "{}" ...'.format(fused_l1b_file, pjoin(tmp_work_dir, fused_l1b_file_new))) - shutil.move(fused_l1b_file, pjoin(tmp_work_dir, fused_l1b_file_new)) - fused_l1b_file = glob(pjoin(tmp_work_dir, fused_l1b_file_new))[0] + # Move the NetCDF4 file to its new filename + LOG.debug('Moving "{}" to "{}" ...'.format(viirs_fused_l1b_file, tmp_work_dir)) + shutil.move(viirs_fused_l1b_file, tmp_work_dir) + viirs_fused_l1b_file = glob(pjoin(tmp_work_dir, basename(viirs_fused_l1b_file)))[0] + LOG.debug('Final fused output file "{}"'.format(viirs_fused_l1b_file)) # Move the matlab file to its new filename - if satellite=='snpp' or satellite=='noaa20': - matlab_file_new = basename(fused_l1b_file_new).replace('.nc','.mat') - if satellite=='aqua': - matlab_file_new = basename(fused_l1b_file_new).replace('.hdf','.mat') + matlab_file_new = viirs_fused_l1b_file.replace('.nc','.mat') - LOG.debug('Moving "{}" to {}...'.format(matlab_file, pjoin(fused_output_dir, matlab_file_new))) - shutil.move(matlab_file, pjoin(fused_output_dir, matlab_file_new)) - matlab_file = glob(pjoin(fused_output_dir, matlab_file_new))[0] + LOG.debug('Moving "{}" to {}...'.format(matlab_file, matlab_file_new)) + shutil.move(matlab_file, matlab_file_new) + matlab_file = glob(matlab_file_new)[0] + LOG.debug('Final fused output matlab file "{}"'.format(matlab_file)) # Remove the fused_outputs directory - #LOG.debug('Removing the fused_outputs dir {} ...'.format(fused_output_dir)) - #shutil.rmtree(fused_output_dir) + LOG.debug('Removing the fused_outputs dir {} ...'.format(fused_output_dir)) + shutil.rmtree(fused_output_dir) output_attrs = {'esdt': esdt, 'collection': product.options['collection'], 'created': dt_create} - return rc_fusion, fused_l1b_file, output_attrs + return rc_fusion, viirs_fused_l1b_file, output_attrs def output_QC(self, l1b_file, fused_l1b_file, band=None, input_rms=0.2, **kwargs): @@ -656,59 +664,95 @@ class FUSION_MATLAB(Computation): return 0 if passfail else 1 + def set_l2_metadata(self, l2_file, viirs_l1b_file, viirs_geo_file, cris_l1b_file, product, context): + ''' + Construct various metadata strings and write them to the level2 file + ''' - # def update_global_attrs(self, netcdf_file, readme_file, **kwargs): - - # satellite = kwargs['satellite'] - - # # Get the git repo information - # repo_attrs = [] - # try: - # LOG.debug('Opening {}...'.format(readme_file)) - # readme_obj = open(readme_file, 'ro') - # line_obj = readme_obj.readlines() - # for idx, line in enumerate(line_obj): - # if '.git' in line: - # repo_line = line.lstrip(' -*,*').rstrip(' -*,;') - # commit_line = line_obj[idx+1].lstrip(' -*,;').rstrip(' -*,;') - # git_line = '{}; {}'.format(repo_line, commit_line).replace('\n', '') - # LOG.debug('{}'.format(git_line)) - # repo_attrs.append(git_line) - # except Exception: - # LOG.debug(traceback.format_exc()) - - # readme_obj.close() - - # # Update the various file global attributes - # LOG.debug('Adding attributes to {} ...'.format(netcdf_file)) - # if splitext(netcdf_file)[-1] == '.nc': - # args = (netcdf_file, "a") - # kwargs = {'format': "NETCDF4"} - # file_open = Dataset - # if splitext(netcdf_file)[-1] == '.hdf': - # args = (netcdf_file, SDC.WRITE) - # kwargs = {} - # file_open = SD - - # try: - - # file_obj = file_open(*args, **kwargs) - - # # Update the attributes, moving to the end - # for idx, attr in enumerate(repo_attrs): - # LOG.debug('{}'.format(attr)) - # setattr(file_obj, 'source_git_repo_{}'.format(idx), attr) - - # except Exception: - # LOG.warning("\tProblem setting attributes in output file {}".format(netcdf_file)) - # LOG.debug(traceback.format_exc()) + #context = {} + #context['satellite'] = satellite + context['nrt'] = True + #context['collection'] = 0 + + #product = sipsprod.lookup_product_recurse('FSNRAD_L2_VIIRS_CRIS', version=version) + #product.options['collection'] = context['collection'] + + satname = 'SNPP' if context['satellite']=='snpp' else 'NOAA20' + esdt = product.name + ('_SNPP' if context['satellite']=='snpp' else '_NOAA20') + dt_create = datetime.strptime(splitext(basename(l2_file))[0].split('.')[-1], '%Y%j%H%M%S') + print("Creation date is {}".format(dt_create)) + + viirs_input_fns, viirs_lut_version, viirs_lut_created = get_viirs_l1_luts(viirs_l1b_file, geo_fn=viirs_geo_file) + ancillary_fns = [] + viirs_l1_version = product.input('viirs_l1').version + cris_l1_version = product.input('cris_l1').version + + set_official_product_metadata( + esdt, + product.version, + product.options['collection'], + product.input('fusion_matlab').version, + context['satellite'], + l2_file, + viirs_geo_file, + viirs_input_fns, + ancillary_fns, + viirs_l1_version, + viirs_lut_version, + viirs_lut_created, + product.inputstr(), + dt_create, + context['nrt']) + + # Remove any troublesome global attributes + try: + nc_l2 = netCDF4.Dataset(l2_file, 'a') + xmlmetadata = nc_l2.getncattr('xmlmetadata') + nc_l2.delncattr('xmlmetadata') + nc_l2.delncattr('l1_version') + nc_l2.delncattr('l1_lut_version') + nc_l2.delncattr('l1_lut_created') + nc_l2.delncattr('input_files') + except: + pass + + nc_l2.close() + + nc_remove_unlimited_dims(basename(l2_file)) + + FIX_ATTRS = [ + ('title', '{0:} VIIRS+CrIS Fusion ({1:})'.format(satname, esdt)), + ('platform', {'snpp':'Suomi NPP', 'noaa20':'NOAA-20'}[context['satellite']]), + ('instrument', 'VIIRS+CrIS'), + ('conventions', 'CF-1.6, ACDD-1.3'), + ('AlgorithmType', 'OPS'), + ('long_name', '{} VIIRS+CrIS Fusion 6-Min L2 Swath 750m'.format(satname)), + ('project', 'NASA Atmosphere Discipline'), + ('creator_name', 'NASA Atmosphere SIPS'), + ('processing_version', product.input('fusion_matlab').version), + ('viirs_l1_version', viirs_l1_version), + ('viirs_lut_version', viirs_lut_version), + ('viirs_lut_created', str(viirs_lut_created.isoformat())), + ('cris_l1_version', cris_l1_version), + ('input_files', ', '.join([basename(x) for x in [viirs_geo_file, viirs_l1b_file, cris_l1b_file]])), + ('xmlmetadata', xmlmetadata), + ] + nc_setattrs(l2_file, FIX_ATTRS) + + extra_attrs = {'esdt': esdt, + 'collection': product.options['collection'], + 'ecstype': 'SCIENCE', + 'viirs_lut_version': viirs_lut_version, + 'viirs_lut_created': viirs_lut_created, + 'created': dt_create} - # if netcdf_file.split('.')[-1] == 'nc': - # file_obj.close() - # elif netcdf_file.split('.')[-1] == 'hdf': - # file_obj.end() - # return + return { + 'fused_l1b': { + 'file': basename(l2_file), + 'extra_attrs': extra_attrs, + } + } def prepare_env(self, dist_root, inputs, context): LOG.debug("Running prepare_env()...") @@ -732,7 +776,7 @@ class FUSION_MATLAB(Computation): return env - @reraise_as(WorkflowNotReady, FileNotFound, prefix='V02FSN') + @reraise_as(WorkflowNotReady, FileNotFound, prefix='FSNRAD_L2_VIIRS_CRIS') def run_task(self, inputs, context): LOG.debug("Running run_task()...") @@ -770,7 +814,7 @@ class FUSION_MATLAB(Computation): LOG.debug("Inputs dir = {}".format(inputs_dir)) # Are we doing a dummy run? - # dummy = True + #dummy = True dummy = False geo_keys = sorted([key for key in inputs.keys() if 'geo' in key]) @@ -817,6 +861,7 @@ class FUSION_MATLAB(Computation): LOG.debug("collo = \n\t{}".format('\n\t'.join(collo))) bin_dir = pjoin(dist_root, 'bin') + cdl_dir = pjoin(dist_root, 'cdl') anc_dir = pjoin(dist_root, 'luts') fused_output_dir = pjoin(work_dir, 'fused_outputs') @@ -824,6 +869,7 @@ class FUSION_MATLAB(Computation): kwargs = {} kwargs['py_interp'] = py_interp kwargs['bin_dir'] = bin_dir + kwargs['cdl_dir'] = cdl_dir kwargs['env'] = env kwargs['fused_output_dir'] = fused_output_dir kwargs['satellite'] = satellite @@ -833,6 +879,7 @@ class FUSION_MATLAB(Computation): if satellite=='snpp' or satellite=='noaa20': geo_file = geo[1] l1b_file = l1b[1] + sounder_file = sounder[1] kwargs['anc_paths'] = [pjoin(anc_dir, 'modis_aqua.srf.nc'), pjoin(anc_dir, 'NG_VIIRS_NPP_RSR_filtered_Oct2011_BA/')] kwargs['fusion_binary'] = 'run_imagersounderfusion_V.sh' @@ -841,6 +888,7 @@ class FUSION_MATLAB(Computation): elif satellite=='aqua': geo_file = geo[0] l1b_file = l1b[0] + sounder_file = sounder[0] kwargs['anc_paths'] = [pjoin(anc_dir, 'L2.chan_prop.2005.03.01.v9.5.1.txt'), pjoin(anc_dir, 'modis_aqua.srf.nc'), pjoin(anc_dir, 'modis_conv_error_2005.mat')] @@ -866,12 +914,22 @@ class FUSION_MATLAB(Computation): if matlab_file is None: raise RuntimeError('Output fusion file fusion_output.mat not created.') + #kwargs['dummy'] = True # dummy + # Now that we've computed the Matlab file, convert to a NetCDF file... - rc_fusion, fused_l1b_file, output_attrs = self.convert_matlab_to_netcdf( - product, - matlab_file, - l1b_file, - **kwargs) + if satellite=='snpp' or satellite=='noaa20': + rc_fusion, fused_l1b_file, output_attrs = self.convert_matlab_to_viirs_netcdf( + product, + l1b_file, + sounder_file, + matlab_file, + **kwargs) + #elif satellite=='aqua': + #rc_fusion, fused_l1b_file, output_attrs = self.convert_matlab_to_netcdf( + #product, + #matlab_file, + #l1b_file, + #**kwargs) LOG.debug('convert_matlab_to_netcdf() return value: {}'.format(rc_fusion)) LOG.info('convert_matlab_to_netcdf() generated {}'.format(fused_l1b_file)) @@ -879,49 +937,24 @@ class FUSION_MATLAB(Computation): if fused_l1b_file is None: raise RuntimeError('NetCDF output fusion file was not created.') - # Update some global attributes in the output file - #readme_file = pjoin(delivery.path, 'README.txt') - #self.update_global_attrs(basename(fused_l1b_file), readme_file, **kwargs) - - # Run a QC check on the output file - #rc_qc = self.output_QC(l1b, fused_l1b_file, **kwargs) - #LOG.debug('output_QC() return value: {}'.format(rc_qc)) - #if rc_qc != 0: - #raise RuntimeError('Output fusion file {} failed RMS error QC check, output aborted.'.format(fused_l1b_file)) - # The staging routine assumes that the output file is located in the work directory # "tmp******", and that the output path is to be prepended, so return the basename. - out_fn = basename(fused_l1b_file) - # Set metadata to be put in the output file. - if satellite=='snpp' or satellite=='noaa20': - viirs_l1 = product.input('viirs_l1') - v02mod_bt_sc = product.input('V02MOD-bt-sc') + # Update the output file metadata to conform to LAADS requirements. + out_dict = self.set_l2_metadata(fused_l1b_file, l1b_file, geo_file, sounder_file, product, context) - if viirs_l1: - l1_version = product.input('viirs_l1').version - elif v02mod_bt_sc: - l1_version = v02mod_bt_sc.input('viirs_l1').version - - input_fns, lut_version, lut_created = get_viirs_l1_luts(l1b_file, geo_fn=geo_file) - ancillary_fns = [] + if satellite=='snpp' or satellite=='noaa20': out_compress = nc_compress - - set_official_product_metadata( - output_attrs['esdt'], product.version, output_attrs['collection'], - product.input('fusion_matlab').version, context['satellite'], - fused_l1b_file, geo_file, - input_fns, ancillary_fns, - l1_version, lut_version, lut_created, - product.inputstr(), output_attrs['created']) - if satellite == 'aqua': out_compress = hdf_compress LOG.debug('We are in {}'.format(os.getcwd())) - LOG.debug('Compressing {}'.format(out_fn)) - return {'fused_l1b': out_compress(out_fn)} + + #LOG.debug('Compressing {}'.format(out_fn)) + #return {'fused_l1b': out_compress(out_fn)} + + return out_dict class FUSION_MATLAB_QL(Computation): @@ -973,8 +1006,8 @@ class FUSION_MATLAB_QL(Computation): def _add_cris_viirs_fusion_l1b_input(self, product, context, task): satellite = context['satellite'] granule = context['granule'] - version = product.input('V02FSN').version - input_name = sipsprod.satellite_esdt('V02FSN', satellite) + version = product.input('FSNRAD_L2_VIIRS_CRIS').version + input_name = sipsprod.satellite_esdt('FSNRAD_L2_VIIRS_CRIS', satellite) interval = TimeInterval(granule, granule+timedelta(days=1.00)-timedelta(seconds=1)) LOG.debug("Ingesting input {} ({}) for V02FSN_DailyQL version {}".format(input_name, version, product.version)) vl1b = dawg_catalog.files(satellite, input_name, interval, version=version) @@ -987,7 +1020,7 @@ class FUSION_MATLAB_QL(Computation): raise WorkflowNotReady('Number of available {} inputs is < 228, for version {} and interval {}, aborting...'.format( input_name, version, interval)) for idx, l1b_file in enumerate(vl1b): - LOG.debug('V02FSN granule {}: {} -> {}'.format(idx, l1b_file.begin_time, l1b_file.end_time)) + LOG.debug('FSNRAD_L2_VIIRS_CRIS granule {}: {} -> {}'.format(idx, l1b_file.begin_time, l1b_file.end_time)) task.input('l1b_{}'.format(idx), l1b_file) def build_task_snpp(self, context, task): @@ -1096,7 +1129,7 @@ class FUSION_MATLAB_QL(Computation): filename_chunks = splitext(orig_fusion_ql_file)[0].split('_') fusion_ql_files.append( '{}.A{}.{}.{}.png'.format( - sipsprod.satellite_esdt('V02FSN', satellite), + sipsprod.satellite_esdt('FSNRAD_L2_VIIRS_CRIS', satellite), granule.strftime('%Y%j'), filename_chunks[1], filename_chunks[3] -- GitLab