diff --git a/buildbucket/package.sh b/buildbucket/package.sh index 4250d496a9aa818a403dcc15e6b61f1027278fb5..f8ac99eee1da20b07f244e2c12cc2f99f5114224 100755 --- a/buildbucket/package.sh +++ b/buildbucket/package.sh @@ -78,6 +78,10 @@ cp $GGLM_DIR/bin/* ./bin/ cp $GGLM_DIR/libexec/* ./libexec/ # Untar the tarball so we can put things where we want tar -xz -C ./libexec/python_runtime -f ../${conda_tb} +# write a conda environment.yml to the python_runtime/ folder so that we can more easily audit what's included in our runtime between builds +conda env export --file ./libexec/python_runtime/environment.yml +# write our version string to a hidden VERSION text file +echo "$version" > ./.VERSION.txt # Go back to original work directory cd .. diff --git a/gridded_glm/libexec/_minute_gridder.py b/gridded_glm/libexec/_minute_gridder.py index cac20871b10ace798ba87c7c1e11ee9e48eaa33c..3e63eee1e5f8f7015f9b8760c40b4cb4d6fcc77f 100644 --- a/gridded_glm/libexec/_minute_gridder.py +++ b/gridded_glm/libexec/_minute_gridder.py @@ -21,6 +21,8 @@ import tempfile import shutil import atexit from glob import glob +import socket +from netCDF4 import Dataset #from multiprocessing import freeze_support # https://docs.python.org/2/library/multiprocessing.html#multiprocessing.freeze_support from functools import partial from lmatools.grid.make_grids import write_cf_netcdf_latlon, write_cf_netcdf_noproj, write_cf_netcdf_fixedgrid @@ -186,6 +188,25 @@ def grid_setup(args, work_dir=os.getcwd()): return gridder, args.filenames, start_time, end_time, grid_kwargs +def get_cspp_gglm_version(): + try: + version_filename = os.path.join(os.getenv('CSPP_GEO_GGLM_HOME'), ".VERSION.txt") + return open(version_filename, 'r').read().rstrip() + except: + return "unknown" + + +def add_gglm_attrs(netcdf_filename, input_filenames): + try: + nc = Dataset(netcdf_filename, 'a') + setattr(nc, 'cspp_geo_gglm_version', get_cspp_gglm_version()) + setattr(nc, 'cspp_geo_gglm_production_host', socket.gethostname()) + setattr(nc, 'cspp_geo_gglm_input_files', ",".join([os.path.basename(f) for f in input_filenames])) + nc.close() + except: + log.error("could not add CSPP Geo GGLM attributes to {}".format(netcdf_filename)) + + if __name__ == '__main__': # freeze_support() # nb. I don't think this is needed as we're not making windows execs at this time parser = create_parser() @@ -222,6 +243,11 @@ if __name__ == '__main__': gridded_files = glob(gridded_path) log.debug(gridded_files) + # we need to add attributes here due to an issue where satpy (or its dependencies) are + # holding the input gridded file open until the process exits + for f in gridded_files: + add_gglm_attrs(f, glm_filenames) + # (optionally) do tiling if args.create_tiles: from satpy import Scene @@ -252,6 +278,7 @@ if __name__ == '__main__': tiled_path = os.path.join(tempdir_path, 'OR_GLM-L2-GLM?-M?_G??_T??_*.nc') tiled_files = glob(tiled_path) for f in tiled_files: + add_gglm_attrs(f, glm_filenames) shutil.move(f, os.path.join(args.output_dir, os.path.basename(f))) for f in gridded_files: shutil.move(f, os.path.join(args.output_dir, os.path.basename(f)))