Add working glmtools configuration

Requires djhoese github branch for glmtools to work without matplotlib

build_package/README.md

-# Gridded GLM Package Build
-
-This directory contains scripts and files needed to build the CSPP Geo
-Gridded GLM packaged from an existing buildbucket docker image.
-
-## Build Instructions
-
-TODO
\ No newline at end of file

buildbucket/README.md

@@ -20,3 +20,21 @@ docker tag gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-gridded-glm/buildbucket:r
 docker push gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-gridded-glm/buildbucket:latest
 
 ```
+
+## Usage
+
+To create the Gridded GLM package with the version number 1.0.0 in your current directory:
+
+```bash
+docker run --rm -v "${PWD}":/dock gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-gridded-glm/buildbucket:latest package.sh v1.0.0
+```
+
+If some things need to be customized you can specify various environment
+variables with `-e` to `docker run`. See the top of the `Dockerfile` for
+more information on available environment variables.
+
+For example, to use a local copy of this repository:
+
+```bash
+docker run --rm -v $PWD:/dock -v $PWD:/work/cspp-geo-gridded-glm gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-gridded-glm/buildbucket:r$(date '+%Y%m%d') ./package.sh v1.0.0-prealpha0
+```

buildbucket/package.sh

 #!/bin/bash -le
 # purpose: Create package and place it in /dock
-# requires: /gridded_glm to be mounted with package contents (gridded_glm in repository)
+# environment variables:
+#   DIST: What mount to write the package to (default: '/dist')
+#   LMATOOLS_REF: git reference (branch name) to use for lmatools (default: minvaluegrids)
+#   GLMTOOLS_REF: git reference (branch name) to use for glmtools (default: ugf-newgrid)
+#   GLMTOOLS_REPOS: git repository URL to use for glmtools (default: https://github.com/deeplycloudy/glmtools.git)
+#   GGLM_REF: git reference (branch name) to use for this package's bash scripts (default: master)
+#   GGLM_REPOS: git repository URL to use for this package's bash scripts
+#               If a "/work/cspp-geo-gridded-glm" directory is already mounted
+#               then that will be used and GGLM_REF will be ignored.
 # note: The shebang at the top of this file is needed as-is. The '-l' will
 #       load the .bashrc which allows us to use conda. The '-e' triggers exit
 #       on error behavior. We cannot use '/usr/bin/env bash -le' since most
@@ -8,19 +16,18 @@
 
 if [ $# -eq 1 ]; then
     version=$1
-    lmatools_ref=minvaluegrids
-    glmtools_ref=ugf-newgrid
-elif [ $# -eq 3 ]; then
-    version=$1
-    lmatools_ref=minvaluegrids
-    glmtools_ref=ugf-newgrid
 else
-    echo "Usage: package.sh vX.Y.Z [lmatools_ref glmtools_ref]"
+    echo "Usage: package.sh vX.Y.Z"
     exit 1
 fi
 
 pkg_name=cspp_geo_gridded_glm-${version}
 DIST=${DIST:-"/dock"}
+LMATOOLS_REF=${LMATOOLS_REF:-"minvaluegrids"}
+GLMTOOLS_REF=${GLMTOOLS_REF:-"ugf-newgrid"}
+GLMTOOLS_REPOS=${GLMTOOLS_REPOS:-"https://github.com/deeplycloudy/glmtools.git"}
+GGLM_REPOS=${GGLM_REPOS:-"https://gitlab.ssec.wisc.edu/cspp_geo/cspp-geo-gridded-glm.git"}
+GGLM_REF=${GGLM_REF:-"master"}
 
 make_dockerfile() {
     cat >$DIST/Dockerfile <<EOF
@@ -44,8 +51,8 @@ which python
 conda info -a
 conda list --export
 echo "Version specified: ${version}"
-echo "lmatools reference: ${lmatools_ref}"
-echo "glmtools reference: ${glmtools_ref}"
+echo "lmatools reference: ${LMATOOLS_REF}"
+echo "glmtools reference: ${GLMTOOLS_REF}"
 
 # Turn on command printing here because we don't need all of the `conda`
 # internal bash commands to pollute our output
@@ -54,13 +61,20 @@ set -x
 # Install glmtools and related packages
 # All dependencies should have been built with the buildbucket
 # Careful: Could result in missing dependencies but we want reproducibility
-pip install --no-deps git+https://github.com/deeplycloudy/lmatools.git@${lmatools_ref}
+pip install --no-deps git+https://github.com/deeplycloudy/lmatools.git@${LMATOOLS_REF}
 pip install --no-deps git+https://github.com/deeplycloudy/stormdrain.git
-pip install --no-deps git+https://github.com/deeplycloudy/glmtools.git@${glmtools_ref}
+pip install --no-deps git+${GLMTOOLS_REPOS}@${GLMTOOLS_REF}
+
+# get the current packages files
+# if the caller mounted the repository already then use that
+GGLM_DIR="/work/cspp-geo-gridded-glm/gridded_glm"
+if [ ! -d "${GGLM_DIR}" ]; then
+    git clone -b "${GGLM_REF}" "${GGLM_REPOS}"
+fi
 
 # Build a tarball version of the current conda environment
 # TODO: Add conda cleanup commands similar to what Polar2Grid uses to save space
-conda_tb=conda_lmatools-${lmatools_ref}_glmtools-${glmtools_ref}.tar.gz
+conda_tb=conda_lmatools-${LMATOOLS_REF}_glmtools-${GLMTOOLS_REF}.tar.gz
 conda pack --n-threads $(nproc) -n build -o ${conda_tb}
 
 # Build up our package directory
@@ -68,14 +82,14 @@ mkdir -p ${pkg_name}
 cd ${pkg_name}
 mkdir -p bin opt/conda
 # Copy package scripts/data to package directory
-cp /gridded_glm/PACKAGE_README.md ./README.md
-cp /gridded_glm/bin/* ./bin/
+cp $GGLM_DIR/PACKAGE_README.md ./README.md
+cp $GGLM_DIR/bin/* ./bin/
 # Untar the tarball so we can put things where we want
 tar -xz -C ./opt/conda -f ../${conda_tb}
 # Go back to original work directory
 cd ..
 
 # Create tarball of package directory
-XZ_DEFAULTS="--threads=$(nproc)" tar -Jcf --uid 0 --gid 0 ${pkg_name}.tar.xz ${pkg_name}
+XZ_DEFAULTS="--threads=$(nproc)" tar -Jc --owner 0 --group 0 -f ${pkg_name}.tar.xz ${pkg_name}
 mv ${pkg_name}.tar.xz ${DIST}/
 make_dockerfile

gridded_glm/README.md

@@ -5,4 +5,13 @@ Gridded GLM package.
 
 ## Usage
 
-TODO
+```bash
\ No newline at end of file
+docker run -it -v $PWD:/work --rm cspp_geo_gridded_glm/run_package:r20191024 \
+    bash /opt/ssec/cspp_geo_gridded_glm-v1.0.0-prealpha0/bin/make_glm_grids.sh -o /work \
+    --fixed_grid --split_events --goes_position east --goes_sector conus --dx=2.0 --dy=2.0 OR*.nc
+```
+
+Note that "OR*.nc" must exist in the current directory to make bash expand
+the file pattern and pass the full list of files to the docker container.
+Running this will produce a NetCDF file in a `YYYY` directory in your current
+directory.
\ No newline at end of file