diff --git a/build_package/README.md b/build_package/README.md deleted file mode 100644 index 77eecd61298d321e7644801503870126e8ceed46..0000000000000000000000000000000000000000 --- a/build_package/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Gridded GLM Package Build - -This directory contains scripts and files needed to build the CSPP Geo -Gridded GLM packaged from an existing buildbucket docker image. - -## Build Instructions - -TODO \ No newline at end of file diff --git a/buildbucket/README.md b/buildbucket/README.md index 221cab9421ae8ac837474df9b08fb547945bda6d..a18f4795e3a449e5f03212aba89b5f630240b820 100644 --- a/buildbucket/README.md +++ b/buildbucket/README.md @@ -20,3 +20,21 @@ docker tag gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-gridded-glm/buildbucket:r docker push gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-gridded-glm/buildbucket:latest ``` + +## Usage + +To create the Gridded GLM package with the version number 1.0.0 in your current directory: + +```bash +docker run --rm -v "${PWD}":/dock gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-gridded-glm/buildbucket:latest package.sh v1.0.0 +``` + +If some things need to be customized you can specify various environment +variables with `-e` to `docker run`. See the top of the `Dockerfile` for +more information on available environment variables. + +For example, to use a local copy of this repository: + +```bash +docker run --rm -v $PWD:/dock -v $PWD:/work/cspp-geo-gridded-glm gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-gridded-glm/buildbucket:r$(date '+%Y%m%d') ./package.sh v1.0.0-prealpha0 +``` diff --git a/buildbucket/package.sh b/buildbucket/package.sh index 3d9f1ea49139c83674a274ab4a118b94c5a0d618..9a8178b110a76d7fa98a3628e286ca9b65238cde 100755 --- a/buildbucket/package.sh +++ b/buildbucket/package.sh @@ -1,6 +1,14 @@ #!/bin/bash -le # purpose: Create package and place it in /dock -# requires: /gridded_glm to be mounted with package contents (gridded_glm in repository) +# environment variables: +# DIST: What mount to write the package to (default: '/dist') +# LMATOOLS_REF: git reference (branch name) to use for lmatools (default: minvaluegrids) +# GLMTOOLS_REF: git reference (branch name) to use for glmtools (default: ugf-newgrid) +# GLMTOOLS_REPOS: git repository URL to use for glmtools (default: https://github.com/deeplycloudy/glmtools.git) +# GGLM_REF: git reference (branch name) to use for this package's bash scripts (default: master) +# GGLM_REPOS: git repository URL to use for this package's bash scripts +# If a "/work/cspp-geo-gridded-glm" directory is already mounted +# then that will be used and GGLM_REF will be ignored. # note: The shebang at the top of this file is needed as-is. The '-l' will # load the .bashrc which allows us to use conda. The '-e' triggers exit # on error behavior. We cannot use '/usr/bin/env bash -le' since most @@ -8,19 +16,18 @@ if [ $# -eq 1 ]; then version=$1 - lmatools_ref=minvaluegrids - glmtools_ref=ugf-newgrid -elif [ $# -eq 3 ]; then - version=$1 - lmatools_ref=minvaluegrids - glmtools_ref=ugf-newgrid else - echo "Usage: package.sh vX.Y.Z [lmatools_ref glmtools_ref]" + echo "Usage: package.sh vX.Y.Z" exit 1 fi pkg_name=cspp_geo_gridded_glm-${version} DIST=${DIST:-"/dock"} +LMATOOLS_REF=${LMATOOLS_REF:-"minvaluegrids"} +GLMTOOLS_REF=${GLMTOOLS_REF:-"ugf-newgrid"} +GLMTOOLS_REPOS=${GLMTOOLS_REPOS:-"https://github.com/deeplycloudy/glmtools.git"} +GGLM_REPOS=${GGLM_REPOS:-"https://gitlab.ssec.wisc.edu/cspp_geo/cspp-geo-gridded-glm.git"} +GGLM_REF=${GGLM_REF:-"master"} make_dockerfile() { cat >$DIST/Dockerfile <<EOF @@ -44,8 +51,8 @@ which python conda info -a conda list --export echo "Version specified: ${version}" -echo "lmatools reference: ${lmatools_ref}" -echo "glmtools reference: ${glmtools_ref}" +echo "lmatools reference: ${LMATOOLS_REF}" +echo "glmtools reference: ${GLMTOOLS_REF}" # Turn on command printing here because we don't need all of the `conda` # internal bash commands to pollute our output @@ -54,13 +61,20 @@ set -x # Install glmtools and related packages # All dependencies should have been built with the buildbucket # Careful: Could result in missing dependencies but we want reproducibility -pip install --no-deps git+https://github.com/deeplycloudy/lmatools.git@${lmatools_ref} +pip install --no-deps git+https://github.com/deeplycloudy/lmatools.git@${LMATOOLS_REF} pip install --no-deps git+https://github.com/deeplycloudy/stormdrain.git -pip install --no-deps git+https://github.com/deeplycloudy/glmtools.git@${glmtools_ref} +pip install --no-deps git+${GLMTOOLS_REPOS}@${GLMTOOLS_REF} + +# get the current packages files +# if the caller mounted the repository already then use that +GGLM_DIR="/work/cspp-geo-gridded-glm/gridded_glm" +if [ ! -d "${GGLM_DIR}" ]; then + git clone -b "${GGLM_REF}" "${GGLM_REPOS}" +fi # Build a tarball version of the current conda environment # TODO: Add conda cleanup commands similar to what Polar2Grid uses to save space -conda_tb=conda_lmatools-${lmatools_ref}_glmtools-${glmtools_ref}.tar.gz +conda_tb=conda_lmatools-${LMATOOLS_REF}_glmtools-${GLMTOOLS_REF}.tar.gz conda pack --n-threads $(nproc) -n build -o ${conda_tb} # Build up our package directory @@ -68,14 +82,14 @@ mkdir -p ${pkg_name} cd ${pkg_name} mkdir -p bin opt/conda # Copy package scripts/data to package directory -cp /gridded_glm/PACKAGE_README.md ./README.md -cp /gridded_glm/bin/* ./bin/ +cp $GGLM_DIR/PACKAGE_README.md ./README.md +cp $GGLM_DIR/bin/* ./bin/ # Untar the tarball so we can put things where we want tar -xz -C ./opt/conda -f ../${conda_tb} # Go back to original work directory cd .. # Create tarball of package directory -XZ_DEFAULTS="--threads=$(nproc)" tar -Jcf --uid 0 --gid 0 ${pkg_name}.tar.xz ${pkg_name} +XZ_DEFAULTS="--threads=$(nproc)" tar -Jc --owner 0 --group 0 -f ${pkg_name}.tar.xz ${pkg_name} mv ${pkg_name}.tar.xz ${DIST}/ make_dockerfile diff --git a/gridded_glm/README.md b/gridded_glm/README.md index 6f8364aa0e6ddaa3528cd1cb3df27d21ab623510..f0c64359bd90278265f79065011b76c7ce3e503e 100644 --- a/gridded_glm/README.md +++ b/gridded_glm/README.md @@ -5,4 +5,13 @@ Gridded GLM package. ## Usage -TODO \ No newline at end of file +```bash +docker run -it -v $PWD:/work --rm cspp_geo_gridded_glm/run_package:r20191024 \ + bash /opt/ssec/cspp_geo_gridded_glm-v1.0.0-prealpha0/bin/make_glm_grids.sh -o /work \ + --fixed_grid --split_events --goes_position east --goes_sector conus --dx=2.0 --dy=2.0 OR*.nc +``` + +Note that "OR*.nc" must exist in the current directory to make bash expand +the file pattern and pass the full list of files to the docker container. +Running this will produce a NetCDF file in a `YYYY` directory in your current +directory. \ No newline at end of file