diff --git a/remote_scripts/README.md b/remote_scripts/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ab226a873425ac88bbd137bf5d3f4854bf44a99f --- /dev/null +++ b/remote_scripts/README.md @@ -0,0 +1,42 @@ +# SPARChive Processing at the SSEC + +Once data is uploaded to the SSEC server(s) it may have additional processing +applied to it. This file describes how to recreate this processing environment +from scratch. + +## Operation Installation + +These instructions assume the operation "standard practice" that is current +at the time of writing. This practice is to create a conda environment +using the "root" user under `/opt/metobs/aoss-cameras`. + +WARNING: These instructions use a generic "/opt/miniconda3/" conda environment +to build the operational environment. Double check that a newer +"/opt/miniconda3_YYYYMMDD" doesn't exist that is newer. If it does exist then +that installation should probably be used instead. + +1. Get and install the conda environment: + + ``` + cd /tmp + curl -O "https://gitlab.ssec.wisc.edu/metobs/sparchive/-/raw/master/remote_scripts/environment.yaml" + sudo /opt/miniconda3/condabin/conda env create --file environment.yaml --prefix /opt/metobs/sparchive + ``` + +2. Clone the git repository into the environment and other setup: + + ``` + cd /opt/metobs/sparchive + sudo mkdir -p repos/git log lock + sudo chown metobs:metobsgrp log lock + sudo git clone "https://gitlab.ssec.wisc.edu/metobs/sparchive.git" repos/git/sparchive + sudo ln -s repos/git/sparchive/remote_scripts ./scripts + sudo git clone "https://gitlab.ssec.wisc.edu/metobs/MetObsCommon.git" repos/git/MetObsCommon + ``` + +3. Install the sparchive python package. + + ``` + sudo /opt/metobs/sparchive/bin/pip install -e repos/git/MetObsCommon/ + sudo /opt/metobs/sparchive/bin/pip install -e repos/git/sparchive/sparchive/ + ``` diff --git a/remote_scripts/archive_aerioe.sh b/remote_scripts/archive_aerioe.sh index 54d854c038e8416f2ff9240fed2d7895a219941b..e10d5a51f2b09812e918460ebb5a7359a9d66144 100755 --- a/remote_scripts/archive_aerioe.sh +++ b/remote_scripts/archive_aerioe.sh @@ -10,7 +10,7 @@ DATE=$1 if [ -z "$DATE" ]; then DATE=`date +%Y%m%d` fi -LOCK="${ENV}/locks/${SCRIPT_NAME}.lock" +LOCK="${ENV}/lock/${SCRIPT_NAME}.lock" logfile="${LOGDIR}/${SCRIPT_NAME}.log" # Get environment variables and common functions diff --git a/remote_scripts/archive_dopplerlidar_quicklooks.sh b/remote_scripts/archive_dopplerlidar_quicklooks.sh index 250e51c941490c4f8068e3d663547c7d2e844453..7ea7c3438ce2f97b56a1729a6f6878744aa1006c 100755 --- a/remote_scripts/archive_dopplerlidar_quicklooks.sh +++ b/remote_scripts/archive_dopplerlidar_quicklooks.sh @@ -10,7 +10,7 @@ DATE=$1 if [ -z "$DATE" ]; then DATE=`date +%Y%m%d` fi -LOCK="${ENV}/locks/${SCRIPT_NAME}.lock" +LOCK="${ENV}/lock/${SCRIPT_NAME}.lock" logfile="${LOGDIR}/${SCRIPT_NAME}.log" # Get environment variables and common functions diff --git a/environment.yaml b/remote_scripts/environment.yaml similarity index 87% rename from environment.yaml rename to remote_scripts/environment.yaml index 2331febff99e931624824ecacbe8d4c49c897ad7..8a25e60c79c08ed26f94ded535e76fea0ef918a3 100644 --- a/environment.yaml +++ b/remote_scripts/environment.yaml @@ -7,3 +7,4 @@ dependencies: - sh - bokeh - matplotlib + - python=3.8 diff --git a/remote_scripts/pull_aerioe.sh b/remote_scripts/pull_aerioe.sh index e34aaba2e907d83744c92f9e2ae4db233350915c..e458e66e4f80fc6f216fe3fa5cf766520d65dbcb 100755 --- a/remote_scripts/pull_aerioe.sh +++ b/remote_scripts/pull_aerioe.sh @@ -10,7 +10,7 @@ DATE=$1 if [ -z "$DATE" ]; then DATE=`date +%Y%m%d` fi -LOCK="${ENV}/locks/${SCRIPT_NAME}.lock" +LOCK="${ENV}/lock/${SCRIPT_NAME}.lock" logfile="${LOGDIR}/${SCRIPT_NAME}.log" SSH_OPTIONS="-o ConnectTimeout=5" diff --git a/remote_scripts/pull_dopplerlidar_quicklooks.sh b/remote_scripts/pull_dopplerlidar_quicklooks.sh index e88f3f4f6b680f7199d2a0a52dfadfeb672cea9a..1834e39d02b998020754e1b829a62f80be2d07ab 100755 --- a/remote_scripts/pull_dopplerlidar_quicklooks.sh +++ b/remote_scripts/pull_dopplerlidar_quicklooks.sh @@ -10,7 +10,7 @@ DATE=$1 if [ -z "$DATE" ]; then DATE=`date +%Y%m%d` fi -LOCK="${ENV}/locks/${SCRIPT_NAME}.lock" +LOCK="${ENV}/lock/${SCRIPT_NAME}.lock" logfile="${LOGDIR}/${SCRIPT_NAME}.log" SSH_OPTIONS="-o ConnectTimeout=5" diff --git a/remote_scripts/run_metobs_rtv_archive.sh b/remote_scripts/run_metobs_rtv_archive.sh index 4fa81d40eb3742c83e8dd05f3d02d72557142156..18977e2c86efcd143f9f3bd1de68c90e73463782 100755 --- a/remote_scripts/run_metobs_rtv_archive.sh +++ b/remote_scripts/run_metobs_rtv_archive.sh @@ -49,7 +49,7 @@ mlab_input_dir="/opt/metobs/sparchive/repos/git/uwrtv_aeri/AERI/${SITE}/AE${AERI output_dir="/opt/metobs/sparchive/repos/git/uwrtv_aeri/output" cache_dir="/mnt/inst-data/cache/sparc/current/aeri/AE${AERI_DATE}" exp_name=`readlink /mnt/inst-data/cache/sparc/current` -LOCK="/opt/metobs/sparchive/locks/$(basename $0).lock" +LOCK="/opt/metobs/sparchive/lock/$(basename $0).lock" if [ ! -d ${cache_dir} ]; then oops "Data cache directory doesn't exist: ${cache_dir}" diff --git a/remote_scripts/sparchive_env.sh b/remote_scripts/sparchive_env.sh index e3367a33a5c1cc07544349f304ad3a70a527b9ed..e33afa96ad979a5750ac5d0d98bb04a0f5fdc9b2 100644 --- a/remote_scripts/sparchive_env.sh +++ b/remote_scripts/sparchive_env.sh @@ -8,7 +8,7 @@ export SPARCHIVE_TMP_ROOT=${SPARCHIVE_TMP_ROOT="/data1/tmp/sparc_post_processing export SPARCHIVE_INCOMING_ROOT=${SPARCHIVE_INCOMING_ROOT="/data1/incoming/sparc"} export SPARCHIVE_RAW_ROOT=${SPARCHIVE_RAW_ROOT="/data1/raw/sparc"} export SPARCHIVE_CACHE_ROOT=${SPARCHIVE_CACHE_ROOT="/data1/cache/sparc"} -export SPARCHIVE_LOG_DIR=${SPARCHIVE_LOG_DIR="${SPARCHIVE_ENV}/logs"} +export SPARCHIVE_LOG_DIR=${SPARCHIVE_LOG_DIR="${SPARCHIVE_ENV}/log"} export SPARCHIVE_INSTRUMENTS=${SPARCHIVE_INSTRUMENTS="aeri windpro radiosondes surfacemet hsrl"} export SPARCHIVE_PAST_DAYS=${SPARCHIVE_PAST_DAYS=3}