Commit 43729c2f authored by Joe Taylor's avatar Joe Taylor
Browse files

Merge branch 'feat-serve-ru' into 'develop'

[WIP] RU code as a service

See merge request !2
parents 73326d90 8cdc4e8d
Pipeline #37458 failed with stage
in 6 minutes and 44 seconds
......@@ -35,6 +35,7 @@ docker-build:
echo "Running on branch '$CI_COMMIT_BRANCH': image = $IMAGE_PATH"
fi
- /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --push-retry 4 --destination "${CI_REGISTRY_IMAGE}${IMAGE_PATH}${tag}" --destination "${CI_REGISTRY_IMAGE}${IMAGE_PATH}${revid}"
- /kaniko/executor --build-arg IMAGE=${CI_REGISTRY_IMAGE}${IMAGE_PATH}${tag} --context $CI_PROJECT_DIR/serve/ru --dockerfile $CI_PROJECT_DIR/serve/ru/Dockerfile --push-retry 4 --destination "${CI_REGISTRY_IMAGE}${IMAGE_PATH}/ru${tag}" --destination "${CI_REGISTRY_IMAGE}${IMAGE_PATH}/ru${revid}"
# Run this job in a branch where a Dockerfile exists
rules:
- if: $CI_COMMIT_BRANCH
......
......@@ -7,7 +7,7 @@ usagexit() {
}
abspath() {
python -c 'import sys,os; print("\n".join([os.path.abspath(x) for x in sys.argv[1:]]))' "$@"
python -c 'import sys,os; print("\n".join([os.path.abspath(x) for x in sys.argv[1:]]))' "$@"
}
test -d "$RU_HOME" || echo >&2 "${Oops:?RU_HOME is not set}"
......@@ -24,4 +24,4 @@ test -d "$OUTPUT_FILE" || echo >&2 "${Oops:?please specify an output directory}"
export MATLABPATH="$RU_HOME"
# always limit the blast radius of LD_LIBRARY_PATH to as small a space as possible
export LD_LIBRARY_PATH="$MCRROOT/runtime/glnxa64:$MCRROOT/bin/glnxa64:$MCRROOT/sys/os/glnxa64"
exec "$RU_HOME/run_ru_main.sh" "$MCRROOT" "$SAT" "$INPUT_FILE" "1" "$OUTPUT_FILE"
exec "$RU_HOME/run_ru_main.sh" "$MCRROOT" "$SAT" "$INPUT_FILE" "2" "$OUTPUT_FILE"
......@@ -2,5 +2,5 @@ function RU = ru_main(sat_name,l1b_fname,saveTotalRU_FLAG,pname_out);
% addpath and userpath will both BREAK compiled code. so this is forbidden:
% userpath(getenv('RU_HOME'));
% however, we need a way for RU instrument ref data files to load from $RU_HOME by default
RU = cris_gran_RU_ncparam(sat_name,l1b_fname,1,pname_out);
RU = cris_gran_RU_ncparam(sat_name,l1b_fname,str2num(saveTotalRU_FLAG),pname_out);
return
ARG ENV_NAME=env
ARG IMAGE
FROM condaforge/mambaforge:4.11.0-0 as environment
RUN apt-get update
RUN mamba install -y --strict-channel-priority -c conda-forge conda-pack
ARG ENV_NAME
ENV ENV_NAME=$ENV_NAME
RUN mamba create -y -n ${ENV_NAME}
RUN conda-pack -n ${ENV_NAME} -o /${ENV_NAME}.tar &&\
mkdir /${ENV_NAME} && cd /${ENV_NAME} && tar xf /${ENV_NAME}.tar &&\
/${ENV_NAME}/bin/conda-unpack
COPY --from=gitlab.ssec.wisc.edu:5555/rayg/gravee/survey:0.4 /${ENV_NAME} /${ENV_NAME}
RUN mamba install -y -p /${ENV_NAME} --strict-channel-priority -c conda-forge flask markdown
RUN mamba install -y -p /${ENV_NAME} --strict-channel-priority -c conda-forge python=3.8 flask markdown
FROM gitlab.ssec.wisc.edu:5555/cris_l1b/user/develop
FROM ${IMAGE}
ARG ENV_NAME
ENV ENV_NAME=$ENV_NAME
COPY --from=environment /$ENV_NAME /$ENV_NAME
RUN echo 'source /$ENV_NAME/bin/activate' > ~/.bashrc
RUN echo "source /$ENV_NAME/bin/activate" > ~/.bashrc
RUN echo 'exec "$@"' > /entrypoint.sh
# --login runs ~/.bashrc which sources /env
ENTRYPOINT ["bash", "--login", "/entrypoint.sh"]
COPY cris_survey_api.yml /cris_survey_api.yml
ENV CSYAML='/cris_survey_api.yml'
ENV OMP_NUM_THREADS=1
ENV MKL_NUM_THREADS=1
ENV OPENBLAS_NUM_THREADS=1
RUN mkdir /output
COPY ru_server.py /app/ru_server.py
COPY README.md /README.md
WORKDIR /app
#RUN chmod a+xwr /
CMD ["python", "ru_server.py"]
COPY --from=environment /$ENV_NAME /$ENV_NAME
Example:
curl --output out.mat 'cris-l1b-ru-test.ssec.wisc.edu/SNDR.SNPP.CRIS.20190806T0312.m06.g033.L1B.std.vXX_XX_XX.T.200529014740.nc'
Use case:
curl --output out.mat 'cris-l1b-ru-test.ssec.wisc.edu/<year>/<month>/<day>/<granule>?source=sounder_sips&instrument=noaa20&resolution=fsr&software_version=3.0.1'
Run case:
docker run --rm -it -v /ships19/crisl1b/data:/ships19/crisl1b/data -p 5432:5432 ru
metadata:
sources:
#atmos_sips: '*asips*'
sounder_sips: '*gesdisc*'
products:
l1b: '*l1b*'
#img: '*img*'
software_versions:
#3.0.1: '*3p0p1*'
#3.0.4: '*3p0p4*'
3.0.1: '*3.0.1*'
3.0.4: '*3.0.4*'
'3.0': '*3.0.x*'
instruments:
noaa20: '*noaa20*'
snpp: '*snpp*'
resolutions:
fsr: '*fsr*'
nsr: '*nsr*'
surveys:
gesdisc_l1b_snpp_nsr_v3.0.4:
username: 'jInAbOVlU9tEiw=='
password: 'NlvKHLxa47mjnQ=='
client_kwargs:
endpoint_url: 'http://survey-source:9000'
default_fill_cache: True
default_cache_type: 'bytes'
cache_regions: True
gesdisc_l1b_snpp_fsr_v3.0.4:
username: 'jInAbOVlU9tEiw=='
password: 'NlvKHLxa47mjnQ=='
client_kwargs:
endpoint_url: 'http://survey-source:9000'
default_fill_cache: True
default_cache_type: 'bytes'
cache_regions: True
gesdisc_l1b_snpp_nsr_v3.0.1:
username: 'jInAbOVlU9tEiw=='
password: 'NlvKHLxa47mjnQ=='
client_kwargs:
endpoint_url: 'http://survey-source:9000'
default_fill_cache: True
default_cache_type: 'bytes'
cache_regions: True
gesdisc_l1b_snpp_fsr_v3.0.x:
username: 'jInAbOVlU9tEiw=='
password: 'NlvKHLxa47mjnQ=='
client_kwargs:
endpoint_url: 'http://survey-source:9000'
default_fill_cache: True
default_cache_type: 'bytes'
cache_regions: True
gesdisc_l1b_snpp_nsr_v3.0.x:
username: 'jInAbOVlU9tEiw=='
password: 'NlvKHLxa47mjnQ=='
client_kwargs:
endpoint_url: 'http://survey-source:9000'
default_fill_cache: True
default_cache_type: 'bytes'
cache_regions: True
gesdisc_l1b_snpp_fsr_v3.0.1:
username: 'jInAbOVlU9tEiw=='
password: 'NlvKHLxa47mjnQ=='
client_kwargs:
endpoint_url: 'http://survey-source:9000'
default_fill_cache: True
default_cache_type: 'bytes'
cache_regions: True
gesdisc_l1b_noaa20_fsr_v3.0.1:
username: 'jInAbOVlU9tEiw=='
password: 'NlvKHLxa47mjnQ=='
client_kwargs:
endpoint_url: 'http://survey-source:9000'
default_fill_cache: True
default_cache_type: 'bytes'
cache_regions: True
from flask import Flask, make_response, request, send_from_directory
import subprocess
import markdown
from gravee.api.cris_survey_api import Survey
import numpy as np
from datetime import datetime
import os
app = Flask(__name__)
def get_survey(request_data={}, **kwargs):
path = request_data.get('path', request.args.get('path'))
source = request_data.get('source', request.args.get('source'))
product = request_data.get('product', request.args.get('product'))
instrument = request_data.get('instrument', request.args.get('instrument'))
resolution = request_data.get('resolution', request.args.get('resolution'))
software_version = request_data.get('software_version', request.args.get('software_version'))
return Survey(path=path, source=source, product=product,
instrument=instrument, resolution=resolution,
software_version=software_version, **kwargs)
def get_full_path(survey, granule):
ver = '.'.join((survey.software_version.split('.') + ['x', 'x', 'x'])[:3])
base = f'/ships19/crisl1b/data/{survey.instrument.lower()}/cris_l1b_{survey.resolution.lower()}/{survey.source.lower()}/{ver}'
year = granule.split('.')[3][:4]
month = granule.split('.')[3][4:6]
day = granule.split('.')[3][6:8]
dt = datetime(int(year), int(month), int(day))
doy = str((dt - datetime(int(year), 1, 1)).days + 1).zfill(3)
return f'{base}/{year}/{doy}/{granule}'
@app.route('/', methods=['GET'])
def readme():
if request.method == 'GET':
......@@ -14,10 +39,15 @@ def readme():
return md_template_string
@app.route('/<granule>', methods=['GET'])
def main(granule):
subprocess.check_output(['bash', '-c', f'cris-l1b-ru SNPP /input/{granule} /output'])
return send_from_directory('/output/', f'{granule.replace(".nc", "")}_RUopts.mat', as_attachment=True)
@app.route('/<year>/<month>/<day>/<granule_num>', methods=['GET'])
def main(year, month, day, granule_num):
survey = get_survey()
day = (np.datetime64(f'{year}-{month.zfill(2)}-{day.zfill(2)}') - np.datetime64('2010')) / np.timedelta64(1, 'D')
fp = get_full_path(survey, str(survey.get_dask('product_name', day=int(day), granule=int(granule_num))))
instrument_map = {'SNPP': 'SNPP', 'NOAA20': 'J1'}
instrument = instrument_map[survey.instrument.upper()]
subprocess.check_output(['bash', '-c', f'cris-l1b-ru {instrument} {fp} /output'])
return send_from_directory('/output/', f'{os.path.basename(fp).replace(".nc", "")}_RU.mat', as_attachment=True)
if __name__ == '__main__':
app.run(host="0.0.0.0", port=5432, debug=True)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment