Commit 7ad76387 authored by William Roberts's avatar William Roberts
Browse files

Add specifiers to RU server

parent 8b435752
Pipeline #37138 passed with stage
in 7 minutes and 8 seconds
......@@ -2,29 +2,28 @@ ARG ENV_NAME=env
ARG IMAGE=gitlab.ssec.wisc.edu:5555/cris_l1b/user/feat-serve-ru:latest
FROM condaforge/mambaforge:4.11.0-0 as environment
RUN apt-get update
RUN mamba install -y --strict-channel-priority -c conda-forge conda-pack
ARG ENV_NAME
ENV ENV_NAME=$ENV_NAME
RUN mamba create -y -n ${ENV_NAME}
RUN conda-pack -n ${ENV_NAME} -o /${ENV_NAME}.tar &&\
mkdir /${ENV_NAME} && cd /${ENV_NAME} && tar xf /${ENV_NAME}.tar &&\
/${ENV_NAME}/bin/conda-unpack
COPY --from=gitlab.ssec.wisc.edu:5555/rayg/gravee/survey /${ENV_NAME} /${ENV_NAME}
RUN mamba install -y -p /${ENV_NAME} --strict-channel-priority -c conda-forge flask markdown
RUN mamba install -y -p /${ENV_NAME} --strict-channel-priority -c conda-forge python=3.8 flask markdown
FROM ${IMAGE}
ARG ENV_NAME
ENV ENV_NAME=$ENV_NAME
COPY --from=environment /$ENV_NAME /$ENV_NAME
RUN echo 'source /$ENV_NAME/bin/activate' > ~/.bashrc
RUN echo "source /$ENV_NAME/bin/activate" > ~/.bashrc
RUN echo 'exec "$@"' > /entrypoint.sh
# --login runs ~/.bashrc which sources /env
ENTRYPOINT ["bash", "--login", "/entrypoint.sh"]
COPY cris_survey_api.yml /cris_survey_api.yml
ENV CSYAML='/cris_survey_api.yml'
ENV OMP_NUM_THREADS=1
ENV MKL_NUM_THREADS=1
ENV OPENBLAS_NUM_THREADS=1
RUN mkdir /output
COPY ru_server.py /app/ru_server.py
COPY README.md /README.md
WORKDIR /app
CMD ["python", "ru_server.py"]
ENV saveTotalRU_FLAG=1
COPY --from=environment /$ENV_NAME /$ENV_NAME
Example:
curl --output out.mat 'cris-l1b-ru-test.ssec.wisc.edu/SNDR.SNPP.CRIS.20190806T0312.m06.g033.L1B.std.vXX_XX_XX.T.200529014740.nc'
Use case:
curl --output out.mat 'cris-l1b-ru-test.ssec.wisc.edu/<year>/<month>/<day>/<granule>?source=sounder_sips&instrument=noaa20&resolution=fsr&software_version=3.0.1'
metadata:
sources:
#atmos_sips: '*asips*'
sounder_sips: '*gesdisc*'
products:
l1b: '*l1b*'
#img: '*img*'
software_versions:
#3.0.1: '*3p0p1*'
#3.0.4: '*3p0p4*'
3.0.1: '*3.0.1*'
3.0.4: '*3.0.4*'
'3.0': '*3.0.x*'
instruments:
noaa20: '*noaa20*'
snpp: '*snpp*'
resolutions:
fsr: '*fsr*'
nsr: '*nsr*'
surveys:
gesdiscl1bnoaa20fsrv3p0p1:
username: 'RWCRISCRISTOFFERSUN'
password: 'wqE2BXLzVmga/qV7eJMrKPSvZLNcECmtvb42iibJ'
client_kwargs:
endpoint_url: 'http://127.0.0.1:9000'
default_fill_cache: True
default_cache_type: 'bytes'
cache_regions: True
gesdiscl1bsnppfsrv3p0p4:
username: 'RWCRISCRISTOFFERSUN'
password: 'wqE2BXLzVmga/qV7eJMrKPSvZLNcECmtvb42iibJ'
client_kwargs:
endpoint_url: 'http://127.0.0.1:9000'
default_fill_cache: True
default_cache_type: 'bytes'
cache_regions: True
/ships19/crisl1b/data/survey/gesdisc_l1b_snpp_nsr_v3.0.4:
/ships19/crisl1b/data/survey/gesdisc_l1b_snpp_fsr_v3.0.4:
/ships19/crisl1b/data/survey/gesdisc_l1b_snpp_nsr_v3.0.1:
/ships19/crisl1b/data/survey/gesdisc_l1b_snpp_fsr_v3.0.x:
/ships19/crisl1b/data/survey/gesdisc_l1b_snpp_nsr_v3.0.x:
/ships19/crisl1b/data/survey/gesdisc_l1b_snpp_fsr_v3.0.1:
/ships19/crisl1b/data/survey/gesdisc_l1b_noaa20_fsr_v3.0.1:
from flask import Flask, make_response, request, send_from_directory
import subprocess
import markdown
from gravee.api.cris_survey_api import Survey
import numpy as np
from datetime import datetime
import os
app = Flask(__name__)
def get_survey(request_data={}, **kwargs):
path = request_data.get('path', request.args.get('path'))
source = request_data.get('source', request.args.get('source'))
product = request_data.get('product', request.args.get('product'))
instrument = request_data.get('instrument', request.args.get('instrument'))
resolution = request_data.get('resolution', request.args.get('resolution'))
software_version = request_data.get('software_version', request.args.get('software_version'))
return Survey(path=path, source=source, product=product,
instrument=instrument, resolution=resolution,
software_version=software_version, **kwargs)
def get_full_path(survey, granule):
ver = '.'.join((survey.software_version.split('.') + ['x', 'x', 'x'])[:3])
base = f'/ships19/crisl1b/data/{survey.instrument.lower()}/cris_l1b_{survey.resolution.lower()}/{survey.source.lower()}/{ver}'
year = granule.split('.')[3][:4]
month = granule.split('.')[3][4:6]
day = granule.split('.')[3][6:8]
dt = datetime(int(year), int(month), int(day))
doy = str((dt - datetime(int(year), 1, 1)).days + 1).zfill(3)
return f'{base}/{year}/{doy}/{granule}'
@app.route('/', methods=['GET'])
def readme():
if request.method == 'GET':
......@@ -14,10 +39,15 @@ def readme():
return md_template_string
@app.route('/<granule>', methods=['GET'])
def main(granule):
subprocess.check_output(['bash', '-c', f'cris-l1b-ru SNPP /input/{granule} /output'])
return send_from_directory('/output/', f'{granule.replace(".nc", "")}_RUopts.mat', as_attachment=True)
@app.route('/<year>/<month>/<day>/<granule_num>', methods=['GET'])
def main(year, month, day, granule_num):
survey = get_survey()
day = (np.datetime64(f'{year}-{month.zfill(2)}-{day.zfill(2)}') - np.datetime64('2010')) / np.timedelta64(1, 'D')
fp = get_full_path(survey, str(survey.get_dask('product_name', day=int(day), granule=int(granule_num))))
instrument_map = {'SNPP': 'SNPP', 'NOAA20': 'J1'}
instrument = instrument_map[survey.instrument.upper()]
subprocess.check_output(['bash', '-c', f'cris-l1b-ru {instrument} {fp} /output'])
return send_from_directory('/output/', f'{os.path.basename(fp).replace(".nc", "")}_RU.mat', as_attachment=True)
if __name__ == '__main__':
app.run(host="0.0.0.0", port=5432, debug=True)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment