Commit 5fbccd58 authored by Joe Taylor's avatar Joe Taylor
Browse files

Merge branch 'develop' into 'master'

interface-changing Matlab updates for RU code intended for end-users

See merge request !3
parents 393b14d4 f1a5996c
Pipeline #37461 passed with stage
in 9 minutes and 10 seconds
*.nc filter=lfs diff=lfs merge=lfs -text
*.mat filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
# Build a Docker image with CI/CD and push to the GitLab registry.
# ref: https://docs.gitlab.com/ee/ci/docker/using_kaniko.html
# Define environment variables for all stages
variables:
GIT_SUBMODULE_STRATEGY: normal
GIT_DEPTH: 1
# additional path in image name, optional (leave empty if not desired)
IMAGE_PATH: ""
docker-build:
image:
name: gcr.io/kaniko-project/executor:debug
entrypoint: [""]
tags:
- crisl1b
- kubernetes
stage: build
before_script:
- |
mkdir -p /kaniko/.docker
echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
# Default branch leaves tag empty (= latest tag)
# All other branches are tagged with the escaped branch name (commit ref slug)
script:
- |
revid=:g${CI_COMMIT_SHA:0:10}
if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then
tag=""
echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'latest'"
else
# tag=":$CI_COMMIT_REF_SLUG"
IMAGE_PATH="$IMAGE_PATH/$CI_COMMIT_REF_SLUG"
tag=""
echo "Running on branch '$CI_COMMIT_BRANCH': image = $IMAGE_PATH"
fi
- /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --push-retry 4 --destination "${CI_REGISTRY_IMAGE}${IMAGE_PATH}${tag}" --destination "${CI_REGISTRY_IMAGE}${IMAGE_PATH}${revid}"
- /kaniko/executor --build-arg IMAGE=${CI_REGISTRY_IMAGE}${IMAGE_PATH}${tag} --context $CI_PROJECT_DIR/serve/ru --dockerfile $CI_PROJECT_DIR/serve/ru/Dockerfile --push-retry 4 --destination "${CI_REGISTRY_IMAGE}${IMAGE_PATH}/ru${tag}" --destination "${CI_REGISTRY_IMAGE}${IMAGE_PATH}/ru${revid}"
# Run this job in a branch where a Dockerfile exists
rules:
- if: $CI_COMMIT_BRANCH
exists:
- Dockerfile
FROM gitlab.ssec.wisc.edu:5555/cris_l1b/glue/buildbucket:r20200324 as matlab
RUN mkdir -p /opt /work
COPY RU /opt/RU
RUN cd /opt/RU \
&& mcc -m ru_main.m
RUN mkdir /work/mcr \
&& cd /work/mcr \
&& curl -sSO https://ssd.mathworks.com/supportfiles/downloads/R2014b/deployment_files/R2014b/installers/glnxa64/MCR_R2014b_glnxa64_installer.zip \
&& unzip MCR_R2014b_glnxa64_installer.zip \
&& mkdir /opt/mcr \
&& ./install -mode silent -agreeToLicense yes -destinationFolder /opt/mcr
FROM centos:7 as shell
RUN yum -y install libXext libXt
ENV RU_HOME=/opt/RU
ENV MCRROOT=/opt/mcr/v84
COPY --from=matlab /opt/mcr /opt/mcr
COPY --from=matlab /opt/RU /opt/RU
RUN ln -s /opt/RU/cris-l1b-ru /usr/local/bin/
# default command, so we get help
CMD ["/opt/RU/cris-l1b-ru"]
......@@ -21,7 +21,8 @@ to set
- the path to the radiometric uncertainty code that you have
downloaded or cloned from gitlab (ru_code_pname)
- the save RU data flag (saveTotalRU_FLAG, set to 1 to save RU file)
- the save RU data flag (saveTotalRU_FLAG, set to 1 to save radiances
and RSS RU to output file; set to 2 to save RSS RU to output file)
- the output path (pname_out)
......
#!/bin/bash -e
usagexit() {
echo >&2 "usage: cris-l1b-ru {SNPP,J1} L1B-FILENAME RU-OUTPUT-DIR"
echo >&2 "docker usage: docker run --rm -v\$PWD:\$PWD -w\$PWD gitlab.ssec.wisc.edu:5555/cris_l1b/user cris-l1b-ru {SNPP,J1} L1B-FILENAME RU-OUTPUT-DIR"
exit 1
}
abspath() {
python -c 'import sys,os; print("\n".join([os.path.abspath(x) for x in sys.argv[1:]]))' "$@"
}
test -d "$RU_HOME" || echo >&2 "${Oops:?RU_HOME is not set}"
test -d "$MCRROOT" || echo >&2 "${Oops:?MCRROOT must point to Matlab runtime}"
SAT="$1"
#INPUT_FILE="$(abspath "$2")"
INPUT_FILE="$2"
OUTPUT_FILE="$3"
test -n "$SAT" || usagexit
test -f "$INPUT_FILE" || echo >&2 "${Oops:?${INPUT_FILE} is not a file}"
test -d "$OUTPUT_FILE" || echo >&2 "${Oops:?please specify an output directory}"
export MATLABPATH="$RU_HOME"
# always limit the blast radius of LD_LIBRARY_PATH to as small a space as possible
export LD_LIBRARY_PATH="$MCRROOT/runtime/glnxa64:$MCRROOT/bin/glnxa64:$MCRROOT/sys/os/glnxa64"
exec "$RU_HOME/run_ru_main.sh" "$MCRROOT" "$SAT" "$INPUT_FILE" "2" "$OUTPUT_FILE"
......@@ -58,7 +58,8 @@ l1b_pname = '/Users/joet/git/cris_l1b_user/RU/TEST_INPUT'; % set to path to v
ru_code_pname = pwd; % set to path of the RU code
saveTotalRU_FLAG = 1; % set to 1 if you want to save the RSS RU to a mat file
saveTotalRU_FLAG = 1; % set to 1 if you want to save the radiances and RSS RU to a mat file
% set to 2 if you want to save the RSS RU to a mat file
pname_out = fullfile(pwd,'RU_OUT_NC'); % set to path for RU output files
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% END: USER INPUT
......
......@@ -8,7 +8,8 @@ function RU = cris_gran_RU_ncparam(sat_name,l1b_fname,saveTotalRU_FLAG,pname_out
% Inputs
% sat_name satellite name ('J1' or 'SNPP')
% l1b_fname L1b granule name (including path if not in same directory as code)
% saveTotalRU_FLAG set to 1 if you want to save the RSS RU to a mat file
% saveTotalRU_FLAG == 1, save radiances and RSS RU to a mat file
% == 2, save RSS RU to a mat file
% pname_out output directory
%
% JKT 2022-Mar-02
......@@ -23,13 +24,20 @@ bands{3} = 'sw';
%%============================================================
%% RU STEP 1a: Load ancillary data from static file (satellite dependent)
%%============================================================
j1jnkfn = 'J1_L1B_ru_tool_param_file.nc';
snppjnkfn = 'SNPP_L1B_ru_tool_param_file.nc';
if (ismcc || isdeployed)
% RKG: addpath does not work with compiled code (MCR) for data file search path
j1jnkfn = fullfile(getenv('RU_HOME'),j1jnkfn);
snppjnkfn = fullfile(getenv('RU_HOME'),snppjnkfn);
end
switch sat_name
case 'J1'
% J1
jnk = rd_nc4('J1_L1B_ru_tool_param_file.nc');
jnk = rd_nc4(j1jnkfn);
case 'SNPP'
% SNPP
jnk = rd_nc4('SNPP_L1B_ru_tool_param_file.nc');
jnk = rd_nc4(snppjnkfn);
end
%%============================================================
%% RU STEP 1b: separate .nc param file contents into ICT_Param, RU,
......@@ -80,7 +88,7 @@ l1b_dat = read_l1b_for_RU(l1b_fname);
%% not needed to calc RU estimate, but used here to record configuration options
%%%%%%%%%%
opts.neonl = l1b_dat.aux.neon_wlen;
save(fullfile(pname_out,sprintf('%sopts.mat',fname_out)), 'opts');
%save(fullfile(pname_out,sprintf('%sopts.mat',fname_out)), 'opts');
%%%%%%%%%%
% store lat and lon data to RU structure
......@@ -276,9 +284,18 @@ for jj = 1:length(bands)
RU_out.(bands{jj}).v = v;
RU_out.(bands{jj}).ru_delta = ru_delta;
RU_out.(bands{jj}).ru_rad_rss = RU.(bands{jj}).ru_rad.total;
RU_out.(bands{jj}).rad = RU.(bands{jj}).rad_polcorr;
if saveTotalRU_FLAG == 1
RU_out.(bands{jj}).rad = RU.(bands{jj}).rad_polcorr;
end
end %%%%%%%%%%%%%%%%%%%% end loop over bands, jj %%%%%%%%%%%%%%%%%%%%
RU_out.opts = opts;
if saveTotalRU_FLAG == 1
fprintf(1,'Saving radiances and Total RU to output file\n');
save(fullfile(pname_out,sprintf('%s.mat',fname_out)), 'RU_out');
elseif saveTotalRU_FLAG == 2
fprintf(1,'Saving Total RU to output file\n');
save(fullfile(pname_out,sprintf('%s.mat',fname_out)), 'RU_out');
end
return
function RU = ru_main(sat_name,l1b_fname,saveTotalRU_FLAG,pname_out);
% addpath and userpath will both BREAK compiled code. so this is forbidden:
% userpath(getenv('RU_HOME'));
% however, we need a way for RU instrument ref data files to load from $RU_HOME by default
RU = cris_gran_RU_ncparam(sat_name,l1b_fname,str2num(saveTotalRU_FLAG),pname_out);
return
*.nc filter=lfs diff=lfs merge=lfs -text
L1B-PCS utilities
=================
## Purpose
Conversion of L1B data into PC Score (L1B_PCS) files storing PC Coefficients for compressed spectra.
This diff is collapsed.
This diff is collapsed.
This source diff could not be displayed because it is stored in LFS. You can view the blob instead.
This source diff could not be displayed because it is stored in LFS. You can view the blob instead.
#!/bin/bash -e
BASE="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
l1a_nc_copy_vars() {
if [[ -d "$CRIS_L1B_HOME" ]]; then
"$CRIS_L1B_HOME/lib/ShellB3/bin/python3" -m sips_cris_l0l1a.netcdf "$@"
elif [[ -x "$(which nc_copy_vars 2>/dev/null)" ]]; then
nc_copy_vars "$@"
else
echo "${Oops:?Unable to find nc_copy_vars utility; is CRIS_L1B_HOME set?}"
fi
}
py_ncrename() {
if [[ -d "$CRIS_L1B_HOME" ]]; then
"$CRIS_L1B_HOME/lib/ShellB3/bin/python3" "${BASE}/nc_rename.py" "$@"
elif [[ -x "$(which python3 2>/dev/null)" ]]; then
python3 "${BASE}/nc_rename.py" "$@"
else
echo "${Oops:?Unable to find python3 with netCDF4 library; is CRIS_L1B_HOME set?}"
fi
}
usage() {
cat <<EOF
Create a L1B_PCS file with PC Scores, given an L1B input and a CRISPCAv0 input.
Usage:
convert-CRISPCAv0-L1B_PCS.sh L1B-file CRISPCAv0-file {optional-output-file}
Example:
/path/to/convert-CRISPCAv0-L1B_PCS.sh \\
input/SNDR.J1.CRIS.20210411T0448.m06.g049.L1B.std.v03_08.G.210411135350.nc \\
input/CRISPCAv0.J1.CRIS.20210411T0448.m06.g049.L1B.std.v03_08.G.210411135350.nc \\
SNDR.J1.CRIS.20210411T0448.m06.g049.L1B_PCS.std.v03_08.G.210411135350.nc
EOF
exit 1
}
default_output_name() {
local fn="$(basename "$1")"
echo "${fn/.L1B./.L1B_PCS.}"
}
FN_L1B="${1}"
FN_PCS="${2}"
FN_L1BPCS="${3:-$(default_output_name "$FN_L1B")}"
FN_GUIDE=${BASE}/anc/cris_l1b_pcs_inherit.nc
FN_TEMPLATE=${BASE}/anc/template.nc
if [[ ! -f "$FN_PCS" ]]; then
usage
exit 1
fi
cp "$FN_TEMPLATE" "${FN_L1BPCS}-"
# reformat the pcs file
cp -f "$FN_PCS" "${FN_L1BPCS}-pcs-"
py_ncrename \
v:globalCoef:rad_global_pcs \
v:localCoef:rad_local_pcs \
v:U:rad_local_pcs_U \
v:M:rad_local_pcs_M \
v:D:rad_local_pcs_D \
d:nFORs:xtrack \
d:nFOVs:fov \
d:nScans:atrack \
d:gpcs:globalpc \
d:pcs:localpc \
d:channels:wnum \
"${FN_L1BPCS}-pcs-"
test ! -f "$FN_L1BPCS" || rm -f "$FN_L1BPCS"
l1a_nc_copy_vars -vv "${FN_L1BPCS}-" / "$FN_L1B" / "$FN_GUIDE"
l1a_nc_copy_vars -vv "${FN_L1BPCS}-" / "${FN_L1BPCS}-pcs-" / "${FN_L1BPCS}-pcs-"
rm -f "${FN_L1BPCS}-pcs-"
mv "${FN_L1BPCS}-" "$FN_L1BPCS"
echo "> $FN_L1BPCS"
#!/usr/bin/env python3
import netCDF4 as nc4
class NcRename:
def __init__(self, filename: str):
super(NcRename, self).__init__()
self.nc = nc4.Dataset(filename, 'r+')
def rename(self, kind: str, f: str, t: str):
if kind.startswith('v'):
self.nc.renameVariable(f, t)
elif kind.startswith('d'):
self.nc.renameDimension(f, t)
elif kind.startswith('g'):
self.nc.renameGroup(f, t)
elif kind.startswith('a'):
self.nc.renameAttribute(f, t)
else:
raise ValueError(f"unknown entity type: {kind}")
def close(self):
self.nc.close()
if __name__ == '__main__':
import sys
changes = [tuple(q.split(':')) for q in sys.argv[1:-1]]
ncr = NcRename(sys.argv[-1])
for knd, frm, to in changes:
ncr.rename(knd, frm, to)
ncr.close()
function d = crisL1B_recrad(globalPC_filename,granule_filename,aflag);
%
% function d = crisL1B_recrad(globalPC,filename,aflag);
%
% A function to produce a structure d with fields analogous to the output of rd_crisL1B
% but containing radiance variables containing reconstructed radiances.
%
% Inputs:
% globalPC_filename: Name of the NetCDF file containing the L1B gloabl PCs. E.g.:
% globalPC_filename = 'globalPCs_L1B_version0_20220204.nc';
%
% granule_filename: Name of file of CrISPCA granule level data. E.g.:
% granule_filename = 'CRISPCAv0.J1.CRIS.20210411T0448.m06.g049.L1B.std.v03_08.G.210411135350.nc';
%
% aflag: Flag to remove Hamming apodization (1) or not (0)
%
% Outputs:
% d.rad_lw(717,9,30,45) LW band reconstructed radiances
% d.rad_mw(869,9,30,45) MW band "
% d.rad_sw(637,9,30,45) SW band "
%
% Other:
% - Uses of pca_expandM.m and removeHamming.m (appended at end)
% - Beware of some NaNs and still some bad points at the ends of the band if aflag=1
% - Execution time per granule is about 0.5s per for aflag=0, 11s for aflag=1
%
% DCT, 15 Feb 2022
%
% Load global PC data
ncfile = globalPC_filename;
globalPC.U = ncread(ncfile,'U'); % PCs (hamming apodized)
globalPC.M = ncread(ncfile,'M'); % Mean (noise normalized) spectrum
globalPC.D = ncread(ncfile,'D'); % Eigenvalues
% Also load the wavenumbers (2211), the NEDN values used for noise
% normalization, and the index vector into the full 2233 spectrum
v = ncread(ncfile,'v');
nedn = ncread(ncfile,'nedn');
index = ncread(ncfile,'index');
% Load the granule-specific data for a sample granule. This includes
% the global Coeficients and the local PCs and local coeficients.
ncfile = granule_filename;
globalCoef = ncread(ncfile,'globalCoef');
localPC.U = ncread(ncfile,'U');
localPC.M = ncread(ncfile,'M');
localPC.D = ncread(ncfile,'D');
localCoef = ncread(ncfile,'localCoef');
% Compute reconstructed radiances
sz = size(localCoef);
globalCoef = globalCoef(:,:);
localCoef = localCoef(:,:);
[~,nrec] = size(localCoef);
nednm = repmat(nedn,1,nrec);
RecRad = ( pca_expandM(globalCoef,globalPC) + pca_expandM(localCoef,localPC) ) .* nednm;
% Remove Hamming if desired. Note this corrupts spectral band end-points
if aflag
RecRad = removeHamming(v,RecRad);
end
% Put the reconstructed radiances back into (nChannels,nFovs,nFORs,nScans) array
sz(1) = length(v);
RecRad = reshape(RecRad,sz);
% And store as normal L1B variable names/arrays
d.rad_lw = ones(717,sz(2),sz(3),sz(4))*NaN;
d.rad_lw(3:715,:,:,:) = RecRad(1:713,:,:,:);
d.rad_mw = ones(869,sz(2),sz(3),sz(4))*NaN;
d.rad_mw(3:867,:,:,:) = RecRad(714:1578,:,:,:);
d.rad_sw = ones(637,sz(2),sz(3),sz(4))*NaN;
d.rad_sw(3:635,:,:,:) = RecRad(1579:2211,:,:,:);
function [x,rc] = pca_expandM( coef, PC )
[ntr,nX] = size(coef);
PCM = repmat(PC.M,1,nX);
col = PC.U(:,1:ntr) * coef + PCM;
x = col;
rc = 0;
function RadOut = removeHamming(v,RadIn);
ind1 = find(v <= 1100);n1 = length(ind1);
ind2 = find(v >= 1190 & v <= 1800);n2 = length(ind2);
ind3 = find(v >= 2100);n3 = length(ind3);
e = ones(n1,1);iham1 = inv(spdiags([e*0.23 e*0.54 e*0.23], -1:1, n1, n1));
e = ones(n2,1);iham2 = inv(spdiags([e*0.23 e*0.54 e*0.23], -1:1, n2, n2));
e = ones(n3,1);iham3 = inv(spdiags([e*0.23 e*0.54 e*0.23], -1:1, n3, n3));
RadOut = RadIn*0;
RadOut(ind1,:) = iham1*double(RadIn(ind1,:));
RadOut(ind2,:) = iham2*double(RadIn(ind2,:));
RadOut(ind3,:) = iham3*double(RadIn(ind3,:));
nBad = 8;
RadOut(ind1(1:nBad),:) = NaN;RadOut(ind1(end-nBad+1:end),:) = NaN;
RadOut(ind2(1:nBad),:) = NaN;RadOut(ind2(end-nBad+1:end),:) = NaN;
RadOut(ind3(1:nBad),:) = NaN;RadOut(ind3(end-nBad+1:end),:) = NaN;
ARG ENV_NAME=env
ARG IMAGE
FROM condaforge/mambaforge:4.11.0-0 as environment
ARG ENV_NAME
COPY --from=gitlab.ssec.wisc.edu:5555/rayg/gravee/survey:0.4 /${ENV_NAME} /${ENV_NAME}
RUN mamba install -y -p /${ENV_NAME} --strict-channel-priority -c conda-forge flask markdown
FROM ${IMAGE}
ARG ENV_NAME
RUN echo "source /$ENV_NAME/bin/activate" > ~/.bashrc
RUN echo 'exec "$@"' > /entrypoint.sh
# --login runs ~/.bashrc which sources /env
ENTRYPOINT ["bash", "--login", "/entrypoint.sh"]
COPY cris_survey_api.yml /cris_survey_api.yml
ENV CSYAML='/cris_survey_api.yml'
ENV OMP_NUM_THREADS=1
ENV MKL_NUM_THREADS=1
ENV OPENBLAS_NUM_THREADS=1
RUN mkdir /output
COPY ru_server.py /app/ru_server.py
COPY README.md /README.md
WORKDIR /app
CMD ["python", "ru_server.py"]
COPY --from=environment /$ENV_NAME /$ENV_NAME
Use case:
curl --output out.mat 'cris-l1b-ru-test.ssec.wisc.edu/<year>/<month>/<day>/<granule>?source=sounder_sips&instrument=noaa20&resolution=fsr&software_version=3.0.1'
Run case:
docker run --rm -it -v /ships19/crisl1b/data:/ships19/crisl1b/data -p 5432:5432 ru
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment