Commit 2746762e authored by Marco Kurzynski's avatar Marco Kurzynski Committed by William Roberts
Browse files

Conda and Refactoring

parent fac9e774
# This file uses two stages to build, denoted by FROM. Previous stages are discarded after building.
# This accomplishes the following:
# 1. It makes the code cleaner since we don't handle cleanup
# 2. It makes the container/image smaller since unused build stuff is thrown away
# 3. It is more secure because sensitive information such as ssh keys are no longer accessible.
# This is the install stage. We get files and libraries we need to make
# maerisst.exe. Anything done here will be deleted after build time.
FROM centos:7 as setup
#########################################################
# miniconda base image in centos
FROM centos:7 as centos-miniconda
RUN yum -y update && yum -y install wget
RUN wget https://repo.anaconda.com/miniconda/Miniconda3-py39_4.9.2-Linux-x86_64.sh
RUN bash Miniconda3-py39_4.9.2-Linux-x86_64.sh -b
ENV PATH="/root/miniconda3/bin:${PATH}"
#########################################################
# setup image
FROM centos-miniconda as setup
MAINTAINER Denny Hackel (denny.hackel@ssec.wisc.edu)
# Install dependencies
......@@ -17,78 +19,67 @@ RUN yum -y install libgfortran-static libquadmath-static
RUN yum -y install python36 gcc-gfortran
RUN yum -y install swig gcc-c++ glibc-headers python3-devel
RUN yum -y install libstdc++-devel git unzip
# Copy over data.
COPY .ssh /root/.ssh
COPY scripts /root/scripts
COPY emissivity /cal_val
COPY vips /cal_val
COPY data.zip /root/data.zip
RUN conda install -y -c conda-forge conda-pack netCDF4
# Add ssh keys and known hosts
COPY .ssh /root/.ssh
RUN ssh-keyscan gitlab.ssec.wisc.edu > /root/.ssh/known_hosts
# This is needed on some systems due to security measures.
RUN chmod -R 700 /root/.ssh
# Get source code.
RUN git clone --depth 1 -b m64_r20200617 git@gitlab.ssec.wisc.edu:aeri/ifg_ret.git
RUN git clone --depth 1 -b m64_r20200630 git@gitlab.ssec.wisc.edu:aeri/ifg_tools.git
RUN git clone --depth 1 -b m64_r20210524 git@gitlab.ssec.wisc.edu:cphillips/aeri_quality_control.git
RUN git clone --depth 1 -b m64_r20210420 git@gitlab.ssec.wisc.edu:aeri/ifg_tools.git
RUN git clone --depth 1 -b m64_r20210722 git@gitlab.ssec.wisc.edu:cphillips/aeri_quality_control.git
RUN git clone --depth 1 -b m64_r20200630 git@gitlab.ssec.wisc.edu:cphillips/aeri_tools.git
RUN git clone --depth 1 -b py2to3 git@gitlab.ssec.wisc.edu:aeri/ifg_aeri.git
# The following directories just need to be in the same directory.
RUN mv /ifg_aeri/py/nli /root/nli
RUN mv /ifg_tools/dev/libutil /root/libutil
RUN mv /ifg_tools/dev/ohwhio /root/ohwhio
RUN mv /ifg_tools/dev/dmv /root/dmv
RUN mv /ifg_ret/AERI/dev/maerisst /root/maerisst
COPY cal_val /root/cal_val/cal_val
COPY setup.py /root/cal_val/setup.py
# Build maerisst source code.
RUN cd /root/ohwhio; make -f makefile
# dmv needs to be in the same directory as ohwhio in order to make.
RUN cd /root/libutil; make -f makefile
# Only need maerisst.exe.
RUN cd /root/maerisst; make -f makefile; mv maerisst.exe /root/scripts
# Get data files.
RUN unzip /root/data.zip -d /cal_val
# dmv needs to be in the same directory as ohwhio in order to make.
RUN cd /root/maerisst; make -f makefile
RUN cd /root/dmv; make python3
#RUN cp /ifg_aeri/py/nli/branch/nli.py /root/scripts
# Aliases don't work outside of interative shells.
# Credit: https://superuser.com/questions/1099396/docker-exec-doesnt-load-aliases/1209426
# Make a script that makes it easier to call run_aeri_qc.py.
RUN echo '#!/bin/bash' > /root/scripts/run_aeri_qc
RUN echo 'python3 /aeri_quality_control/run_aeri_qc.py $@' >> /root/scripts/run_aeri_qc
RUN chmod u+x /root/scripts/run_aeri_qc
# Make a script that makes it easier to call quick_vis.py.
RUN echo '#!/bin/bash' > /root/scripts/quick_vis
RUN echo 'python3 /aeri_quality_control/plotting/quick_vis.py $@' >> /root/scripts/quick_vis
RUN chmod u+x /root/scripts/quick_vis
# This is the user stage. Everything done here will be usable and seeable by the user.
# Install dependencies.
RUN cd /root/dmv; pip install .
RUN cd /root/nli; pip install .
RUN cd /aeri_tools; pip install .
RUN cd /root/cal_val; pip install .
RUN cd /aeri_quality_control; pip install .
# Credit: https://pythonspeed.com/articles/conda-docker-image-size
# tarball conda environment to reduce size and increase portability
RUN conda-pack -o /aeri_armory_env.tar && \
mkdir /aeri_armory_env && cd /aeri_armory_env && tar xf /aeri_armory_env.tar && \
rm /aeri_armory_env.tar
RUN /aeri_armory_env/bin/conda-unpack
#########################################################
# deployment image
FROM centos:7
MAINTAINER Denny Hackel (denny.hackel@ssec.wisc.edu)
# Get all files from setup stage.
COPY --from=setup /root/scripts /bin
COPY --from=setup /cal_val /cal_val
COPY --from=setup /root/dmv /root/dmv
COPY --from=setup /aeri_tools /aeri_tools
COPY --from=setup /aeri_quality_control /aeri_quality_control
COPY lib /lib/python3.6/site-packages/
RUN yum -y install libgfortran-static
# Install dependencies.
RUN yum -y update
RUN yum -y install epel-release
RUN yum -y install nano libgfortran python36 netcdf-devel netcdf-fortran-devel seq
#RUN python3.6 -m pip install -U pip setuptools
RUN cd /root/dmv; python3.6 -m easy_install dist/dmv*.egg; rm -rf /root/dmv
RUN cd /aeri_tools; python3.6 -m pip install .
RUN cd /aeri_quality_control; python3.6 -m pip install .
COPY --from=setup /aeri_armory_env /aeri_armory_env
COPY --from=setup /root/maerisst/maerisst.exe /bin
# setup python env
ENV VIRTUAL_ENV="/aeri_armory_env"
ENV PATH="$VIRTUAL_ENV/bin:${PATH}"
# Create directory user will copy data into for processing
RUN mkdir /root/data
RUN chmod -R +x /bin
#########################################################
......@@ -2,9 +2,12 @@ use_ifg is the preffered branch for 32bit work
master is the preffered branch for 64bit work
Note: `"` and `'` are slightly different in bash. When copying and pasting code, some editors will
NOTE: `"` and `'` are slightly different in bash. When copying and pasting code, some editors will
replace one with the other.
NOTE: Floating point errors may occur when testing output. This could propegate to the images produced,
so if the images look the same after a diff fails then that's good enough.
# Remove current image
`docker rmi aeri_armory`
......@@ -19,6 +22,8 @@ NOTE: Every time that an update is made, you must pull or rebuild the current im
source: Absolute path of directory on your local machine in which to store products
*If on windows, use %cd% in place of $PWD*
destination: Absolute path of <source> in docker container. Note that this does not
need to match `<source>`.
......@@ -26,61 +31,73 @@ Be careful when copying and pasting; there is a difference between `, ', and ".
# Run aeri quality control and visualize results
`run_aeri_qc <FILEPATH>`
```console
usage: run_aeri_qc.py [-h] [--output OUTPUT] [--hint HINT] [-f] [-v] ftp [sci]
Files are written in FILEPATH unless specified by --qc_dir.
positional arguments:
ftp The directory where the AEYYMMDD ftp subdirectories with the .CXS and .SUM files are located
sci
FILEPATH: The directory where the AEYYMMDD ftp subdirectories with the .CXS and .SUM files are located.
optional arguments:
-h, --help show this help message and exit
--output OUTPUT, -o OUTPUT
Directory where QC netcdf files are stored. Defaults to the same directory as ftp
--hint HINT inspect this file and others in similar time period more closely
-f, --force Overwrite any existing QC files in output directory
-v, --verbose Each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG.
```
`quick_vis <FILEPATH>`
```console
usage: quick_vis.py [-h] [--output OUTPUT] [--exclude EXCLUDE] filepath
Files are written in FILEPATH.
positional arguments:
filepath path to directory containing QC netcdf output
FILEPATH: The directory where the .nc file is stored. This will be there the AEYYMMDD ftp subdirectories are located
or the filepath specified by --qc_dir.
optional arguments:
-h, --help show this help message and exit
--output OUTPUT, -o OUTPUT
Directory to write images to. Defaults to directory containg QC netcdf files provided
--exclude EXCLUDE, -e EXCLUDE
QC filenames to exlude from quick_vis. A string of QC filenames separated by spaces
```
Interactive shell example:
```
$ docker run --rm -itv $PWD/dev/test_data/aeri_qc:/aeri_qc aeri_armory
# ls /aeri_qc
AE190201 test_output
# run_aeri_qc /aeri_qc -f
```console
$ docker run --rm -itv $PWD/tests/aeri_qc:/aeri_qc aeri_armory
# ls /aeri_qc
20190201QC.nc AE190201 test_output
# quick_vis /aeri_qc
# ls /aeri_qc
20190201QC.nc 20190201REP.txt 20190201VIS.png AE190201 test_output
AE190201 expected_output
# run_aeri_qc.py /aeri_qc -o /aeri_qc/user_output -f
# ls /aeri_qc/user_output
20190201QC.nc
# quick_vis.py /aeri_qc/user_output
# ls /aeri_qc/user_output/
20190201QC.nc 20190201REP.txt 20190201VIS.png
# exit
```
Noninteractive shell example:
```console
$ docker run --rm -itv $PWD/tests/aeri_qc:/aeri_qc aeri_armory run_aeri_qc.py /aeri_qc -o /aeri_qc/user_output -f
$ docker run --rm -itv $PWD/tests/aeri_qc:/aeri_qc aeri_armory quick_vis.py /aeri_qc/user_output
```
$ ls $PWD/dev/test_data/aeri_qc
AE190129 test_output
$ docker run --rm -itv $PWD/dev/test_data/aeri_qc:/aeri_qc aeri_armory run_aeri_qc /aeri_qc -f
$ docker run --rm -itv $PWD/dev/test_data/aeri_qc:/aeri_qc aeri_armory quick_vis /aeri_qc
$ ls $PWD/dev/test_data/aeri_qc
20190201QC.nc 20190201REP.txt 20190201VIS.png AE190201 test_output
```
You can pass multiple shell commands to docker using `bash -c 'command1; command2; ...'`:
```
$ docker run --rm -itv $PWD/dev/test_data/aeri_qc:/aeri_qc aeri_armory bash -c 'run_aeri_qc /aeri_qc -f; quick_vis /aeri_qc'
```console
$ docker run --rm -itv $PWD/tests/aeri_qc:/aeri_qc aeri_armory bash -c 'run_aeri_qc.py /aeri_qc -o /aeri_qc/user_output -f; quick_vis.py /aeri_qc/user_output'
```
You can test aeri quality control output by doing the following:
```
$ docker run --rm -itv $PWD/dev/test_data/aeri_qc:/aeri_qc aeri_armory bash -c 'diff -U 0 -I :history <(ncdump /aeri_qc/test_output/20190201QC.nc) <(ncdump /aeri_qc/20190201QC.nc)'
$ docker run --rm -itv $PWD/dev/test_data/aeri_qc:/aeri_qc aeri_armory diff -U 0 /aeri_qc/test_output/20190201REP.txt /aeri_qc/20190201REP.txt
$ docker run --rm -itv $PWD/dev/test_data/aeri_qc:/aeri_qc aeri_armory diff -U 0 /aeri_qc/test_output/20190201VIS.png /aeri_qc/20190201VIS.png
```console
$ docker run --rm -itv $PWD/tests/aeri_qc:/aeri_qc aeri_armory bash -c 'diff -U 0 -I :history <(ncdump /aeri_qc/expected_output/20190201QC.nc) <(ncdump /aeri_qc/user_output/20190201QC.nc)'
$ docker run --rm -itv $PWD/tests/aeri_qc:/aeri_qc aeri_armory diff -U 0 /aeri_qc/expected_output/20190201REP.txt /aeri_qc/user_output/20190201REP.txt
$ docker run --rm -itv $PWD/tests/aeri_qc:/aeri_qc aeri_armory diff -U 0 /aeri_qc/expected_output/20190201VIS.png /aeri_qc/user_output/20190201VIS.png
```
# maerisst
Interactive shell example:
```
$ docker run --rm -itv $PWD/dev/test_data/maerisst:/maerisst aeri_armory
```console
$ docker run --rm -itv $PWD/tests/maerisst:/maerisst aeri_armory
# cd /maerisst
# maerisst.exe
...
......@@ -91,12 +108,17 @@ $ docker run --rm -itv $PWD/dev/test_data/maerisst:/maerisst aeri_armory
Enter date (YYMMDD or YYYYMMDD): 960318
Enter main-program screen-output parameter.
(1="Display screen messages in MAERISST") 1
...
# mkdir user_output
# mv asciiout.mlt TSKINDMP.ASC user_output
# exit
```
Noninteractive shell example:
```
$ docker run --rm -itv $PWD/dev/test_data/maerisst:/maerisst aeri_armory bash -c 'cd /maerisst; maerisst.exe'
```console
$ docker run --rm -itv $PWD/tests/maerisst:/maerisst aeri_armory \
bash -c 'cd /maerisst; maerisst.exe; \
mkdir user_output; mv asciiout.mlt TSKINDMP.ASC user_output'
...
----------------------------------------------------
$Id: maerisst.for,v 1.2 2005-07-12 15:23:06 benh Exp $
......@@ -108,110 +130,83 @@ $ docker run --rm -itv $PWD/dev/test_data/maerisst:/maerisst aeri_armory bash -c
```
You can test maerisst output by doing the following:
```
$ docker run --rm -itv $PWD/dev/test_data/maerisst:/maerisst aeri_armory diff /maerisst/asciiout.mlt /maerisst/test_output/asciiout.mlt
```console
$ docker run --rm -itv $PWD/tests/maerisst:/maerisst aeri_armory diff -r /maerisst/user_output /maerisst/expected_output
```
# Run cal val
```
usage: cal_val_plots.py [-h] [-w WORK_DIR] [-o OUTPUT_DIR] [-r RECORD_RANGE]
[-e EXCLUDED_RECORDS] [-s] [-n] [-v]
input
```console
usage: cal_val_plots.py [-h] [-o OUTPUT_DIR] [-r RECORD_RANGE] [-e EXCLUDED_RECORDS] [-s] [-n] [-v] input
Generate cal val plots for IBB and NBB. The first record is record 1
positional arguments:
input Channel 1 input filename in HHMMSSC1.RNC format.
Realtive or absolute path
input Channel 1 input filename in HHMMSSC1.RNC format. Realtive or absolute path
optional arguments:
-h, --help show this help message and exit
-w WORK_DIR, --work-dir WORK_DIR
Directory containing bbcal.in files. Defaults to
current directory. Relative or absolute path
-o OUTPUT_DIR, --output-dir OUTPUT_DIR
Output file directory. It will be created if it does
not exist. Defaults to current directory. Relative or
absolute path
Output file directory. It will be created if it does not exist. Defaults to current directory. Relative or absolute path
-r RECORD_RANGE, --record-range RECORD_RANGE
Record range. Either a number or a string of two
numbers separated by a space. Defaults to all records
Record range. Either a number or a string of two numbers separated by a space. Defaults to all records
-e EXCLUDED_RECORDS, --excluded-records EXCLUDED_RECORDS
Records to exclude. Either a number or a string of
numbers separated by spaces
-s, --separate Plot each record individually. Also generates IBB.avg
and NBB.avg files
-n, --no-spectral Skip generation of spectral plots and only plot
thermistor temperatures and diffmeans.
-v, --verbose Each occurrence increases verbosity 1 level through
ERROR-WARNING-INFO-DEBUG.
Records to exclude. Either a number or a string of numbers separated by spaces
-s, --separate Plot each record individually. Also generates IBB.avg and NBB.avg files
-n, --no-spectral Skip generation of spectral plots and only plot thermistor temperatures and diffmeans.
-v, --verbose Each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG.
```
Interactive shell example:
```
$ docker run --rm -it aeri_armory
Mount the test data volume in your docker container. Make sure you are in the aeri_armory directory for PWD
```console
$ docker run --rm -tiv $PWD/tests/cal_val:/cal_val aeri_armory
# cd /cal_val
# cal_val_plots.py 191010C1.RNC -r "1 4" -o /out -vv
[INFO: 2020-09-10 19:07:34 : __main__] Creating directory: /out
# cal_val_plots.py 191010C1.RNC -r "1 4" -o /cal_val/user_output -vv
[INFO: 2020-09-10 19:07:34 : __main__] Creating directory: /cal_val/user_output
[INFO: 2020-09-10 19:07:34 : __main__] start through end records: 1 - 4
[INFO: 2020-09-10 19:07:34 : __main__] excluded records:
[INFO: 2020-09-10 19:07:34 : __main__] Record 1 scene mirror position = 83
[INFO: 2020-09-10 19:07:34 : __main__] Record 4 scene mirror position = 84
[root@b0b03380d819 cal_val]# ls /out
[root@b0b03380d819 cal_val]# ls user_output/
aeri_119_3rdbody_191010.png bbcal.mlt1 bbcal.sum1 bbcal3p.lo1
aeri_119_icebody_191010.png bbcal.mlt2 bbcal.sum2 bbcal3p.lo2
```
Noninteractive shell example:
```console
$ docker run --rm -itv $PWD/tests/cal_val:/cal_val aeri_armory bash -c 'cd /cal_val; cal_val_plots.py 191010C1.RNC -o /cal_val/user_output -r "1 4" -vv'
```
$ docker run --rm -it aeri_armory bash -c 'cd /cal_val; cal_val_plots.py 191010C1.RNC -o /out -r "1 4" -vv; ls /out'
[INFO: 2020-09-10 19:07:34 : __main__] Creating directory: /out
[INFO: 2020-09-10 19:07:34 : __main__] start through end records: 1 - 4
[INFO: 2020-09-10 19:07:34 : __main__] excluded records:
[INFO: 2020-09-10 19:07:34 : __main__] Record 1 scene mirror position = 83
[INFO: 2020-09-10 19:07:34 : __main__] Record 4 scene mirror position = 84
aeri_119_3rdbody_191010.png bbcal.mlt1 bbcal.sum1 bbcal3p.lo1
aeri_119_icebody_191010.png bbcal.mlt2 bbcal.sum2 bbcal3p.lo2
```
If you are not in the directory containing the bbcal files, you can specify it as the working directory (/cal_val).
```
$ docker run --rm -itv $PWD/AE200909_3body:/AE200909_3body aeri_armory bash -c 'cd /AE200909_3body; ls; cal_val_plots.py 200909C1.RNC -w /cal_val -o out'; ls $PWD/AE200909_3body/out
200909.PAR 200909B1.UVS 200909C2.RNC 200909F2.CSV RADIANCE.SCR
200909.QC 200909B2.CXS 200909F1.CSV 200909F2.CXS SUMARY.SCR
200909.SUM 200909B2.UVS 200909F1.CXS 200909F2.UVS
200909B1.CXS 200909C1.RNC 200909F1.UVS AESITTER.SCR
aeri_120_3rdbody_200909.png bbcal.mlt2 bbcal.sum2 bbcal3p.lo2
bbcal.mlt1 bbcal.sum1 bbcal3p.lo1
compare output, diff should return nothing
```console
$ docker run --rm -itv $PWD/tests/cal_val:/cal_val aeri_armory bash -c 'diff -r -I file: /cal_val/user_output/ /cal_val/expected_output/'
```
Note that if bbcal runs into a value error with nan values, it will be skipped and a warning will be displayed
(use -v to show warning message). Output will still be generated, but may be invalid.
```
$ docker run --rm -it aeri_armory bash -c 'cd /cal_val; cal_val_plots.py 191010C1.RNC -v'
```console
$ docker run --rm -itv $PWD/tests/cal_val:/cal_val aeri_armory bash -c 'cd /cal_val; cal_val_plots.py 191010C1.RNC -v'
[WARNING: 2020-09-10 18:45:36 : __main__] Records flagged for missing data: 119, 120. Adding them to excluded records
```
When plotting, if nans are found in the long or short waves, a warning will be printed with which wave numbers
contain nan values. If there is part of the graph with all nans, that file will not be written.
```
$ docker run --rm -itv $PWD/out:/out aeri_armory bash -c 'cd /cal_val; cal_val_plots.py 191010C1.RNC -r "118 120" -v'
```console
$ docker run --rm -itv $PWD/tests/cal_val:/cal_val aeri_armory bash -c 'cd /cal_val; cal_val_plots.py 191010C1.RNC -r "118 120" -v'
[WARNING: 2020-09-10 18:46:45 : __main__] Records flagged for missing data: 119, 120. Adding them to excluded records
[WARNING: 2020-09-10 18:46:46 : __main__] All nans in NBB observed longwave, records 118-118. Skipping plot generation
[WARNING: 2020-09-10 18:46:46 : __main__] nan values found in IBB observed longwave, records 118-118, in the following wave numbers: 1558.2998046875, 1700.0510864257812, 1733.3192443847656
```
The following commands have the same result:
```
$ docker run --rm -itv $PWD/AE200909_3body:/AE200909_3body aeri_armory bash -c 'cd /AE200909_3body; cal_val_plots.py 200909C1.RNC -o out -w /cal_val'
$ docker run --rm -itv $PWD/AE200909_3body:/AE200909_3body aeri_armory bash -c 'cal_val_plots.py /AE200909_3body/200909C1.RNC -o /AE200909_3body/out -w /cal_val'
$ docker run --rm -itv $PWD/AE200909_3body:/AE200909_3body aeri_armory bash -c 'cd /cal_val; cal_val_plots.py /AE200909_3body/200909C1.RNC -o /AE200909_3body/out'
```console
$ docker run --rm -itv $PWD/tests/cal_val:/cal_val aeri_armory bash -c 'cd /cal_val; cal_val_plots.py 191010C1.RNC -o user_output'
$ docker run --rm -itv $PWD/tests/cal_val:/cal_val aeri_armory bash -c 'cal_val_plots.py /cal_val/191010C1.RNC -o /cal_val/user_output'
```
Full use of flags:
```
$ docker run --rm -itv $PWD/AE200909_3body:/AE200909_3body aeri_armory bash -c 'cd /AE200909_3body; cal_val_plots.py 200909C1.RNC -o out -w /cal_val -r "1 10" -e "$(seq 2 2 10)" -snvv'
```console
$ docker run --rm -itv $PWD/tests/cal_val:/cal_val aeri_armory bash -c 'cd /cal_val; cal_val_plots.py 191010C1.RNC -o user_output -r "1 10" -e "$(seq 2 2 10)" -snvv'
[INFO: 2020-09-10 19:34:36 : __main__] start through end records: 1 - 10
[INFO: 2020-09-10 19:34:36 : __main__] excluded records: 2, 4, 6, 8, 10
[INFO: 2020-09-10 19:34:36 : __main__] Record 1 scene mirror position = 83
......@@ -220,3 +215,44 @@ $ docker run --rm -itv $PWD/AE200909_3body:/AE200909_3body aeri_armory bash -c '
[INFO: 2020-09-10 19:34:42 : __main__] Record 7 scene mirror position = 83
[INFO: 2020-09-10 19:34:44 : __main__] Record 9 scene mirror position = 83
```
# Run nli
```console
Usage: nli.py [options] [paramfile]
Based on Fortran NLI Version 2.7 from 27 Jan 97
Options:
-h, --help show this help message and exit
-u Use UW-AERI configuration
-v Enable verbose output
-a Average NLAPP.PAR values
--plotall Plot everything
-p PLOT1 Plot a single data set [i(fg), c(xs), r(esponsivity)]
--zpd1 Use erroneous ft2ifg settings from pre- r2949 ft2ifg.for.
--scale=SCALE Scale plot output from default values
```
Mount the test data volume in your docker container. Make sure you are in the aeri_armory directory for PWD
Make sure to run the script in the environment, not the python2 nli.py in the folder (nli.py not ./nli.py)
Interactive shell example:
```console
$ docker run --rm -tiv $PWD/tests/nli:/nli aeri_armory
# cd /nli
# nli.py -a --plotall contrl4.in contrl5.in
```
Noninteractive shell example:
```console
$ docker run --rm -tiv $PWD/tests/nli:/nli aeri_armory bash -c 'cd /nli; nli.py -a --plotall contrl4.in contrl5.in'
```
Compare output. Floating point errors are exceptable. Read the NOTE at the top of the README
```console
$ docker run --rm -tiv $PWD/tests/nli:/nli aeri_armory bash -c 'cd /nli; mv ARRA190923 user_output; diff -r user_output/ expected_output/'
```
import os
def set_env():
# Chan 1
os.environ.setdefault('C1_IDNBB', 'S')
os.environ.setdefault('C1_IDIBB', 'T')
os.environ.setdefault('C1_NBEMFL', 'nbb-emis.asc')
os.environ.setdefault('C1_IBEMFL', 'ibb-emis.asc')
os.environ.setdefault('C1_CAVFAC2', '39.0')
os.environ.setdefault('C1_WTNB', '0.811 0.189 0.0')
os.environ.setdefault('C1_TOSNB', '0.0 0.0 0.0')
os.environ.setdefault('C1_WTIB', '0.811 0.0945 0.0945')
os.environ.setdefault('C1_TOSIB', '0.0 0.0 0.0')
os.environ.setdefault('C1_NRNGS', '1')
# Chan 1 redeclared for Chan 2
os.environ.setdefault('C2_IDNBB', 'S')
os.environ.setdefault('C2_IDIBB', 'T')
os.environ.setdefault('C2_NBEMFL', 'nbb-emis.asc')
os.environ.setdefault('C2_IBEMFL', 'ibb-emis.asc')
os.environ.setdefault('C2_CAVFAC2', '39.0')
os.environ.setdefault('C2_WTNB', '0.811 0.189 0.0')
os.environ.setdefault('C2_TOSNB', '0.0 0.0 0.0')
os.environ.setdefault('C2_WTIB', '0.811 0.0945 0.0945')
os.environ.setdefault('C2_TOSIB', '0.0 0.0 0.0')
os.environ.setdefault('C2_NRNGS', '1')
# Unique Chan 1
os.environ.setdefault('C1_LOGFIL', 'bbcal3p.lo1')
os.environ.setdefault('C1_MLTFILE', 'bbcal.mlt1')
os.environ.setdefault('C1_SUMFILE', 'bbcal.sum1')
os.environ.setdefault('C1_WN', '900 1100')
# Unique Chan 2
os.environ.setdefault('C2_LOGFIL', 'bbcal3p.lo2')
os.environ.setdefault('C2_MLTFILE', 'bbcal.mlt2')
os.environ.setdefault('C2_SUMFILE', 'bbcal.sum2')
os.environ.setdefault('C2_WN', '2100 2200')
set_env()
......@@ -7,13 +7,14 @@
# FIXME: MeanSigmaPred doesn't come out right
# FIXME: Mean Predicted NBB Uncertainty doesn't come put right
import sys
import logging
import os
from datetime import datetime
import numbers
import math
import AERI_lib
from cal_val import AERI_lib
import numpy as np
logger = logging.getLogger(__name__)
......@@ -47,7 +48,10 @@ def interpolate(xin, yin, xout):
return val
def process(bbcal_file, channel_file, records, NBBtemp=None, IBBtemp=None, abb_breakout=None, output_path=''):
def process(c1_file, c2_file, records, NBBtemp=None, IBBtemp=None, abb_breakout=None, output_path=''):
return process_helper(1, c1_file, records, NBBtemp, IBBtemp, abb_breakout, output_path), process_helper(2, c2_file, records, NBBtemp, IBBtemp, abb_breakout, output_path)
def process_helper(channel_number, channel_file, records, NBBtemp=None, IBBtemp=None, abb_breakout=None, output_path=''):
if isinstance(records, numbers.Number):
records = [records]
......@@ -57,11 +61,18 @@ def process(bbcal_file, channel_file, records, NBBtemp=None, IBBtemp=None, abb_
# Maximum number of ranges
MAXRNGS = 10
fp = open(bbcal_file, 'r')
lines = fp.readlines()
if channel_number == 1:
prefix = 'C1_'
elif channel_number == 2:
prefix = 'C2_'
else:
prefix = ''
logger.error("Invalid channel_number, must be 1 or 2")
# line[5] Log file
LOGFIL = os.path.join(output_path, str.strip(lines[5]))
LOGFIL = os.path.join(output_path, str.strip(os.environ.get(prefix+'LOGFIL')))
# Open logfile
lf = open(LOGFIL, 'w')
......@@ -75,20 +86,21 @@ def process(bbcal_file, channel_file, records, NBBtemp=None, IBBtemp=None, abb_
lf.write("INPUT RECORD RANGE: {0}\n".format(records))
# NBB & IBB Scene IDs
IDNBB = str.strip(lines[6])
IDNBB = str.strip(os.environ.get(prefix+'IDNBB'))
lf.write("NBB identifier: %s\n" % IDNBB)
IDIBB = str.strip(lines[7])
IDIBB = str.strip(os.environ.get(prefix+'IDIBB'))
lf.write("IBB identifier: %s\n" % IDIBB)
# NBB & IBB Emissivity Files
NBEMFL = str.strip(lines[8])
NBEMFL = str.strip(os.environ.get(prefix+'NBEMFL'))
lf.write("NBB EMISSIVITY FILE: %s\n" % NBEMFL)
IBEMFL = str.strip(lines[9])
IBEMFL = str.strip(os.environ.get(prefix+'IBEMFL'))
lf.write("IBB EMISSIVITY FILE: %s\n" % IBEMFL)
# Read NBB & IBB Emissivity Files
emfp = open(NBEMFL, 'r')
imfp = open(IBEMFL, 'r')
d = os.path.dirname(sys.modules[__name__].__file__)
emfp = open(os.path.join(d, "emissivity/"+NBEMFL), 'r')
imfp = open(os.path.join(d, "emissivity/"+IBEMFL), 'r')
# Read first line for Emissivity Files
lf.write("FIRST LINE OF NBB EMISSIVITY-SPECTRUM FILE:\n")
......@@ -113,50 +125,50 @@ def process(bbcal_file, channel_file, records, NBBtemp=None, IBBtemp=None, abb_
imfp.close()
# BB Cavity Factor
CAVFAC2 = float(str.strip(lines[10]))
CAVFAC2 = float(str.strip(os.environ.get(prefix+'CAVFAC2')))
lf.write("BB CAVITY FACTOR: %f\n" % CAVFAC2)
RCF = 1.0 / CAVFAC2
# NBB Apex, Bottom and Top weightings
WTNBA = float(str.split(str.strip(lines[11]))[0])
WTNBB = float(str.split(str.strip(lines[11]))[1])
WTNBT = float(str.split(str.strip(lines[11]))[2])
WTNBA = float(str.split(str.strip(os.environ.get(prefix+'WTNB')))[0])
WTNBB = float(str.split(str.strip(os.environ.get(prefix+'WTNB')))[1])
WTNBT = float(str.split(str.strip(os.environ.get(prefix+'WTNB')))[2])
# Write Weighting to Logfile
lf.write("NBB Weights: %1.5f %1.5f %1.5f\n" % (WTNBA, WTNBB, WTNBT))
# NBB Apex, Bottom and Top offsets
TOSNBA = float(str.split(str.strip(lines[12]))[0])
TOSNBB = float(str.split(str.strip(lines[12]))[1])
TOSNBT = float(str.split(str.strip(lines[12]))[2])
TOSNBA = float(str.split(str.strip(os.environ.get(prefix+'TOSNB')))[0])
TOSNBB = float(str.split(str.strip(os.environ.get(prefix+'TOSNB')))[1])
TOSNBT = float(str.split(str.strip(os.environ.get(prefix+'TOSNB')))[2])
# Write Offsets to Logfile
lf.write("NBB Offsets: %1.5f %1.5f %1.5f\n" % (TOSNBA, TOSNBB, TOSNBT))
# IBB Apex, Bottom and Top weightings
WTIBA = float(str.split(str.strip(lines[13]))[0])
WTIBB = float(str.split(str.strip(lines[13]))[1])
WTIBT = float(str.split(str.strip(lines[13]))[2])
WTIBA = float(str.split(str.strip(os.environ.get(prefix+'WTIB')))[0])
WTIBB = float(str.split(str.strip(os.environ.get(prefix+'WTIB')))[1])
WTIBT = float(str.split(str.strip(os.environ.get(prefix+'WTIB')))[2])
# Write Weighting to Logfile
lf.write("IBB Weights: %1.5f %1.5f %1.5f\n" % (WTIBA, WTIBB, WTIBT))
# IBB Apex, Bottom and Top offsets
TOSIBA = float(str.split(str.strip(lines[