Commit 8174ee5a authored by Marco Kurzynski's avatar Marco Kurzynski
Browse files

Merge branch 'master' into util_refactoring

parents 5c81b442 a88f0aa9
set -ex
image_url="${CI_REGISTRY_IMAGE}"
if [[ $(echo "$CI_COMMIT_TAG" | grep -E '^v[0-9]+.[0-9]+$') ]]; then
echo CI_COMMIT_TAG: "$CI_COMMIT_TAG";
docker pull "${image_url}:latest";
old_tag=$(docker run --rm "${image_url}:latest" bash -c 'echo $AERI_ARMORY_VERSION');
old_primaryv="$(echo ${old_tag%.*} | sed 's/v//')";
new_primaryv="$(echo ${CI_COMMIT_TAG%.*} | sed 's/v//')";
old_lesserv="${old_tag#*.}";
new_lesserv="${CI_COMMIT_TAG#*.}";
docker build --no-cache -f "${DOCKERFILE}" -t "${image_url}:latest" -t "${image_url}:${CI_COMMIT_TAG}" --build-arg AERI_ARMORY_VERSION="${CI_COMMIT_TAG}" ${IMAGE_DIR};
docker run --rm -v $PWD/tests:/tests "${image_url}:latest" test_suite.sh /tests;
docker push ${image_url}:${CI_COMMIT_TAG};
if [[ -z "${old_tag}" ]]; then
docker push ${image_url}:latest;
elif [[ ${new_primaryv} -gt ${old_primaryv} ]]; then
docker push ${image_url}:latest;
elif [[ ${new_primaryv} -eq ${old_primaryv} ]] && [[ ${new_lesserv} -ge ${old_lesserv} ]]; then
docker push ${image_url}:latest;
fi;
fi;
# ref: https://gitlab.ssec.wisc.edu/ssec-dev/brown-bag/blob/master/.gitlab-ci.yml
# Define environment variables for all stages
variables:
# Tell docker images where the host docker daemon is
# and how to talk to it
DOCKER_TLS_CERTDIR: ""
DOCKER_HOST: "tcp://docker:2375"
GIT_SUBMODULE_STRATEGY: recursive
# base template for docker-in-runner
.docker_based_job:
image: docker:19.03.13
tags:
- docker
- ssec_shared
services:
- docker:19.03.13-dind
before_script:
- docker info
- docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}
# specialization to build an image and push back to registry with :latest and :githash versions
.build_image:
extends: .docker_based_job
variables:
DOCKERFILE: "Dockerfile"
IMAGE_NAME: ""
IMAGE_DIR: ""
stage: build image
before_script:
- docker info
- docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}
script:
- ./.gitlab-ci.sh
stages:
- build image
# build aeri_armory when vx.x tags are pushed to master branch.
build_image:
extends: .build_image
only:
- tags
except:
- branches
variables:
IMAGE_NAME: ""
IMAGE_DIR: "."
DOCKERFILE: "Dockerfile"
......@@ -78,6 +78,7 @@ environment for a container to run in.
- Each container is an isolated entity which is not tied to local directories by default.
- The flag `-v` is used to mount a local directory to a docker container.
- You can mount multiple local directories.
- If inside an interactive docker container, type "exit" and enter.
- You can pass environment variables using the `-e` flag.
- You can set the default working directory using the `-w` flag.
......@@ -92,7 +93,7 @@ environment for a container to run in.
# Quick Start
```console
$ docker run --rm -it -v <data_dir>:<destination> aeri_armory
$ docker run --rm -it -v <data_dir>:<destination> aeri_armory UTILITY_SCRIPT ...
```
- `<data_dir>`: Absolute path of directory on your local machine to read data from and write data to.
......@@ -102,7 +103,7 @@ If input and output are not located in the same directory, you can mount both, t
script you decide to run:
```console
$ docker run --rm -it -v <input_dir>:/input -v <output_dir>:/output aeri_armory
$ docker run --rm -it -v <input_dir>:/input -v <output_dir>:/output aeri_armory UTILITY_SCRIPT ...
```
You are not restricted to `/input` and `/output`, they're just intuitive:
......@@ -131,16 +132,16 @@ Run the AERI Quality Control algorithm on AERI data and visualize the results.
### run_aeri_qc.py
```console
usage: run_aeri_qc.py [-h] [--output OUTPUT] [--hint HINT] [-f] [-v] ftp [sci]
usage: run_aeri_qc.py [-h] [--output OUTPUT] [--hint HINT] [-f] [-v] input_dir [sci]
positional arguments:
ftp The directory where the AEYYMMDD ftp subdirectories with the .CXS and .SUM files are located
input_dir The AEYYMMDD directory or the directory containing AEYYMMDD subdirectories where the .CXS and .SUM files are located
sci
optional arguments:
-h, --help show this help message and exit
--output OUTPUT, -o OUTPUT
Directory where QC netcdf files are stored. Defaults to the same directory as ftp
Directory where QC netcdf files are stored. Defaults to the same directory as input_dir
--hint HINT inspect this file and others in similar time period more closely
-f, --force Overwrite any existing QC files in output directory
-v, --verbose Each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG.
......@@ -167,7 +168,7 @@ optional arguments:
```console
$ docker run --rm -it -v <data_dir>:/aeri_qc aeri_armory run_aeri_qc.py /aeri_qc/AEYYMMDD -o /aeri_qc/output -vv -f
INFO:aeri_qc.main:Performing quality control for /aeri_qc/AEYYMMDD/YYMMDDB1.CXS
$ docker run --rm -it -v <data_dir>:/aeri_qc/AEYYMMDD aeri_armory quick_vis.py /aeri_qc/output
$ docker run --rm -it -v <data_dir>:/aeri_qc aeri_armory quick_vis.py /aeri_qc/output
```
Or equivalently
......@@ -313,27 +314,28 @@ Calculate the nonlinearity parameter for the channel 1 detector, for use in nlap
### Output
- b1
- contrl-b1.in
- NLAPP.PAR
- plots
- CXS
- .png files
- IFG
- .png files
- responsivity
- .png files
- f1
- contrl-f1.in
- NLAPP.PAR
- plots
- CXS
- .png files
- IFG
- .png files
- responsivity
- .png files
- NLAPP.SIP
- ARRAYYMMDD
- b1
- contrl-b1.in
- NLAPP.PAR
- plots
- CXS
- .png files
- IFG
- .png files
- responsivity
- .png files
- f1
- contrl-f1.in
- NLAPP.PAR
- plots
- CXS
- .png files
- IFG
- .png files
- responsivity
- .png files
- NLAPP.SIP
### nli.py
......@@ -389,7 +391,7 @@ One of:
### Shell Example
```console
$ docker run --rm -it -v <data_dir>:/dmv_to_ncdf aeri_armory dmv_to_ncdf.py AEYYMMDD -o output
$ docker run --rm -it -v <data_dir>:/dmv_to_ncdf aeri_armory dmv_to_ncdf.py /dmv_to_ncdf/AEYYMMDD -o /dmv_to_ncdf/output
```
# Tests
......
......@@ -25,7 +25,7 @@ aeri_qc () {
quick_vis.py $TEST_DIR/aeri_qc/user_output &>> $output_file
echo 'running diff...'
compare.py $TEST_DIR/aeri_qc/expected_output $TEST_DIR/aeri_qc/user_output --ignore 'history' --percent-error 0.05
compare.py $TEST_DIR/aeri_qc/expected_output $TEST_DIR/aeri_qc/user_output --ignore 'history' --ignore 'aeri_armory_version' --percent-error 0.05
)
result
fi
......
......@@ -71,7 +71,7 @@ def compare_nc(nc1, nc2, percent_error=0.0, ignore=[]):
attr1 = getattr(nc1, attr)
attr2 = getattr(nc2, attr)
if attr1 != attr2:
print(f"{tcolor.GREEN} Attribute {name} differs\n< {attr1}\n> {attr2}{tcolor.ENDC}")
print(f"{tcolor.GREEN} Attribute {attr} differs\n< {attr1}\n> {attr2}{tcolor.ENDC}")
ret = -1
nc1_attrs.remove(attr)
nc2_attrs.remove(attr)
......@@ -230,10 +230,10 @@ def main(argv=sys.argv[1:]):
help="percent of pixels allowed to be different")
parser.add_argument(
"--ignore",
nargs='+',
action='append',
type=str,
default='',
help="list of strings to ignore in text files and ncdf attributes")
default=[],
help="String to ignore in text files and ncdf attributes")
parser.add_argument("input1", help="first file to compare")
parser.add_argument("input2", help="second file to compare")
args = parser.parse_args(argv)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment