diff --git a/buildbucket/Dockerfile b/buildbucket/Dockerfile index 586163233bf6568c7a65f5992370e169065f65cb..5bafdf81b31c7198fb1ef686ca0e899eb77ee367 100644 --- a/buildbucket/Dockerfile +++ b/buildbucket/Dockerfile @@ -1,6 +1,6 @@ # To build: # docker build -t gitlab.ssec.wisc.edu:5555/cspp_geo/cspp-geo-gridded-glm/buildbucket:r$(date '+%Y%m%d') . -FROM --platform=linux/amd64 centos:7 +FROM --platform=linux/amd64 rockylinux:8 WORKDIR /work @@ -8,7 +8,7 @@ COPY buildbucket_environment.yaml . COPY package.sh . # Install the `which` command for debugging -RUN yum -y install which git && yum clean all +RUN dnf -y install which git findutils && dnf clean all # Download and install miniforge # https://github.com/conda-forge/miniforge#miniforge diff --git a/buildbucket/buildbucket_environment.yaml b/buildbucket/buildbucket_environment.yaml index 9b98a3b51bfd71800c19d5e2c9149ee19b0c5992..79992b1ab0822bb32df75612a35680798ade5fa0 100644 --- a/buildbucket/buildbucket_environment.yaml +++ b/buildbucket/buildbucket_environment.yaml @@ -25,7 +25,7 @@ dependencies: - shapely - pyorbital - pip: - - git+https://github.com/deeplycloudy/lmatools.git@1501be30c32b948a3ecd56adf1dfb2fc57316346 + - git+https://github.com/deeplycloudy/lmatools.git@392eff5f15735be7b7f5ccc20d2835a617000117 - git+https://github.com/deeplycloudy/stormdrain.git@3620621267215bf84b1f5199b4f395b430f309fc - git+https://github.com/nbearson/glmtools.git@c61202aaaabd91b96cb98b5af7c59f05486cf4c6 - - git+https://github.com/pytroll/satpy.git@v0.36.0 \ No newline at end of file + - git+https://github.com/pytroll/satpy.git@v0.36.0 diff --git a/gridded_glm/libexec/gridded_glm/_minute_gridder.py b/gridded_glm/libexec/gridded_glm/_minute_gridder.py index 4e9cabd8924b4b391712b91057cbc2e0f2b46738..295d64c72cc47471a6a1c049310d188438131f4b 100644 --- a/gridded_glm/libexec/gridded_glm/_minute_gridder.py +++ b/gridded_glm/libexec/gridded_glm/_minute_gridder.py @@ -65,6 +65,9 @@ warnings.filterwarnings("ignore") import dask dask.config.set(num_workers=1) +GOES_SAT_POSITIONS={'G16': 'east', 'G17': 'west', 'G18': 'west', 'G19': 'east',} + + def create_parser(): def bounded_float(afloat, inclusive_min, inclusive_max): value = float(afloat) @@ -118,12 +121,11 @@ def get_resolution(args): # if provided "auto" position, we determine the sensor from the filename def get_goes_position(filenames): - if all("_G16_" in f for f in filenames): - return "east" - if all("_G17_" in f for f in filenames): - return "west" - if all("_G18_" in f for f in filenames): - return "west" + +#Check that all files are the same sat + for key, value in GOES_SAT_POSITIONS.items(): + if all(f"_{key}_" in f for f in filenames): + return value # we require that all files are from the same sensor and raise an exception if not raise ValueError("could not determine GOES position - did you provide a mix of satellites?") @@ -484,4 +486,4 @@ def main(): return 0 if __name__ == '__main__': - sys.exit(main()) \ No newline at end of file + sys.exit(main()) diff --git a/tests/test-inputs/.gitignore b/tests/test-inputs/.gitignore deleted file mode 100644 index 0bee393995d9dc961b2f064b033b21a163fc25c3..0000000000000000000000000000000000000000 --- a/tests/test-inputs/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# ignore all G18 test files until G18 data can be made public -OR_GLM-L2-LCFA_G18* diff --git a/tests/test-inputs/OR_GLM-L2-LCFA_G18_s20223141900000_e20223141900200_c20223141900209.nc b/tests/test-inputs/OR_GLM-L2-LCFA_G18_s20223141900000_e20223141900200_c20223141900209.nc new file mode 100644 index 0000000000000000000000000000000000000000..dda450e4bf796bb06c87004a09d89c67c696c214 --- /dev/null +++ b/tests/test-inputs/OR_GLM-L2-LCFA_G18_s20223141900000_e20223141900200_c20223141900209.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9567ad519143956a5b55250d5c99b28ade9825be057e51b62586ef8fc3bacef9 +size 156288 diff --git a/tests/test-inputs/OR_GLM-L2-LCFA_G18_s20223141900200_e20223141900400_c20223141900415.nc b/tests/test-inputs/OR_GLM-L2-LCFA_G18_s20223141900200_e20223141900400_c20223141900415.nc new file mode 100644 index 0000000000000000000000000000000000000000..0e6e9e9704602023d44d788120ef5cb7acbae0ce --- /dev/null +++ b/tests/test-inputs/OR_GLM-L2-LCFA_G18_s20223141900200_e20223141900400_c20223141900415.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71926b1555ec071d230fe9c6083a92535abe00387480155357af0e1e95dc5ba8 +size 152468 diff --git a/tests/test-inputs/OR_GLM-L2-LCFA_G18_s20223141900400_e20223141901000_c20223141901015.nc b/tests/test-inputs/OR_GLM-L2-LCFA_G18_s20223141900400_e20223141901000_c20223141901015.nc new file mode 100644 index 0000000000000000000000000000000000000000..335c88a67cba0a9e21b48b054e5dc530a148e5fd --- /dev/null +++ b/tests/test-inputs/OR_GLM-L2-LCFA_G18_s20223141900400_e20223141901000_c20223141901015.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2da2fbe749fc94dbc793d311ec7795d7106119b1f29b3cb336e9683976a875d3 +size 156288 diff --git a/tests/test-inputs/goes19_sim/OR_GLM-L2-LCFA_G19_s20240040000000_e20240040000200_c20240040000219.nc b/tests/test-inputs/goes19_sim/OR_GLM-L2-LCFA_G19_s20240040000000_e20240040000200_c20240040000219.nc new file mode 100644 index 0000000000000000000000000000000000000000..b1ae425272000e8a3c86b309b540cc5ce15f68d6 --- /dev/null +++ b/tests/test-inputs/goes19_sim/OR_GLM-L2-LCFA_G19_s20240040000000_e20240040000200_c20240040000219.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c19d812a13cf7112fe47dfaa95c528c6398ca735b2fcbadd771ebd10193464eb +size 345916 diff --git a/tests/test-inputs/goes19_sim/OR_GLM-L2-LCFA_G19_s20240040000200_e20240040000400_c20240040000416.nc b/tests/test-inputs/goes19_sim/OR_GLM-L2-LCFA_G19_s20240040000200_e20240040000400_c20240040000416.nc new file mode 100644 index 0000000000000000000000000000000000000000..432795b43163198800d74dc497681d3c97c44eab --- /dev/null +++ b/tests/test-inputs/goes19_sim/OR_GLM-L2-LCFA_G19_s20240040000200_e20240040000400_c20240040000416.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ac544e1dc2325c11b8a11098a0604a3903887b585b6ce9a22989e8f91aebe71 +size 315196 diff --git a/tests/test-inputs/goes19_sim/OR_GLM-L2-LCFA_G19_s20240040000400_e20240040001000_c20240040001016.nc b/tests/test-inputs/goes19_sim/OR_GLM-L2-LCFA_G19_s20240040000400_e20240040001000_c20240040001016.nc new file mode 100644 index 0000000000000000000000000000000000000000..e528bf470d5851cd8b08e6048f68f931293da47e --- /dev/null +++ b/tests/test-inputs/goes19_sim/OR_GLM-L2-LCFA_G19_s20240040000400_e20240040001000_c20240040001016.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e917d350318a0f8e9e780c1ec906a4ec025505ada5c7d6d0f6bfe5f435b17128 +size 384828 diff --git a/tests/test_functionality.py b/tests/test_functionality.py index ece6033a63ff0ef473a4e72a9a919963bf270d44..6e206631f77a0ad61dab39822fdf29955efdde30 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -131,9 +131,9 @@ def test_g17(): assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G17_T23_20220920192200.nc', 1)) def test_g18(): - inputs = [os.path.join(TESTDIR, "test-inputs/OR_GLM-L2-LCFA_G18_s20222210000000_e20222210000200_c20222210000225.nc"), - os.path.join(TESTDIR, "test-inputs/OR_GLM-L2-LCFA_G18_s20222210000200_e20222210000400_c20222210000424.nc"), - os.path.join(TESTDIR, "test-inputs/OR_GLM-L2-LCFA_G18_s20222210000400_e20222210001000_c20222210001027.nc")] + inputs = [os.path.join(TESTDIR, "test-inputs/OR_GLM-L2-LCFA_G18_s20223141900000_e20223141900200_c20223141900209.nc"), + os.path.join(TESTDIR, "test-inputs/OR_GLM-L2-LCFA_G18_s20223141900200_e20223141900400_c20223141900415.nc"), + os.path.join(TESTDIR, "test-inputs/OR_GLM-L2-LCFA_G18_s20223141900400_e20223141901000_c20223141901015.nc")] for i in inputs: if not os.path.exists(i): warnings.warn(UserWarning("GOES-18 test files are not available in this environment. GOES-18 functionality has not been tested.")) @@ -148,22 +148,58 @@ def test_g18(): ]) assert completed_process.returncode == 0 assert(check_fileglob("CG_GLM-L2-GLMF-M3_G18_*.nc", 1)) - assert(check_fileglob("CSPP_OR_GLM-L2-GLMF-M3_G18_T*.nc", 15)) - - assert(check_fileglob("CG_GLM-L2-GLMF-M3_G18_s20222210000000_e20222210001000_c*.nc", 1)) - assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T04_20220809000100.nc', 1)) - assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T30_20220809000100.nc', 1)) - assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T18_20220809000100.nc', 1)) - assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T29_20220809000100.nc', 1)) - assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T49_20220809000100.nc', 1)) - assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T16_20220809000100.nc', 1)) - assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T27_20220809000100.nc', 1)) - assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T23_20220809000100.nc', 1)) - assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T17_20220809000100.nc', 1)) - assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T20_20220809000100.nc', 1)) - assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T08_20220809000100.nc', 1)) - assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T11_20220809000100.nc', 1)) - assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T21_20220809000100.nc', 1)) - assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T24_20220809000100.nc', 1)) - assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T10_20220809000100.nc', 1)) + assert(check_fileglob("CSPP_OR_GLM-L2-GLMF-M3_G18_T*.nc", 10)) + + assert(check_fileglob("CG_GLM-L2-GLMF-M3_G18_s20223141900000_e20223141901000_c*.nc", 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T11_20221110190100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T18_20221110190100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T19_20221110190100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T24_20221110190100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T26_20221110190100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T30_20221110190100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T37_20221110190100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T45_20221110190100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T49_20221110190100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G18_T52_20221110190100.nc', 1)) + +def test_g19_sim(): + inputs = [os.path.join(TESTDIR, "test-inputs/goes19_sim/OR_GLM-L2-LCFA_G19_s20240040000000_e20240040000200_c20240040000219.nc"), + os.path.join(TESTDIR, "test-inputs/goes19_sim/OR_GLM-L2-LCFA_G19_s20240040000400_e20240040001000_c20240040001016.nc"), + os.path.join(TESTDIR, "test-inputs/goes19_sim/OR_GLM-L2-LCFA_G19_s20240040000200_e20240040000400_c20240040000416.nc")] + for i in inputs: + if not os.path.exists(i): + warnings.warn(UserWarning("GOES-19 sim test files are not available in this environment. GOES-19 functionality has not been tested.")) + return + + completed_process = subprocess.run(['python', + MINUTE_GRIDDER, + "--create-tiles", + inputs[0], + inputs[1], + inputs[2], + ]) + assert completed_process.returncode == 0 + assert(check_fileglob("CG_GLM-L2-GLMF-M3_G19_*.nc", 1)) + assert(check_fileglob("CSPP_OR_GLM-L2-GLMF-M3_G19_T*.nc", 18)) + + assert(check_fileglob("CG_GLM-L2-GLMF-M3_G19_s20240040000000_e20240040001000_c*.nc", 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G19_T10_20240104000100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G19_T11_20240104000100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G19_T14_20240104000100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G19_T17_20240104000100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G19_T27_20240104000100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G19_T30_20240104000100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G19_T33_20240104000100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G19_T34_20240104000100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G19_T35_20240104000100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G19_T38_20240104000100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G19_T39_20240104000100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G19_T40_20240104000100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G19_T41_20240104000100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G19_T46_20240104000100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G19_T47_20240104000100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G19_T52_20240104000100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G19_T53_20240104000100.nc', 1)) + assert(check_fileglob('CSPP_OR_GLM-L2-GLMF-M3_G19_T58_20240104000100.nc', 1)) +