Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
M
MVCM
Manage
Activity
Members
Labels
Plan
Issues
1
Issue boards
Milestones
Wiki
Code
Merge requests
0
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package Registry
Container Registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Paolo Veglio
MVCM
Commits
ae315f28
Commit
ae315f28
authored
2 years ago
by
Paolo Veglio
Browse files
Options
Downloads
Patches
Plain Diff
implemented first batch of testing for the spectral tests
parent
8ca96f22
No related branches found
No related tags found
No related merge requests found
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
mvcm/spectral_tests.py
+8
-11
8 additions, 11 deletions
mvcm/spectral_tests.py
tests/test_spectral_tests.py
+58
-9
58 additions, 9 deletions
tests/test_spectral_tests.py
with
66 additions
and
20 deletions
mvcm/spectral_tests.py
+
8
−
11
View file @
ae315f28
...
...
@@ -111,20 +111,17 @@ class CloudTests(object):
return
cmin
,
kwargs
[
'
test_bit
'
]
@run_if_test_exists_for_scene
@run_if_test_exists_for_scene
(
'
SST_Test
'
)
def
sst_test
(
self
,
band31
:
str
,
band32
:
str
,
cmin
:
np
.
ndarray
,
test_name
:
str
=
'
SST_Test
'
)
->
np
.
ndarray
:
**
kwargs
)
->
np
.
ndarray
:
confidence
=
np
.
ones
(
self
.
data
[
band31
].
shape
)
qa_bit
=
np
.
zeros
(
self
.
data
[
band31
].
shape
)
test_bit
=
np
.
zeros
(
self
.
data
[
band31
].
shape
)
threshold
=
self
.
thresholds
[
self
.
scene_name
][
test_name
]
threshold
=
kwargs
[
'
thresholds
'
]
if
(
threshold
[
'
perform
'
]
is
True
and
self
.
pixels_in_scene
is
True
):
qa_bit
[
self
.
scene_idx
]
=
1
kwargs
[
'
qa_bit
'
]
[
self
.
scene_idx
]
=
1
m31
=
self
.
data
[
band31
].
values
-
273.16
bt_diff
=
self
.
data
[
band31
].
values
-
self
.
data
[
band32
].
values
sst
=
self
.
data
.
sst
.
values
-
273.16
...
...
@@ -137,14 +134,14 @@ class CloudTests(object):
idx
=
np
.
nonzero
((
sfcdif
<
threshold
[
'
thr
'
][
1
])
&
(
self
.
data
[
self
.
scene_name
]
==
1
))
test_bit
[
idx
]
=
1
kwargs
[
'
test_bit
'
]
[
idx
]
=
1
print
(
f
'
Testing
"
{
self
.
scene_name
}
"
\n
'
)
confidence
[
self
.
scene_idx
]
=
conf
.
conf_test_new
(
sfcdif
[
self
.
scene_idx
],
threshold
[
'
thr
'
])
kwargs
[
'
confidence
'
]
[
self
.
scene_idx
]
=
conf
.
conf_test_new
(
sfcdif
[
self
.
scene_idx
],
threshold
[
'
thr
'
])
cmin
=
np
.
fmin
(
cmin
,
confidence
)
cmin
=
np
.
fmin
(
cmin
,
kwargs
[
'
confidence
'
]
)
# return cmin, np.abs(1-test_bit)*qa_bit
return
cmin
,
test_bit
return
cmin
,
kwargs
[
'
test_bit
'
]
@run_if_test_exists_for_scene
def
bt_diff_86_11um
(
self
,
...
...
This diff is collapsed.
Click to expand it.
tests/test_spectral_tests.py
+
58
−
9
View file @
ae315f28
...
...
@@ -5,22 +5,31 @@ import numpy as np
import
pytest
# from typing import Dict
# from attrs import define, field, Factory
import
mvcm.spectral_tests
as
tst
# CREATE PATHS
@pytest.fixture
def
fixturepath
():
return
os
.
path
.
join
(
os
.
path
.
dirname
(
__file__
),
'
fixtures
'
)
@pytest.fixture
def
data_path
():
return
'
/ships19/hercules/pveglio/mvcm_cleanup
'
# SET FILENAME FIXTURES
@pytest.fixture
def
thresholds_file
(
fixturepath
):
return
os
.
path
.
join
(
fixturepath
,
'
thresholds.mvcm.snpp.v0.0.1.yaml
'
)
@pytest.fixture
def
data_path
(
):
return
'
/ships19/hercules/pveglio/mvcm_cleanup
'
def
ref_confidence_file
(
data_path
):
return
os
.
path
.
join
(
data_path
,
'
ref_confidence.nc
'
)
@pytest.fixture
...
...
@@ -28,17 +37,57 @@ def data_file(data_path):
return
os
.
path
.
join
(
data_path
,
'
viirs_data_A2022173.1312.nc
'
)
def
test_11um_test
(
data_file
,
thresholds_file
):
with
open
(
thresholds_file
,
'
r
'
)
as
f
:
cfg_text
=
f
.
read
()
thresholds
=
yaml
.
safe_load
(
cfg_text
)
# SET DATA FIXTURES
@pytest.fixture
def
thresholds
(
thresholds_file
):
return
yaml
.
safe_load
(
open
(
thresholds_file
))
@pytest.fixture
def
data
(
data_file
):
return
xr
.
open_dataset
(
data_file
)
@pytest.fixture
def
ref_confidence
(
ref_confidence_file
):
return
xr
.
open_dataset
(
ref_confidence_file
)
viirs_data
=
xr
.
open_dataset
(
data_file
)
cmin
=
np
.
ones
(
viirs_data
.
latitude
.
shape
)
def
test_11um_test
(
data
,
thresholds
,
ref_confidence
):
cmin
=
np
.
ones
(
data
.
latitude
.
shape
)
for
scene_name
in
[
'
Ocean_Day
'
,
'
Ocean_Night
'
,
'
Polar_Day_Ocean
'
,
'
Polar_Night_Ocean
'
]:
SceneType
=
tst
.
CloudTests
(
data
=
viirs_
data
,
SceneType
=
tst
.
CloudTests
(
data
=
data
,
scene_name
=
scene_name
,
thresholds
=
thresholds
)
cmin
,
bit
=
SceneType
.
test_11um
(
'
M15
'
,
cmin
)
assert
np
.
allclose
(
cmin
,
ref_confidence
.
bt11um_confidence
.
values
)
def
test_surface_temperature_test
(
data
,
thresholds
,
ref_confidence
):
cmin
=
np
.
ones
(
data
.
latitude
.
shape
)
for
scene_name
in
[
'
Land_Night
'
,
'
Polar_Night_Land
'
]:
SceneType
=
tst
.
CloudTests
(
data
=
data
,
scene_name
=
scene_name
,
thresholds
=
thresholds
)
cmin
,
bit
=
SceneType
.
surface_temperature_test
(
'
M15
'
,
data
,
cmin
)
assert
np
.
allclose
(
cmin
,
ref_confidence
.
surface_temperature_confidence
.
values
)
def
test_sst_test
(
data
,
thresholds
,
ref_confidence
):
cmin
=
np
.
ones
(
data
.
latitude
.
shape
)
for
scene_name
in
[
'
Ocean_Day
'
,
'
Ocean_Night
'
,
'
Polar_Day_Ocean
'
,
'
Polar_Night_Ocean
'
]:
SceneType
=
tst
.
CloudTests
(
data
=
data
,
scene_name
=
scene_name
,
thresholds
=
thresholds
)
cmin
,
bit
=
SceneType
.
sst_test
(
'
M15
'
,
'
M16
'
,
cmin
)
assert
np
.
allclose
(
cmin
,
ref_confidence
.
sst_confidence
.
values
)
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment