Commit b40c6971 authored by William Roberts's avatar William Roberts
Browse files

Merge branch 'util_refactoring' into 'master'

Update with Util refactoring

See merge request aeri/aeri_armory!12
parents 0fd64127 470fd20d
......@@ -41,8 +41,6 @@ def compare_nc(nc1, nc2, percent_error=0.0, ignore=[]):
for dim in dims:
nc1_dim = nc1.dimensions[dim]
nc2_dim = nc2.dimensions[dim]
if nc1_dim.name != nc2_dim.name:
print(f"{tcolor.GREEN} Dimension {dim} names do not match\n< {nc1_dim.name}\n> {nc2_dim.name}")
if nc1_dim.size != nc2_dim.size:
print(f"{tcolor.GREEN} Dimension {dim} sizes do not match\n< {nc1_dim.size}\n> {nc2_dim.size}")
nc1_dims.remove(dim)
......@@ -98,7 +96,7 @@ def compare_nc(nc1, nc2, percent_error=0.0, ignore=[]):
var1 = nc1[var_key]
var2 = nc2[var_key]
if var2.shape != var1.shape:
print(f"{tcolor.GREEN} Shape {var1.size} and {var2.size} differ{tcolor.ENDC}")
print(f"{tcolor.GREEN} Variable {var_key} shapes do not match\n< {var1.size}\n> {var2.size}{tcolor.ENDC}")
ret = -1
elif var1[:].any() and var2[:].any():
......@@ -119,10 +117,10 @@ def compare_nc(nc1, nc2, percent_error=0.0, ignore=[]):
nc2_variables.remove(var_key)
if nc1_variables:
print(f"{tcolor.GREEN} Attributes {nc1_attrs} only in {nc1.filepath()}{tcolor.ENDC}")
print(f"{tcolor.GREEN} Variables {nc1_variables} only in {nc1.filepath()}{tcolor.ENDC}")
ret = -1
if nc2_variables:
print(f"{tcolor.GREEN} Attributes {nc2_attrs} only in {nc2.filepath()}{tcolor.ENDC}")
print(f"{tcolor.GREEN} Variables {nc2_variables} only in {nc2.filepath()}{tcolor.ENDC}")
ret = -1
return ret
......
......@@ -4,38 +4,41 @@ import os
from datetime import datetime
def getDirs(fp):
# helper for list comprehension
def valid_dir(d):
if os.path.isdir(fp):
try:
datetime.strptime(os.path.basename(d)[:8], "AE%y%m%d")
return True
except ValueError:
pass
return False
# default in case of file passed
dirs = ()
directory = os.path.realpath(fp)
dirs = []
if os.path.isdir(directory):
try:
datetime.strptime(os.path.basename(directory)[:8], "AE%y%m%d")
dirs.append(directory)
except ValueError:
for d in os.listdir(directory):
if os.path.isdir(os.path.join(directory, d)):
try:
datetime.strptime(os.path.basename(d)[:8], "AE%y%m%d")
dirs.append(os.path.join(directory, d))
except ValueError:
pass
if valid_dir(directory):
dirs = (directory, )
elif os.path.isdir(directory):
dirs = tuple(os.path.join(directory, d) for d in os.listdir(directory) if valid_dir(d))
return dirs
def getFiles(fp, extensions=[]):
if os.path.isfile(fp):
tree = {os.path.dirname(os.path.realpath(fp)):[]}
extension = os.path.splitext(fp)[1]
if not extensions:
tree[os.path.dirname(os.path.realpath(fp))].append(os.path.basename(fp))
elif extension.upper() in extensions or extension.lower() in extensions:
tree[os.path.dirname(os.path.realpath(fp))].append(os.path.basename(fp))
else:
tree = {d:[] for d in getDirs(fp)}
for d in tree.keys():
for f in os.listdir(d):
extension = os.path.splitext(f)[1]
if not extensions:
tree[d].append(f)
elif extension.upper() in extensions or extension.lower() in extensions:
tree[d].append(f)
# helper for list comprehension
def valid_file(f):
if os.path.isfile(f):
extension = os.path.splitext(f)[1]
if not extensions or extension.upper() in extensions or extension.lower() in extensions:
return True
return False
tree = {};
fp = os.path.realpath(fp)
if valid_file(fp):
tree = {os.path.dirname(fp) : (os.path.basename(fp),)}
elif os.path.isdir(fp):
tree = {d : tuple(os.path.basename(f) for f in os.listdir(d) if valid_file(os.path.join(d,f))) for d in getDirs(fp)}
return tree
from utils import file_finder
from utils import compare
import os
from netCDF4 import Dataset
FILL_STRING = "foo"
# helper function for creating file structure
def create_dirs(tmp_path, dirs):
# create file structure for testing
for d in dirs:
d.mkdir()
files = (d / f"file1_{d.name[2:]}.RNC", d / f"file2_{d.name[2:]}.txt", d / f"file3_{d.name[2:]}.CXS", d / f"file4_{d.name[2:]}.sum")
for f in files:
f.write_text(FILL_STRING)
assert f.read_text() == FILL_STRING # validate file exists and was written to
def test_file_finder(tmp_path):
# create file structure
dirs = (tmp_path / "AE210101", tmp_path / "AE210102", tmp_path / "AE210103")
create_dirs(tmp_path, dirs)
# run on multiple directories
# check getDirs
found_dirs = file_finder.getDirs(tmp_path)
expected_dirs = (str(d) for d in dirs)
assert(sorted(found_dirs) == sorted(expected_dirs))
# check getFiles
found_files = file_finder.getFiles(tmp_path)
expected_files = {str(d):(f"file1_{d.name[2:]}.RNC", f"file2_{d.name[2:]}.txt", f"file3_{d.name[2:]}.CXS", f"file4_{d.name[2:]}.sum") for d in dirs}
for k in found_files.keys():
assert(sorted(expected_files[k]) == sorted(found_files[k]))
# run on single directory
# check getDirs
found_dirs = file_finder.getDirs(dirs[0]) # run on directory
expected_dirs = (str(dirs[0]),)
assert(found_dirs == expected_dirs)
# check getFiles
found_files = file_finder.getFiles(dirs[0])
expected_files = {str(dirs[0]):(f"file1_{dirs[0].name[2:]}.RNC", f"file2_{dirs[0].name[2:]}.txt", f"file3_{dirs[0].name[2:]}.CXS", f"file4_{dirs[0].name[2:]}.sum")}
for k in found_files.keys():
assert(sorted(expected_files[k]) == sorted(found_files[k]))
# check getFiles with extensions
found_files = file_finder.getFiles(dirs[0], extensions=[".RNC", ".CXS"])
expected_files = {str(dirs[0]):(f"file1_{dirs[0].name[2:]}.RNC", f"file3_{dirs[0].name[2:]}.CXS")}
for k in found_files.keys():
assert(sorted(expected_files[k]) == sorted(found_files[k]))
# run on a single file
# create single file
f = tmp_path / "just_a_file.txt"
f.write_text(FILL_STRING)
assert f.read_text() == FILL_STRING
# getDirs should be empty
found_dirs = file_finder.getDirs(f)
assert(found_dirs == ())
# check getFiles, should only have single dir and file
found_files = file_finder.getFiles(str(f))
expected_files = {os.path.dirname(f) : (str(f.name),)}
assert(found_files == expected_files)
def test_dimensions(tmp_path, capsys):
# test nc
nc_files = [Dataset(str(tmp_path / "nc1.nc"), mode="w"), Dataset(str(tmp_path / "nc2.nc"), mode="w")]
# make dimensions with different shapes
nc_files[0].createDimension("dim1", None) # unlimited
nc_files[1].createDimension("dim1", 1)
nc_files[0].createDimension("only1", 1)
for f in nc_files: f.sync()
ret = compare.main(argv=[str(tmp_path / "nc1.nc"), str(tmp_path / "nc2.nc")])
captured = capsys.readouterr()
print(captured.out)
assert 'Dimension dim1 sizes do not match' in captured.out
assert f"Dimensions ['only1'] only in {str(tmp_path / 'nc1.nc')}" in captured.out
def test_variables(tmp_path, capsys):
# test nc
nc_files = [Dataset(str(tmp_path / "nc1.nc"), mode="w"), Dataset(str(tmp_path / "nc2.nc"), mode="w")]
# make dimensions with different shapes
for f in nc_files:
f.createDimension("dim1", 1)
f.createDimension("dim2", 2)
f.sync()
# two variables with different dimensions
nc_files[0].createVariable("var1", "i", dimensions=["dim1"])
nc_files[1].createVariable("var1", "i", dimensions=["dim2"])
# variable only in one file
nc_files[0].createVariable("var2", "i")
# data will be outside margin of error
nc1_var1 = nc_files[0].createVariable("var3", "i", dimensions=["dim1"])
nc2_var1 = nc_files[1].createVariable("var3", "i", dimensions=["dim1"])
# data will be inside margin of error
nc1_var2 = nc_files[0].createVariable("var4", "i", dimensions=["dim1"])
nc2_var2 = nc_files[1].createVariable("var4", "i", dimensions=["dim1"])
nc1_var1[:] = [2]
nc2_var1[:] = [4]
nc1_var2[:] = [2]
nc2_var2[:] = [3]
for f in nc_files: f.sync()
ret = compare.main(argv=[str(tmp_path / "nc1.nc"), str(tmp_path / "nc2.nc"), "--percent-error", "50"])
captured = capsys.readouterr()
print(captured.out)
assert 'Variable var1 shapes do not match' in captured.out
assert f"Variables ['var2'] only in {str(tmp_path / 'nc1.nc')}" in captured.out
assert f"FP Error: variable var3" in captured.out
assert f"FP Error: variable var4" not in captured.out
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment