Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

print MAE for dp test #2310

Merged
merged 2 commits into from
Feb 12, 2023
Merged

print MAE for dp test #2310

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
309eaa6
print MAE when testing
njzjz Feb 10, 2023
d548c1d
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Feb 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 54 additions & 11 deletions deepmd/entrypoints/test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,8 +156,8 @@ def test(
log.info("# ----------------------------------------------- ")


def rmse(diff: np.ndarray) -> np.ndarray:
"""Calculate average root mean square error.
def mae(diff: np.ndarray) -> float:
"""Calcalte mean absulote error.

Parameters
----------
Expand All @@ -166,8 +166,24 @@ def rmse(diff: np.ndarray) -> np.ndarray:

Returns
-------
np.ndarray
array with normalized difference
float
mean absulote error
"""
return np.mean(np.abs(diff))


def rmse(diff: np.ndarray) -> float:
"""Calculate root mean square error.

Parameters
----------
diff : np.ndarray
difference

Returns
-------
float
root mean square error
"""
return np.sqrt(np.average(diff * diff))

Expand Down Expand Up @@ -289,25 +305,38 @@ def test_ener(
ae = ae.reshape([numb_test, -1])
av = av.reshape([numb_test, -1])

rmse_e = rmse(energy - test_data["energy"][:numb_test].reshape([-1, 1]))
rmse_f = rmse(force - test_data["force"][:numb_test])
rmse_v = rmse(virial - test_data["virial"][:numb_test])
diff_e = energy - test_data["energy"][:numb_test].reshape([-1, 1])
mae_e = mae(diff_e)
rmse_e = rmse(diff_e)
diff_f = force - test_data["force"][:numb_test]
mae_f = mae(diff_f)
rmse_f = rmse(diff_f)
diff_v = virial - test_data["virial"][:numb_test]
mae_v = mae(diff_v)
rmse_v = rmse(diff_v)
mae_ea = mae_e / natoms
rmse_ea = rmse_e / natoms
mae_va = mae_v / natoms
rmse_va = rmse_v / natoms
if has_atom_ener:
rmse_ae = rmse(
test_data["atom_ener"][:numb_test].reshape([-1]) - ae.reshape([-1])
)
diff_ae = test_data["atom_ener"][:numb_test].reshape([-1]) - ae.reshape([-1])
mae_ae = mae(diff_ae)
rmse_ae = rmse(diff_ae)

# print ("# energies: %s" % energy)
log.info(f"# number of test data : {numb_test:d} ")
log.info(f"Energy MAE : {mae_e:e} eV")
log.info(f"Energy RMSE : {rmse_e:e} eV")
log.info(f"Energy MAE/Natoms : {mae_ea:e} eV")
log.info(f"Energy RMSE/Natoms : {rmse_ea:e} eV")
log.info(f"Force MAE : {mae_f:e} eV/A")
log.info(f"Force RMSE : {rmse_f:e} eV/A")
if data.pbc:
log.info(f"Virial MAE : {mae_v:e} eV")
log.info(f"Virial RMSE : {rmse_v:e} eV")
log.info(f"Virial MAE/Natoms : {mae_va:e} eV")
log.info(f"Virial RMSE/Natoms : {rmse_va:e} eV")
if has_atom_ener:
log.info(f"Atomic ener MAE : {mae_ae:e} eV")
log.info(f"Atomic ener RMSE : {rmse_ae:e} eV")

if detail_file is not None:
Expand Down Expand Up @@ -371,8 +400,15 @@ def test_ener(
append=append_detail,
)
return {
"mae_e": (mae_e, energy.size),
"mae_ea": (mae_ea, energy.size),
"mae_f": (mae_f, force.size),
"mae_v": (mae_v, virial.size),
"mae_va": (mae_va, virial.size),
"rmse_e": (rmse_e, energy.size),
"rmse_ea": (rmse_ea, energy.size),
"rmse_f": (rmse_f, force.size),
"rmse_v": (rmse_v, virial.size),
"rmse_va": (rmse_va, virial.size),
}

Expand All @@ -385,8 +421,15 @@ def print_ener_sys_avg(avg: Dict[str, float]):
avg : np.ndarray
array with summaries
"""
log.info(f"Energy MAE : {avg['mae_e']:e} eV")
log.info(f"Energy RMSE : {avg['rmse_e']:e} eV")
log.info(f"Energy MAE/Natoms : {avg['mae_ea']:e} eV")
log.info(f"Energy RMSE/Natoms : {avg['rmse_ea']:e} eV")
log.info(f"Force MAE : {avg['mae_f']:e} eV/A")
log.info(f"Force RMSE : {avg['rmse_f']:e} eV/A")
log.info(f"Virial MAE : {avg['mae_v']:e} eV")
log.info(f"Virial RMSE : {avg['rmse_v']:e} eV")
log.info(f"Virial MAE/Natoms : {avg['mae_va']:e} eV")
log.info(f"Virial RMSE/Natoms : {avg['rmse_va']:e} eV")


Expand Down
26 changes: 18 additions & 8 deletions deepmd/utils/weight_avg.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
from collections import (
defaultdict,
)
from typing import (
TYPE_CHECKING,
Dict,
Expand All @@ -10,30 +13,37 @@


def weighted_average(errors: List[Dict[str, Tuple[float, float]]]) -> Dict:
"""Compute wighted average of prediction errors for model.
"""Compute wighted average of prediction errors (MAE or RMSE) for model.

Parameters
----------
errors : List[Dict[str, Tuple[float, float]]]
List: the error of systems
Dict: the error of quantities, name given by the key
str: the name of the quantity, must starts with 'mae' or 'rmse'
Tuple: (error, weight)

Returns
-------
Dict
weighted averages
"""
sum_err = {}
sum_siz = {}
sum_err = defaultdict(float)
sum_siz = defaultdict(int)
for err in errors:
for kk, (ee, ss) in err.items():
if kk in sum_err:
if kk.startswith("mae"):
sum_err[kk] += ee * ss
elif kk.startswith("rmse"):
sum_err[kk] += ee * ee * ss
sum_siz[kk] += ss
else:
sum_err[kk] = ee * ee * ss
sum_siz[kk] = ss
raise RuntimeError("unknown error type")
sum_siz[kk] += ss
for kk in sum_err.keys():
sum_err[kk] = np.sqrt(sum_err[kk] / sum_siz[kk])
if kk.startswith("mae"):
sum_err[kk] = sum_err[kk] / sum_siz[kk]
elif kk.startswith("rmse"):
sum_err[kk] = np.sqrt(sum_err[kk] / sum_siz[kk])
else:
raise RuntimeError("unknown error type")
return sum_err