Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix ci test error introduced by (#9781) #9838

Merged
merged 6 commits into from
Feb 8, 2023
Merged

fix ci test error introduced by (#9781) #9838

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b8cefa1
modify flow.allclose in test_swautils
process852 Feb 7, 2023
99b39fd
make of_format
process852 Feb 7, 2023
ccd15b1
use np.allclose to test scalar
process852 Feb 8, 2023
26d1fe4
make of_format
process852 Feb 8, 2023
52f7d01
Merge branch 'master' into gcc7
process852 Feb 8, 2023
9863f56
Merge branch 'master' into gcc7
mergify[bot] Feb 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 27 additions & 27 deletions python/oneflow/test/modules/test_swautils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import math
import unittest
import itertools
import numpy as np

import oneflow as flow
import oneflow.optim as optim
Expand Down Expand Up @@ -67,7 +68,9 @@ def _test(self, schedulers, targets, epochs=10):
for epoch in range(epochs):
for param_group, target in zip(self.opt.param_groups, targets):
self.assertTrue(
math.fabs(target[epoch] - param_group["lr"]) < 1e-5,
np.allclose(
target[epoch], param_group["lr"], atol=1e-6, rtol=1e-5,
),
msg="LR is wrong in epoch {}: expected {}, got {}".format(
epoch, target[epoch], param_group["lr"]
),
Expand Down Expand Up @@ -136,7 +139,9 @@ def _test_swalr(self, swa_scheduler, scheduler, targets, swa_start, epochs):
for epoch in range(epochs):
for param_group, target in zip(self.opt.param_groups, targets):
self.assertTrue(
math.fabs(target[epoch] - param_group["lr"]) < 1e-7,
np.allclose(
target[epoch], param_group["lr"], atol=1e-6, rtol=1e-5,
),
msg="LR is wrong in epoch {}: expected {}, got {}".format(
epoch, target[epoch], param_group["lr"]
),
Expand All @@ -160,19 +165,6 @@ def test_swalr_hypers(self):
with self.assertRaisesRegex(ValueError, "swa_lr must"):
swa_scheduler = SWALR(self.opt, swa_lr=[1.0, 0.1, 0.01])

def test_multiplicative_lr(self):
epochs = 10
self.opt.param_groups[0]["lr"] = 0.05
self.opt.param_groups[1]["lr"] = 0.4
targets = [
[0.05 * (0.9 ** x) for x in range(epochs)],
[0.4 * (0.8 ** x) for x in range(epochs)],
]
scheduler = MultiplicativeLR(
self.opt, lr_lambda=[lambda x1: 0.9, lambda x2: 0.8]
)
self._test(scheduler, targets, epochs)


@flow.unittest.skip_unless_1n1d()
class TestSWAUtils(flow.unittest.TestCase):
Expand Down Expand Up @@ -204,7 +196,9 @@ def _test_averaged_model(self, net_device, swa_device):
averaged_dnn.update_parameters(dnn)

for p_avg, p_swa in zip(averaged_params, averaged_dnn.parameters()):
self.assertTrue(flow.max(flow.abs(p_avg.cpu() - p_swa.cpu())).item() < 1e-5)
self.assertTrue(
flow.allclose(p_avg.cpu(), p_swa.cpu(), atol=1e-5, rtol=1e-4)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这里需要 .cpu() 吗

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这里是是需要的，在test_averaged_model_all_devices中进行了不同设备之间的测试
image

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

好的

)
# Check that AveragedModel is on the correct device
self.assertTrue(p_swa.device == swa_device)
self.assertTrue(p.device == net_device)
Expand Down Expand Up @@ -237,7 +231,7 @@ def test_averaged_model_mixed_device(self):
averaged_dnn.update_parameters(dnn)

for p_avg, p_swa in zip(averaged_params, averaged_dnn.parameters()):
self.assertTrue(flow.max(flow.abs(p_avg - p_swa)).item() < 1e-5)
self.assertTrue(flow.allclose(p_avg, p_swa, atol=1e-5, rtol=1e-4))
# Check that AveragedModel is on the correct device
self.assertTrue(p_avg.device == p_swa.device)

Expand All @@ -254,7 +248,7 @@ def test_averaged_model_state_dict(self):
averaged_dnn.update_parameters(dnn)
averaged_dnn2.load_state_dict(averaged_dnn.state_dict())
for p_swa, p_swa2 in zip(averaged_dnn.parameters(), averaged_dnn2.parameters()):
self.assertTrue(flow.max(flow.abs(p_swa - p_swa2)) < 1e-5)
self.assertTrue(flow.allclose(p_swa, p_swa2, atol=1e-5, rtol=1e-4))
self.assertTrue(averaged_dnn.n_averaged == averaged_dnn2.n_averaged)

def test_averaged_model_exponential(self):
Expand Down Expand Up @@ -292,9 +286,9 @@ def avg_fn(p_avg, p, n_avg):
averaged_params = updated_averaged_params

for p_avg, p_swa in zip(averaged_params, averaged_dnn.parameters()):
self.assertTrue(flow.max(flow.abs(p_avg - p_swa)) < 1e-5)
self.assertTrue(flow.allclose(p_avg, p_swa, atol=1e-5, rtol=1e-4))
for b_avg, b_swa in zip(dnn.buffers(), averaged_dnn.module.buffers()):
self.assertTrue(flow.max(flow.abs(b_avg - b_swa)) < 1e-5)
self.assertTrue(flow.allclose(b_avg, b_swa, atol=1e-5, rtol=1e-4))

def test_averaged_model_exponential_buffers(self):
# Test AveragedModel with EMA as avg_fn and use_buffers as True.
Expand Down Expand Up @@ -337,7 +331,7 @@ def avg_fn(p_avg, p, n_avg):
averaged_dnn.module.parameters(), averaged_dnn.module.buffers()
),
):
self.assertTrue(flow.max(flow.abs(p_avg - p_swa)) < 1e-5)
self.assertTrue(flow.allclose(p_avg, p_swa, atol=1e-5, rtol=1e-4))

def _test_update_bn(self, dnn, dl_x, dl_xy, momentum, cuda):

Expand Down Expand Up @@ -367,9 +361,11 @@ def _test_update_bn(self, dnn, dl_x, dl_xy, momentum, cuda):
preactivation_var = preactivation_var - preactivation_mean ** 2

update_bn(dl_xy, dnn, device=x.device)
self.assertTrue(flow.allclose(preactivation_mean, dnn.bn.running_mean))
self.assertTrue(
flow.allclose(preactivation_var, dnn.bn.running_var, atol=1e-1, rtol=0)
flow.allclose(preactivation_mean, dnn.bn.running_mean, atol=1e-6, rtol=1e-3)
)
self.assertTrue(
flow.allclose(preactivation_var, dnn.bn.running_var, atol=1e-1, rtol=1e-1)
)

def _reset_bn(module):
Expand All @@ -380,16 +376,20 @@ def _reset_bn(module):
# reset batch norm and run update_bn again
dnn.apply(_reset_bn)
update_bn(dl_xy, dnn, device=x.device)
self.assertTrue(flow.allclose(preactivation_mean, dnn.bn.running_mean))
self.assertTrue(
flow.allclose(preactivation_var, dnn.bn.running_var, atol=1e-1, rtol=0)
flow.allclose(preactivation_mean, dnn.bn.running_mean, atol=1e-6, rtol=1e-3)
)
self.assertTrue(
flow.allclose(preactivation_var, dnn.bn.running_var, atol=1e-1, rtol=1e-1)
)
# using the dl_x loader instead of dl_xy
dnn.apply(_reset_bn)
update_bn(dl_x, dnn, device=x.device)
self.assertTrue(flow.allclose(preactivation_mean, dnn.bn.running_mean))
self.assertTrue(
flow.allclose(preactivation_var, dnn.bn.running_var, atol=1e-1, rtol=0)
flow.allclose(preactivation_mean, dnn.bn.running_mean, atol=1e-6, rtol=1e-3)
)
self.assertTrue(
flow.allclose(preactivation_var, dnn.bn.running_var, atol=1e-1, rtol=1e-1)
)

def test_update_bn_dnn(self):
Expand Down