From 98acd487c1dbda6671fcc6f808ed34cd4059c4a0 Mon Sep 17 00:00:00 2001 From: awaelchli Date: Fri, 30 Jun 2023 22:17:25 -0700 Subject: [PATCH 1/4] show cuda matmul info only ever once --- src/lightning/fabric/accelerators/cuda.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lightning/fabric/accelerators/cuda.py b/src/lightning/fabric/accelerators/cuda.py index ca093e6a49203..8ecb7a89912e2 100644 --- a/src/lightning/fabric/accelerators/cuda.py +++ b/src/lightning/fabric/accelerators/cuda.py @@ -337,6 +337,7 @@ def _device_count_nvml() -> int: return len(visible_devices) +@lru_cache(1) # show the warning only ever once def _check_cuda_matmul_precision(device: torch.device) -> None: if not _TORCH_GREATER_EQUAL_1_12: # before 1.12, tf32 was used by default From f24c15bceee1934a302e6e43c2c0447a73ccdb91 Mon Sep 17 00:00:00 2001 From: awaelchli Date: Fri, 30 Jun 2023 22:19:47 -0700 Subject: [PATCH 2/4] update test --- tests/tests_fabric/accelerators/test_cuda.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/tests_fabric/accelerators/test_cuda.py b/tests/tests_fabric/accelerators/test_cuda.py index 06b23923871c0..11e4162cdaeb4 100644 --- a/tests/tests_fabric/accelerators/test_cuda.py +++ b/tests/tests_fabric/accelerators/test_cuda.py @@ -100,6 +100,7 @@ def test_tf32_message(_, __, caplog, monkeypatch): with caplog.at_level(logging.INFO): _check_cuda_matmul_precision(device) assert expected in caplog.text + _check_cuda_matmul_precision.cache_clear() caplog.clear() torch.backends.cuda.matmul.allow_tf32 = True # changing this changes the string @@ -107,6 +108,7 @@ def test_tf32_message(_, __, caplog, monkeypatch): with caplog.at_level(logging.INFO): _check_cuda_matmul_precision(device) assert not caplog.text + _check_cuda_matmul_precision.cache_clear() caplog.clear() torch.backends.cuda.matmul.allow_tf32 = False @@ -115,11 +117,13 @@ def test_tf32_message(_, __, caplog, monkeypatch): with caplog.at_level(logging.INFO): _check_cuda_matmul_precision(device) assert not caplog.text + _check_cuda_matmul_precision.cache_clear() torch.set_float32_matmul_precision("highest") # can be reverted with caplog.at_level(logging.INFO): _check_cuda_matmul_precision(device) assert expected in caplog.text + _check_cuda_matmul_precision.cache_clear() def test_find_usable_cuda_devices_error_handling(): From fa6912967d997b9925be08aa281bc0acd00a840e Mon Sep 17 00:00:00 2001 From: awaelchli Date: Fri, 30 Jun 2023 22:24:28 -0700 Subject: [PATCH 3/4] changelog --- src/lightning/fabric/CHANGELOG.md | 3 +++ src/lightning/pytorch/CHANGELOG.md | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/lightning/fabric/CHANGELOG.md b/src/lightning/fabric/CHANGELOG.md index 6e75f250f5d1f..b715248cf583f 100644 --- a/src/lightning/fabric/CHANGELOG.md +++ b/src/lightning/fabric/CHANGELOG.md @@ -148,6 +148,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Fixed automatic step tracking in Fabric's CSVLogger ([#17942](https://github.com/Lightning-AI/lightning/pull/17942)) +- Fixed an issue causing the `torch.set_float32_matmul_precision` info message to show multiple times ([#17960](https://github.com/Lightning-AI/lightning/pull/17960)) + + ## [2.0.3] - 2023-06-07 - Added support for `Callback` registration through entry points ([#17756](https://github.com/Lightning-AI/lightning/pull/17756)) diff --git a/src/lightning/pytorch/CHANGELOG.md b/src/lightning/pytorch/CHANGELOG.md index 26121858746eb..3d7777d8c2840 100644 --- a/src/lightning/pytorch/CHANGELOG.md +++ b/src/lightning/pytorch/CHANGELOG.md @@ -136,6 +136,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Fixed incorrect parsing of arguments when augmenting exception messages in DDP ([#17948](https://github.com/Lightning-AI/lightning/pull/17948)) +- Fixed an issue causing the `torch.set_float32_matmul_precision` info message to show multiple times ([#17960](https://github.com/Lightning-AI/lightning/pull/17960)) + + ## [2.0.3] - 2023-06-07 ### Changed From dbece2615ddf6e1234cc15446683eafb8ed5678e Mon Sep 17 00:00:00 2001 From: awaelchli Date: Fri, 30 Jun 2023 22:28:32 -0700 Subject: [PATCH 4/4] update test --- tests/tests_fabric/accelerators/test_cuda.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/tests_fabric/accelerators/test_cuda.py b/tests/tests_fabric/accelerators/test_cuda.py index 11e4162cdaeb4..9408ad4292683 100644 --- a/tests/tests_fabric/accelerators/test_cuda.py +++ b/tests/tests_fabric/accelerators/test_cuda.py @@ -123,6 +123,12 @@ def test_tf32_message(_, __, caplog, monkeypatch): with caplog.at_level(logging.INFO): _check_cuda_matmul_precision(device) assert expected in caplog.text + + # subsequent calls don't produce more messages + caplog.clear() + with caplog.at_level(logging.INFO): + _check_cuda_matmul_precision(device) + assert expected not in caplog.text _check_cuda_matmul_precision.cache_clear()