From 60f90413dcb1f6e6ad85ab18f67c0bf238c6b4c4 Mon Sep 17 00:00:00 2001 From: Sterling Date: Thu, 4 May 2023 14:40:18 +0000 Subject: [PATCH 1/6] set cuda devices for no-docker mode --- armory/eval/evaluator.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/armory/eval/evaluator.py b/armory/eval/evaluator.py index 21233e4f3..4e77538c9 100644 --- a/armory/eval/evaluator.py +++ b/armory/eval/evaluator.py @@ -95,7 +95,15 @@ def _gather_env_variables(self): if self.config["sysconfig"].get("use_gpu", None): gpus = self.config["sysconfig"].get("gpus") if gpus is not None: - self.extra_env_vars["NVIDIA_VISIBLE_DEVICES"] = gpus + if self.no_docker: + self.extra_env_vars["CUDA_VISIBLE_DEVICES"] = gpus + else: + self.extra_env_vars["NVIDIA_VISIBLE_DEVICES"] = gpus + else: + if self.no_docker: + # Must be explicitly set for no-docker + self.extra_env_vars["CUDA_VISIBLE_DEVICES"] = "-1" + if self.config["sysconfig"].get("set_pythonhashseed"): self.extra_env_vars["PYTHONHASHSEED"] = "0" From 84d7902a026e0837c983df899fe94461c93dfcd1 Mon Sep 17 00:00:00 2001 From: Sterling Date: Thu, 4 May 2023 17:16:31 +0000 Subject: [PATCH 2/6] don't set cuda devices if already set --- armory/eval/evaluator.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/armory/eval/evaluator.py b/armory/eval/evaluator.py index 4e77538c9..4e33682da 100644 --- a/armory/eval/evaluator.py +++ b/armory/eval/evaluator.py @@ -92,15 +92,19 @@ def _gather_env_variables(self): if not self.armory_global_config["verify_ssl"]: self.extra_env_vars["VERIFY_SSL"] = "false" + cuda_var_exists = True if os.getenv("CUDA_VISIBLE_DEVICES") is not None else False + # Existing value should override armory flags + + # Set visible gpus if self.config["sysconfig"].get("use_gpu", None): gpus = self.config["sysconfig"].get("gpus") if gpus is not None: - if self.no_docker: + if self.no_docker and not cuda_var_exists: self.extra_env_vars["CUDA_VISIBLE_DEVICES"] = gpus - else: + if not self.no_docker: self.extra_env_vars["NVIDIA_VISIBLE_DEVICES"] = gpus else: - if self.no_docker: + if self.no_docker and not cuda_var_exists: # Must be explicitly set for no-docker self.extra_env_vars["CUDA_VISIBLE_DEVICES"] = "-1" From 8918cf13175a102ba54674a2186b980f82638cfe Mon Sep 17 00:00:00 2001 From: Sterling Date: Thu, 4 May 2023 17:18:37 +0000 Subject: [PATCH 3/6] formatting --- armory/eval/evaluator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/armory/eval/evaluator.py b/armory/eval/evaluator.py index 4e33682da..60fc30a15 100644 --- a/armory/eval/evaluator.py +++ b/armory/eval/evaluator.py @@ -92,7 +92,9 @@ def _gather_env_variables(self): if not self.armory_global_config["verify_ssl"]: self.extra_env_vars["VERIFY_SSL"] = "false" - cuda_var_exists = True if os.getenv("CUDA_VISIBLE_DEVICES") is not None else False + cuda_var_exists = ( + True if os.getenv("CUDA_VISIBLE_DEVICES") is not None else False + ) # Existing value should override armory flags # Set visible gpus From 863711241a458728c6b50e16f6c8ed4fda104754 Mon Sep 17 00:00:00 2001 From: Sterling Date: Fri, 5 May 2023 16:53:24 +0000 Subject: [PATCH 4/6] log warning if cuda_visible_devices is already set in no-docker mode --- armory/eval/evaluator.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/armory/eval/evaluator.py b/armory/eval/evaluator.py index 60fc30a15..97f47b476 100644 --- a/armory/eval/evaluator.py +++ b/armory/eval/evaluator.py @@ -95,7 +95,9 @@ def _gather_env_variables(self): cuda_var_exists = ( True if os.getenv("CUDA_VISIBLE_DEVICES") is not None else False ) - # Existing value should override armory flags + if cuda_var_exists and self.no_docker: + log.warning("CUDA_VISIBLE_DEVICES is set; any Armoy gpu instructions will be ignored") + # Existing value should override armory flags # Set visible gpus if self.config["sysconfig"].get("use_gpu", None): @@ -107,7 +109,7 @@ def _gather_env_variables(self): self.extra_env_vars["NVIDIA_VISIBLE_DEVICES"] = gpus else: if self.no_docker and not cuda_var_exists: - # Must be explicitly set for no-docker + # Block gpus for no-docker mode self.extra_env_vars["CUDA_VISIBLE_DEVICES"] = "-1" if self.config["sysconfig"].get("set_pythonhashseed"): From 038f3e94c820818c5233d402f623f7794b476af3 Mon Sep 17 00:00:00 2001 From: Sterling Date: Fri, 5 May 2023 16:54:27 +0000 Subject: [PATCH 5/6] formatting --- armory/eval/evaluator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/armory/eval/evaluator.py b/armory/eval/evaluator.py index 97f47b476..cea9193d9 100644 --- a/armory/eval/evaluator.py +++ b/armory/eval/evaluator.py @@ -96,7 +96,9 @@ def _gather_env_variables(self): True if os.getenv("CUDA_VISIBLE_DEVICES") is not None else False ) if cuda_var_exists and self.no_docker: - log.warning("CUDA_VISIBLE_DEVICES is set; any Armoy gpu instructions will be ignored") + log.warning( + "CUDA_VISIBLE_DEVICES is set; any Armoy gpu instructions will be ignored" + ) # Existing value should override armory flags # Set visible gpus From f8ae1497ebbe8bacc44b01c3f28df2133556caba Mon Sep 17 00:00:00 2001 From: Sterling Date: Fri, 5 May 2023 17:12:36 +0000 Subject: [PATCH 6/6] minor format update --- armory/eval/evaluator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/armory/eval/evaluator.py b/armory/eval/evaluator.py index cea9193d9..0f6b16ae5 100644 --- a/armory/eval/evaluator.py +++ b/armory/eval/evaluator.py @@ -96,10 +96,10 @@ def _gather_env_variables(self): True if os.getenv("CUDA_VISIBLE_DEVICES") is not None else False ) if cuda_var_exists and self.no_docker: + # Existing value should override armory flags log.warning( "CUDA_VISIBLE_DEVICES is set; any Armoy gpu instructions will be ignored" ) - # Existing value should override armory flags # Set visible gpus if self.config["sysconfig"].get("use_gpu", None):