From 1d097dd092cfa764761fc5a548325ddf1fbbff69 Mon Sep 17 00:00:00 2001 From: Samir Nasibli Date: Fri, 14 Jun 2024 14:22:55 -0700 Subject: [PATCH 1/5] TEST: moved checks for banned primitives into certain folder, out of pkgs --- onedal/tests/test_common.py | 41 ------------- sklearnex/tests/test_common.py | 54 ----------------- tests/test_common.py | 102 +++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 95 deletions(-) delete mode 100644 onedal/tests/test_common.py delete mode 100644 sklearnex/tests/test_common.py create mode 100644 tests/test_common.py diff --git a/onedal/tests/test_common.py b/onedal/tests/test_common.py deleted file mode 100644 index 2746af802a..0000000000 --- a/onedal/tests/test_common.py +++ /dev/null @@ -1,41 +0,0 @@ -# ============================================================================== -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -import os -from glob import glob - -import pytest - - -def test_sklearn_check_version_ban(): - """This test blocks the use of sklearn_check_version - in onedal files. The versioning should occur in the - sklearnex package for clarity and maintainability. - """ - from onedal import __file__ as loc - - path = loc.replace("__init__.py", "") - files = [y for x in os.walk(path) for y in glob(os.path.join(x[0], "*.py"))] - - output = [] - - for f in files: - if open(f, "r").read().find("sklearn_check_version") != -1: - output += [f.replace(path, "onedal" + os.sep)] - - # remove this file from the list - output = "\n".join([i for i in output if "test_common.py" not in i]) - assert output == "", f"sklearn versioning is occuring in: \n{output}" diff --git a/sklearnex/tests/test_common.py b/sklearnex/tests/test_common.py deleted file mode 100644 index 311c1cd285..0000000000 --- a/sklearnex/tests/test_common.py +++ /dev/null @@ -1,54 +0,0 @@ -# ============================================================================== -# Copyright 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -import os -from glob import glob - -import pytest - -ALLOWED_LOCATIONS = [ - "_config.py", - "_device_offload.py", - "test", - "svc.py", - "svm" + os.sep + "_common.py", -] - - -def test_target_offload_ban(): - """This test blocks the use of target_offload in - in sklearnex files. Offloading computation to devices - via target_offload should only occur externally, and not - within the architecture of the sklearnex classes. This - is for clarity, traceability and maintainability. - """ - from sklearnex import __file__ as loc - - path = loc.replace("__init__.py", "") - files = [y for x in os.walk(path) for y in glob(os.path.join(x[0], "*.py"))] - - output = [] - - for f in files: - if open(f, "r").read().find("target_offload") != -1: - output += [f.replace(path, "sklearnex" + os.sep)] - - # remove this file from the list - for allowed in ALLOWED_LOCATIONS: - output = [i for i in output if allowed not in i] - - output = "\n".join(output) - assert output == "", f"sklearn versioning is occuring in: \n{output}" diff --git a/tests/test_common.py b/tests/test_common.py new file mode 100644 index 0000000000..0cdc8dbe99 --- /dev/null +++ b/tests/test_common.py @@ -0,0 +1,102 @@ +# ============================================================================== +# Copyright 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import importlib +import os +from glob import glob + +TARGET_OFFLOAD_ALLOWED_LOCATIONS = [ + "_config.py", + "_device_offload.py", + "test", + "svc.py", + "svm" + os.sep + "_common.py", +] + + +def _test_primitive_usage_ban(primimtive_name, banned_locations, allowed_locations=None): + """This test blocks the usage of the primitive in + in certain files. + """ + + loc = importlib.import_module(banned_locations).__file__ + + path = loc.replace("__init__.py", "") + files = [y for x in os.walk(path) for y in glob(os.path.join(x[0], "*.py"))] + + output = [] + + for f in files: + if open(f, "r").read().find(primimtive_name) != -1: + output += [f.replace(path, banned_locations + os.sep)] + + # remove this file from the list + if allowed_locations: + for allowed in allowed_locations: + output = [i for i in output if allowed not in i] + + return output + + +def test_target_offload_ban(): + """This test blocks the use of target_offload in + in sklearnex files. Offloading computation to devices + via target_offload should only occur externally, and not + within the architecture of the sklearnex classes. This + is for clarity, traceability and maintainability. + """ + output = _test_primitive_usage_ban( + primimtive_name="target_offload", + banned_locations="sklearnex", + allowed_locations=TARGET_OFFLOAD_ALLOWED_LOCATIONS, + ) + output = "\n".join(output) + assert output == "", f"target offloading is occuring in: \n{output}" + + +def test_sklearn_check_version_ban(): + """This test blocks the use of sklearn_check_version + in onedal files. The versioning should occur in the + sklearnex package for clarity and maintainability. + """ + output = _test_primitive_usage_ban( + primimtive_name="sklearn_check_version", banned_locations="onedal" + ) + + # remove this file from the list + output = "\n".join([i for i in output if "test_common.py" not in i]) + assert output == "", f"sklearn versioning is occuring in: \n{output}" + + +def test_sklearn_check_version_ban_1(): + """This test blocks the use of sklearn_check_version + in onedal files. The versioning should occur in the + sklearnex package for clarity and maintainability. + """ + from onedal import __file__ as loc + + path = loc.replace("__init__.py", "") + files = [y for x in os.walk(path) for y in glob(os.path.join(x[0], "*.py"))] + + output = [] + + for f in files: + if open(f, "r").read().find("sklearn_check_version") != -1: + output += [f.replace(path, "onedal" + os.sep)] + + # remove this file from the list + output = "\n".join([i for i in output if "test_common.py" not in i]) + assert output == "", f"sklearn versioning is occuring in: \n{output}" From 6da45303599f11aace259ac3ffe6b7a65cbf032d Mon Sep 17 00:00:00 2001 From: Samir Nasibli Date: Fri, 14 Jun 2024 23:25:07 +0200 Subject: [PATCH 2/5] Update test_common.py --- tests/test_common.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/tests/test_common.py b/tests/test_common.py index 0cdc8dbe99..fe1138e82c 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -79,24 +79,3 @@ def test_sklearn_check_version_ban(): # remove this file from the list output = "\n".join([i for i in output if "test_common.py" not in i]) assert output == "", f"sklearn versioning is occuring in: \n{output}" - - -def test_sklearn_check_version_ban_1(): - """This test blocks the use of sklearn_check_version - in onedal files. The versioning should occur in the - sklearnex package for clarity and maintainability. - """ - from onedal import __file__ as loc - - path = loc.replace("__init__.py", "") - files = [y for x in os.walk(path) for y in glob(os.path.join(x[0], "*.py"))] - - output = [] - - for f in files: - if open(f, "r").read().find("sklearn_check_version") != -1: - output += [f.replace(path, "onedal" + os.sep)] - - # remove this file from the list - output = "\n".join([i for i in output if "test_common.py" not in i]) - assert output == "", f"sklearn versioning is occuring in: \n{output}" From 4b5d85f8d6f16e1e6870800cc2968706f068f50c Mon Sep 17 00:00:00 2001 From: Samir Nasibli Date: Sat, 19 Oct 2024 15:52:10 -0700 Subject: [PATCH 3/5] fix linting --- sklearnex/tests/test_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearnex/tests/test_common.py b/sklearnex/tests/test_common.py index 03724ceabd..11c8d039a9 100644 --- a/sklearnex/tests/test_common.py +++ b/sklearnex/tests/test_common.py @@ -23,10 +23,10 @@ import trace import pytest -from onedal.tests.test_common import _test_primitive_usage_ban from sklearn.utils import all_estimators from daal4py.sklearn._utils import sklearn_check_version +from onedal.tests.test_common import _test_primitive_usage_ban from sklearnex.tests.utils import ( PATCHED_MODELS, SPECIAL_INSTANCES, From 3a33d9bc7a923af2ced5d67d72389d9fff53cebe Mon Sep 17 00:00:00 2001 From: Samir Nasibli Date: Mon, 21 Oct 2024 08:05:09 -0700 Subject: [PATCH 4/5] refactoring using pkgutil.get_loader instead of importlib.import_module --- onedal/tests/test_common.py | 18 +++++++++++------- sklearnex/tests/test_common.py | 8 ++++---- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/onedal/tests/test_common.py b/onedal/tests/test_common.py index ad21dfb2b1..5bf71ed440 100644 --- a/onedal/tests/test_common.py +++ b/onedal/tests/test_common.py @@ -14,17 +14,21 @@ # limitations under the License. # ============================================================================== -import importlib import os +import pkgutil from glob import glob -def _test_primitive_usage_ban(primimtive_name, banned_locations, allowed_locations=None): +def _check_primitive_usage_ban(primitive_name, package, allowed_locations=None): """This test blocks the usage of the primitive in in certain files. """ - loc = importlib.import_module(banned_locations).__file__ + # TODO: + # Address deprecation warning. + # The function "get_loader" is deprecated Use importlib.util.find_spec() instead. + # Will be removed in Python 3.14. + loc = pkgutil.get_loader(package).get_filename() path = loc.replace("__init__.py", "") files = [y for x in os.walk(path) for y in glob(os.path.join(x[0], "*.py"))] @@ -32,8 +36,8 @@ def _test_primitive_usage_ban(primimtive_name, banned_locations, allowed_locatio output = [] for f in files: - if open(f, "r").read().find(primimtive_name) != -1: - output += [f.replace(path, banned_locations + os.sep)] + if open(f, "r").read().find(primitive_name) != -1: + output += [f.replace(path, package + os.sep)] # remove this file from the list if allowed_locations: @@ -48,8 +52,8 @@ def test_sklearn_check_version_ban(): in onedal files. The versioning should occur in the sklearnex package for clarity and maintainability. """ - output = _test_primitive_usage_ban( - primimtive_name="sklearn_check_version", banned_locations="onedal" + output = _check_primitive_usage_ban( + primitive_name="sklearn_check_version", package="onedal" ) # remove this file from the list diff --git a/sklearnex/tests/test_common.py b/sklearnex/tests/test_common.py index 11c8d039a9..df3cceb64f 100644 --- a/sklearnex/tests/test_common.py +++ b/sklearnex/tests/test_common.py @@ -26,7 +26,7 @@ from sklearn.utils import all_estimators from daal4py.sklearn._utils import sklearn_check_version -from onedal.tests.test_common import _test_primitive_usage_ban +from onedal.tests.test_common import _check_primitive_usage_ban from sklearnex.tests.utils import ( PATCHED_MODELS, SPECIAL_INSTANCES, @@ -106,9 +106,9 @@ def test_target_offload_ban(): within the architecture of the sklearnex classes. This is for clarity, traceability and maintainability. """ - output = _test_primitive_usage_ban( - primimtive_name="target_offload", - banned_locations="sklearnex", + output = _check_primitive_usage_ban( + primitive_name="target_offload", + package="sklearnex", allowed_locations=TARGET_OFFLOAD_ALLOWED_LOCATIONS, ) output = "\n".join(output) From 3792ed8aef74f37487aa454f848236169f28bb1e Mon Sep 17 00:00:00 2001 From: Samir Nasibli Date: Wed, 23 Oct 2024 11:04:29 +0200 Subject: [PATCH 5/5] Update test_common.py --- onedal/tests/test_common.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/onedal/tests/test_common.py b/onedal/tests/test_common.py index 5bf71ed440..7687d12f1d 100644 --- a/onedal/tests/test_common.py +++ b/onedal/tests/test_common.py @@ -14,8 +14,8 @@ # limitations under the License. # ============================================================================== +import importlib import os -import pkgutil from glob import glob @@ -24,11 +24,7 @@ def _check_primitive_usage_ban(primitive_name, package, allowed_locations=None): in certain files. """ - # TODO: - # Address deprecation warning. - # The function "get_loader" is deprecated Use importlib.util.find_spec() instead. - # Will be removed in Python 3.14. - loc = pkgutil.get_loader(package).get_filename() + loc = importlib.util.find_spec(package).origin path = loc.replace("__init__.py", "") files = [y for x in os.walk(path) for y in glob(os.path.join(x[0], "*.py"))]