From a1da38a853011da53923188707dca4f0e22f44ec Mon Sep 17 00:00:00 2001 From: jgsogo <jgsogo@gmail.com> Date: Mon, 13 Jun 2022 15:11:10 +0200 Subject: [PATCH 01/11] [linter] Add member validation --- linter/.pylintrc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/linter/.pylintrc b/linter/.pylintrc index 557983608fbc4..def56cd695005 100644 --- a/linter/.pylintrc +++ b/linter/.pylintrc @@ -7,7 +7,8 @@ unsafe-load-any-extension=no [MESSAGES CONTROL] disable=all -enable=conan-bad-name, +enable=no-member, + conan-bad-name, conan-missing-name [REPORTS] From 74018afd5a24880767be6c0aa6af1361f058a7bb Mon Sep 17 00:00:00 2001 From: jgsogo <jgsogo@gmail.com> Date: Mon, 13 Jun 2022 15:19:05 +0200 Subject: [PATCH 02/11] linter - transform ConanFile to match actual class --- linter/.pylintrc | 16 +++++++++---- linter/conanfile_transform.py | 43 +++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 linter/conanfile_transform.py diff --git a/linter/.pylintrc b/linter/.pylintrc index def56cd695005..fd65988b48e49 100644 --- a/linter/.pylintrc +++ b/linter/.pylintrc @@ -1,14 +1,22 @@ [MASTER] -load-plugins=linter.conanv2_transition +load-plugins=linter.conanv2_transition, + linter.conanfile_transform py-version=3.6 recursive=no suggestion-mode=yes unsafe-load-any-extension=no [MESSAGES CONTROL] -disable=all -enable=no-member, - conan-bad-name, +disable=fixme, + line-too-long, + missing-module-docstring, + missing-function-docstring, + missing-class-docstring, + invalid-name, + wrong-import-order, # TODO: Remove + import-outside-toplevel # TODO: Remove + +enable=conan-bad-name, conan-missing-name [REPORTS] diff --git a/linter/conanfile_transform.py b/linter/conanfile_transform.py new file mode 100644 index 0000000000000..031075fc9f849 --- /dev/null +++ b/linter/conanfile_transform.py @@ -0,0 +1,43 @@ + +# Class ConanFile doesn't declare all the valid members and functions, +# some are injected by Conan dynamically to the class. + +import astroid + + +def register(_): + pass + +def transform_conanfile(node): + """Transform definition of ConanFile class so dynamic fields are visible to pylint""" + + str_class = astroid.builtin_lookup("str") + info_class = astroid.MANAGER.ast_from_module_name("conans.model.info").lookup( + "ConanInfo") + build_requires_class = astroid.MANAGER.ast_from_module_name( + "conans.client.graph.graph_manager").lookup("_RecipeBuildRequires") + file_copier_class = astroid.MANAGER.ast_from_module_name( + "conans.client.file_copier").lookup("FileCopier") + file_importer_class = astroid.MANAGER.ast_from_module_name( + "conans.client.importer").lookup("_FileImporter") + python_requires_class = astroid.MANAGER.ast_from_module_name( + "conans.client.graph.python_requires").lookup("PyRequires") + + dynamic_fields = { + "conan_data": str_class, + "build_requires": build_requires_class, + "info_build": info_class, + "info": info_class, + "copy": file_copier_class, + "copy_deps": file_importer_class, + "python_requires": [str_class, python_requires_class], + "recipe_folder": str_class, + } + + for f, t in dynamic_fields.items(): + node.locals[f] = [i for i in t] + + +astroid.MANAGER.register_transform( + astroid.ClassDef, transform_conanfile, + lambda node: node.qname() == "conans.model.conan_file.ConanFile") From 2e3861f8f89b088d7993896219811038d4b7979a Mon Sep 17 00:00:00 2001 From: jgsogo <jgsogo@gmail.com> Date: Mon, 13 Jun 2022 15:39:58 +0200 Subject: [PATCH 03/11] install matching conan version --- .github/workflows/linter-conan-v2.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linter-conan-v2.yml b/.github/workflows/linter-conan-v2.yml index 0db9064730bb8..eeedf7db0c403 100644 --- a/.github/workflows/linter-conan-v2.yml +++ b/.github/workflows/linter-conan-v2.yml @@ -23,6 +23,12 @@ jobs: with: files: | linter/** + - name: Get Conan v1 version + id: parse_conan_v1_version + if: steps.changed_files.outputs.any_changed == 'true' + uses: mikefarah/yq@master + with: + cmd: yq '.conan.version' '.c3i/c3i.yml' - uses: actions/setup-python@v3 if: steps.changed_files.outputs.any_changed == 'true' with: @@ -30,7 +36,7 @@ jobs: - name: Install requirements if: steps.changed_files.outputs.any_changed == 'true' run: | - pip install ${{ env.REQUIREMENTS }} + pip install ${{ env.REQUIREMENTS }} conan==${{ steps.parse_conan_v1_version.outputs.result }} - name: Execute linter over all recipes in the repository if: steps.changed_files.outputs.any_changed == 'true' run: | @@ -63,6 +69,12 @@ jobs: with: files: | **/conanfile.py + - name: Get Conan v1 version + id: parse_conan_v1_version + if: steps.changed-files.outputs.any_changed == 'true' + uses: mikefarah/yq@master + with: + cmd: yq '.conan.version' '.c3i/c3i.yml' - uses: actions/setup-python@v3 if: steps.changed-files.outputs.any_changed == 'true' with: @@ -70,7 +82,7 @@ jobs: - name: Install dependencies if: steps.changed-files.outputs.any_changed == 'true' run: | - pip install ${{ env.REQUIREMENTS }} + pip install ${{ env.REQUIREMENTS }} conan==${{ steps.parse_conan_v1_version.outputs.result }} - name: Run linter if: steps.changed-files.outputs.any_changed == 'true' run: | From 35f6e2303e7ba1c55e7d999984235d7bbcf529c6 Mon Sep 17 00:00:00 2001 From: jgsogo <jgsogo@gmail.com> Date: Mon, 13 Jun 2022 18:57:54 +0200 Subject: [PATCH 04/11] add more dynamics --- linter/conanfile_transform.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/linter/conanfile_transform.py b/linter/conanfile_transform.py index 031075fc9f849..81764f40f7df0 100644 --- a/linter/conanfile_transform.py +++ b/linter/conanfile_transform.py @@ -12,6 +12,7 @@ def transform_conanfile(node): """Transform definition of ConanFile class so dynamic fields are visible to pylint""" str_class = astroid.builtin_lookup("str") + dict_class = astroid.builtin_lookup("dict") info_class = astroid.MANAGER.ast_from_module_name("conans.model.info").lookup( "ConanInfo") build_requires_class = astroid.MANAGER.ast_from_module_name( @@ -26,12 +27,17 @@ def transform_conanfile(node): dynamic_fields = { "conan_data": str_class, "build_requires": build_requires_class, + "tool_requires": build_requires_class, "info_build": info_class, + "user_info_build": info_class, "info": info_class, "copy": file_copier_class, "copy_deps": file_importer_class, "python_requires": [str_class, python_requires_class], "recipe_folder": str_class, + "settings_build": dict_class, + "settings_target": dict_class, + "conf": dict_class, } for f, t in dynamic_fields.items(): From 7e810ee0c1e6da29a53349d5abbd15d2f0a44c66 Mon Sep 17 00:00:00 2001 From: jgsogo <jgsogo@gmail.com> Date: Mon, 13 Jun 2022 19:17:53 +0200 Subject: [PATCH 05/11] no settings --- linter/conanfile_transform.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/linter/conanfile_transform.py b/linter/conanfile_transform.py index 81764f40f7df0..378c36fab403c 100644 --- a/linter/conanfile_transform.py +++ b/linter/conanfile_transform.py @@ -35,11 +35,9 @@ def transform_conanfile(node): "copy_deps": file_importer_class, "python_requires": [str_class, python_requires_class], "recipe_folder": str_class, - "settings_build": dict_class, - "settings_target": dict_class, "conf": dict_class, } - + for f, t in dynamic_fields.items(): node.locals[f] = [i for i in t] From beb355aa81cf8909f1502c02129842d2fd5fbd14 Mon Sep 17 00:00:00 2001 From: jgsogo <jgsogo@gmail.com> Date: Mon, 13 Jun 2022 19:47:03 +0200 Subject: [PATCH 06/11] these are failing --- linter/conanfile_transform.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/linter/conanfile_transform.py b/linter/conanfile_transform.py index 378c36fab403c..c8a7083656fdf 100644 --- a/linter/conanfile_transform.py +++ b/linter/conanfile_transform.py @@ -23,18 +23,22 @@ def transform_conanfile(node): "conans.client.importer").lookup("_FileImporter") python_requires_class = astroid.MANAGER.ast_from_module_name( "conans.client.graph.python_requires").lookup("PyRequires") + settings_class = astroid.MANAGER.ast_from_module_name( + "conans.model.settings").lookup("Settings") dynamic_fields = { "conan_data": str_class, "build_requires": build_requires_class, "tool_requires": build_requires_class, "info_build": info_class, - "user_info_build": info_class, + #"user_info_build": info_class, "info": info_class, "copy": file_copier_class, "copy_deps": file_importer_class, "python_requires": [str_class, python_requires_class], "recipe_folder": str_class, + #"settings_build": settings_class, + #"settings_target": settings_class, "conf": dict_class, } From ce22609676e717ca16fd5f8c106511a66ba90c74 Mon Sep 17 00:00:00 2001 From: jgsogo <jgsogo@gmail.com> Date: Mon, 13 Jun 2022 21:17:14 +0200 Subject: [PATCH 07/11] mock settings --- linter/conanfile_transform.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/linter/conanfile_transform.py b/linter/conanfile_transform.py index c8a7083656fdf..3ea2a301b8c8c 100644 --- a/linter/conanfile_transform.py +++ b/linter/conanfile_transform.py @@ -2,7 +2,24 @@ # Class ConanFile doesn't declare all the valid members and functions, # some are injected by Conan dynamically to the class. +import textwrap import astroid +from astroid.builder import extract_node, parse +from astroid.builder import AstroidBuilder +from astroid.manager import AstroidManager + + +def _settings_transform(): + module = AstroidBuilder(AstroidManager()).string_build( + textwrap.dedent(""" + class Settings(object): + os = None + arch = None + compiler = None + build_type = None + """) + ) + return module['Settings'] def register(_): @@ -23,22 +40,20 @@ def transform_conanfile(node): "conans.client.importer").lookup("_FileImporter") python_requires_class = astroid.MANAGER.ast_from_module_name( "conans.client.graph.python_requires").lookup("PyRequires") - settings_class = astroid.MANAGER.ast_from_module_name( - "conans.model.settings").lookup("Settings") dynamic_fields = { "conan_data": str_class, "build_requires": build_requires_class, "tool_requires": build_requires_class, "info_build": info_class, - #"user_info_build": info_class, + "user_info_build": info_class, "info": info_class, "copy": file_copier_class, "copy_deps": file_importer_class, "python_requires": [str_class, python_requires_class], "recipe_folder": str_class, - #"settings_build": settings_class, - #"settings_target": settings_class, + "settings_build": [_settings_transform()], + "settings_target": [_settings_transform()], "conf": dict_class, } From 4a29f3fe6660f24b490a1222fb14dd13643f2c4d Mon Sep 17 00:00:00 2001 From: jgsogo <jgsogo@gmail.com> Date: Tue, 14 Jun 2022 08:59:14 +0200 Subject: [PATCH 08/11] user_info_build --- linter/conanfile_transform.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/linter/conanfile_transform.py b/linter/conanfile_transform.py index 3ea2a301b8c8c..8e33d368643b1 100644 --- a/linter/conanfile_transform.py +++ b/linter/conanfile_transform.py @@ -4,7 +4,6 @@ import textwrap import astroid -from astroid.builder import extract_node, parse from astroid.builder import AstroidBuilder from astroid.manager import AstroidManager @@ -21,6 +20,15 @@ class Settings(object): ) return module['Settings'] +def _user_info_build_transform(): + module = AstroidBuilder(AstroidManager()).string_build( + textwrap.dedent(""" + class UserInfoBuild(defaultdict): + pass + """) + ) + return module['UserInfoBuild'] + def register(_): pass @@ -46,7 +54,7 @@ def transform_conanfile(node): "build_requires": build_requires_class, "tool_requires": build_requires_class, "info_build": info_class, - "user_info_build": info_class, + "user_info_build": [_user_info_build_transform()], "info": info_class, "copy": file_copier_class, "copy_deps": file_importer_class, From ec8038f08bf7cae6d88c9b6626fb1d1f1f199c36 Mon Sep 17 00:00:00 2001 From: ericLemanissier <ericLemanissier@users.noreply.github.com> Date: Wed, 15 Jun 2022 11:31:07 +0200 Subject: [PATCH 09/11] add fatals to matcher (#11) --- linter/recipe_linter.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/linter/recipe_linter.json b/linter/recipe_linter.json index 3320af6a40a0d..e0679808a6ead 100644 --- a/linter/recipe_linter.json +++ b/linter/recipe_linter.json @@ -1,5 +1,18 @@ { "problemMatcher": [ + { + "owner": "recipe_linter_fatals", + "severity": "error", + "pattern": [ + { + "regexp": "(\\S+):(\\d+): \\[(F\\d+\\(\\S+\\)),\\s(.+?)\\](.+)", + "file": 1, + "line": 2, + "message": 5, + "code": 3 + } + ] + }, { "owner": "recipe_linter_errors", "severity": "error", From 0a917484a0530a8b1b6b82f8015559f307305339 Mon Sep 17 00:00:00 2001 From: jgsogo <jgsogo@gmail.com> Date: Thu, 16 Jun 2022 21:44:24 +0200 Subject: [PATCH 10/11] increase threshold --- .github/workflows/linter-conan-v2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linter-conan-v2.yml b/.github/workflows/linter-conan-v2.yml index 272a6cb7ad671..129336406a0dc 100644 --- a/.github/workflows/linter-conan-v2.yml +++ b/.github/workflows/linter-conan-v2.yml @@ -6,7 +6,7 @@ on: env: PYTHONPATH: ${{github.workspace}} PYVER: "3.8" - SCORE_THRESHOLD: "9.0" + SCORE_THRESHOLD: "9.5" REQUIREMENTS: "pylint==2.14" jobs: From fa2f5faa6a37460575bdf887fde2d4422effbcd8 Mon Sep 17 00:00:00 2001 From: jgsogo <jgsogo@gmail.com> Date: Fri, 17 Jun 2022 09:52:21 +0200 Subject: [PATCH 11/11] touch