Exclude licenses pages from online search results (#2665)

* Exclude licenses pages from online search results Fixes #2664 * [MegaLinter] Apply linters fixes * [MegaLinter] Apply linters fixes * [MegaLinter] Apply linters fixes * [MegaLinter] Apply linters fixes * [MegaLinter] Apply linters fixes * [MegaLinter] Apply linters fixes * Ignore licenses --------- Co-authored-by: nvuillam <nvuillam@users.noreply.github.com>
oxsecurity · May 19, 2023 · fe51dce · fe51dce
1 parent 4b714b1
commit fe51dce
Show file tree

Hide file tree

Showing 109 changed files with 4,864 additions and 176 deletions.
diff --git a/.automation/build.py b/.automation/build.py
@@ -2667,7 +2667,11 @@ def generate_documentation_all_linters():
                     # Update license key for licenses file
                     resp = r.json()
                     if resp is not None and not isinstance(resp, type(None)):
-                        if "license" in resp and "spdx_id" in resp["license"]:
+                        if (
+                            "license" in resp
+                            and resp["license"] is not None
+                            and "spdx_id" in resp["license"]
+                        ):
                             license = (
                                 resp["license"]["spdx_id"]
                                 if resp["license"]["spdx_id"] != "NOASSERTION"
@@ -2692,12 +2696,22 @@ def generate_documentation_all_linters():
                             resp_license = r_license.json()
                             if "download_url" in resp_license:
                                 license_downloaded = session.get(
-                                    resp_license["download_url"]
+                                    resp_license["download_url"],
+                                    headers=api_github_headers,
                                 )
                                 with open(
                                     linter_license_md_file, "w", encoding="utf-8"
                                 ) as license_out_file:
-                                    license_out_file.write(license_downloaded.text)
+                                    license_header = (
+                                        "---\n"
+                                        f"title: License info for {linter.linter_name} within MegaLinter\n"
+                                        "search:\n"
+                                        "  exclude: true\n"
+                                        "---\n"
+                                    )
+                                    license_out_file.write(
+                                        license_header + license_downloaded.text
+                                    )
                                     logging.info(
                                         f"Copied license of {linter.linter_name} in {linter_license_md_file}"
                                     )

diff --git a/.automation/generated/flavors-stats.json b/.automation/generated/flavors-stats.json
@@ -1493,8 +1493,8 @@
             2957431
         ],
         [
-            "2023-05-19T08:27:51",
-            2958209
+            "2023-05-19T14:40:36",
+            2958611
         ]
     ],
     "ci_light": [
@@ -2991,8 +2991,8 @@
             64933
         ],
         [
-            "2023-05-19T08:27:51",
-            65067
+            "2023-05-19T14:40:36",
+            65113
         ]
     ],
     "cupcake": [
@@ -3469,8 +3469,8 @@
             11130
         ],
         [
-            "2023-05-19T08:27:51",
-            11225
+            "2023-05-19T14:40:36",
+            11257
         ]
     ],
     "dart": [
@@ -5877,8 +5877,8 @@
             149270
         ],
         [
-            "2023-05-19T08:27:51",
-            149471
+            "2023-05-19T14:40:36",
+            149545
         ]
     ],
     "dotnet": [
@@ -7375,8 +7375,8 @@
             386087
         ],
         [
-            "2023-05-19T08:27:51",
-            386329
+            "2023-05-19T14:40:36",
+            386482
         ]
     ],
     "go": [
@@ -8873,8 +8873,8 @@
             20760
         ],
         [
-            "2023-05-19T08:27:51",
-            20793
+            "2023-05-19T14:40:36",
+            20802
         ]
     ],
     "java": [
@@ -10371,8 +10371,8 @@
             119481
         ],
         [
-            "2023-05-19T08:27:51",
-            119535
+            "2023-05-19T14:40:36",
+            119572
         ]
     ],
     "javascript": [
@@ -11869,8 +11869,8 @@
             263290
         ],
         [
-            "2023-05-19T08:27:51",
-            263602
+            "2023-05-19T14:40:36",
+            263721
         ]
     ],
     "php": [
@@ -13367,8 +13367,8 @@
             49564
         ],
         [
-            "2023-05-19T08:27:51",
-            49606
+            "2023-05-19T14:40:36",
+            49613
         ]
     ],
     "python": [
@@ -14865,8 +14865,8 @@
             206554
         ],
         [
-            "2023-05-19T08:27:51",
-            206903
+            "2023-05-19T14:40:36",
+            207026
         ]
     ],
     "ruby": [
@@ -16359,8 +16359,8 @@
             3589
         ],
         [
-            "2023-05-19T08:27:51",
-            3595
+            "2023-05-19T14:40:36",
+            3597
         ]
     ],
     "rust": [
@@ -17853,8 +17853,8 @@
             5641
         ],
         [
-            "2023-05-19T08:27:51",
-            5648
+            "2023-05-19T14:40:36",
+            5650
         ]
     ],
     "salesforce": [
@@ -19351,8 +19351,8 @@
             19677
         ],
         [
-            "2023-05-19T08:27:51",
-            19727
+            "2023-05-19T14:40:36",
+            19743
         ]
     ],
     "scala": [
@@ -20847,8 +20847,8 @@
             9391
         ],
         [
-            "2023-05-19T08:27:51",
-            9402
+            "2023-05-19T14:40:36",
+            9414
         ]
     ],
     "swift": [
@@ -22341,8 +22341,8 @@
             3734
         ],
         [
-            "2023-05-19T08:27:51",
-            3740
+            "2023-05-19T14:40:36",
+            3743
         ]
     ],
     "terraform": [
@@ -23839,8 +23839,8 @@
             239156
         ],
         [
-            "2023-05-19T08:27:51",
-            239512
+            "2023-05-19T14:40:36",
+            239715
         ]
     ]
 }
diff --git a/.automation/generated/linter-licenses.json b/.automation/generated/linter-licenses.json
@@ -63,28 +63,43 @@
     "powershell_formatter": "MIT",
     "prettier": "MIT",
     "proselint": "BSD-3-Clause",
+    "protolint": "MIT",
     "psalm": "MIT",
     "puppet-lint": "MIT",
     "pylint": "GPL-2.0",
+    "pyright": "Other",
     "raku": "Artistic-2.0",
     "remark-lint": "MIT",
     "revive": "MIT",
+    "rst-lint": "Unlicense",
+    "rstcheck": "MIT",
     "rubocop": "MIT",
+    "ruff": "MIT",
     "scalafix": "Other",
+    "scss-lint": "MIT",
     "secretlint": "MIT",
+    "semgrep": "LGPL-2.1",
     "sfdx-scanner-apex": "MIT",
     "sfdx-scanner-aura": "MIT",
     "sfdx-scanner-lwc": "MIT",
+    "shellcheck": "GPL-3.0",
+    "shfmt": "BSD-3-Clause",
+    "snakefmt": "MIT",
     "snakemake": "MIT",
     "spectral": "Apache-2.0",
+    "sql-lint": "MIT",
     "sqlfluff": "MIT",
     "standard": "MIT",
     "stylelint": "MIT",
     "swiftlint": "MIT",
+    "syft": "Apache-2.0",
     "terraform-fmt": "MPL-2.0",
     "terragrunt": "MIT",
     "terrascan": "Apache-2.0",
+    "tflint": "MPL-2.0",
     "trivy": "Apache-2.0",
     "tsqllint": "MIT",
+    "v8r": "MIT",
+    "vale": "MIT",
     "yamllint": "GPL-3.0"
 }
diff --git a/.mega-linter.yml b/.mega-linter.yml
@@ -29,7 +29,7 @@ YAML_YAMLLINT_FILTER_REGEX_EXCLUDE: '(templates/\.mega-linter\.yml)'
 YAML_PRETTIER_FILTER_REGEX_EXCLUDE: '(templates/\.mega-linter\.yml|mkdocs\.yml)'
 YAML_V8R_FILTER_REGEX_EXCLUDE: '(descriptors|templates/\.mega-linter\.yml|\.codecov\.yml)'
 BASH_FILTER_REGEX_EXCLUDE: "(lib)"
-MARKDOWN_FILTER_REGEX_EXCLUDE: '(license\.md)'
+MARKDOWN_FILTER_REGEX_EXCLUDE: '(license\.md|docs/licenses)'
 MARKDOWN_MARKDOWN_LINK_CHECK_DISABLE_ERRORS: true
 SPELL_MISSPELL_FILTER_REGEX_EXCLUDE: '(\.automation/generated|docs/descriptors|used-by-stats|docs/used-by-stats\.md)'
 SPELL_VALE_FILE_EXTENSIONS:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -62,6 +62,7 @@ Note: Can be used with `oxsecurity/megalinter@beta` in your GitHub Action mega-l
   - Apply many updates after the use of [Vale](https://vale.sh/) on MegaLinter own sources and docs
   - Remove ASCII characters from linters helps displayed in MegaLinter documentation
   - Add instructions to install MegaLinter on Bitbucket Pipelines
+  - Exclude licenses from search results
 
 - Linter versions upgrades
   - [cfn-lint](https://github.com/aws-cloudformation/cfn-lint) from 0.76.1 to **0.76.2** on 2023-04-04