Merge remote-tracking branch 'origin/main' into features-checkmake-re-enable

Author: nvuillam

.automation/build.py

@@ -42,10 +42,14 @@
     DEFAULT_DOCKERFILE_RUST_ARGS,
     DEFAULT_RELEASE,
     DEFAULT_REPORT_FOLDER_NAME,
+    DOCKER_PACKAGES_ROOT_URL,
+    GHCR_PACKAGES_ROOT_URL,
     ML_DOC_URL_BASE,
     ML_DOCKER_IMAGE,
     ML_DOCKER_IMAGE_LEGACY,
     ML_DOCKER_IMAGE_LEGACY_V5,
+    ML_DOCKER_IMAGE_WITH_HOST,
+    ML_DOCKER_NAME,
     ML_REPO,
     ML_REPO_URL,
 )
@@ -229,7 +233,7 @@ def generate_flavor(flavor, flavor_info):
     description: "0 if no source file has been updated, 1 if source files has been updated"
 runs:
   using: "docker"
-  image: "docker://{ML_DOCKER_IMAGE}:{image_release}"
+  image: "docker://{ML_DOCKER_IMAGE_WITH_HOST}:{image_release}"
   args:
     - "-v"
     - "/var/run/docker.sock:/var/run/docker.sock:rw"
@@ -293,7 +297,7 @@ def generate_flavor(flavor, flavor_info):
     description: "0 if no source file has been updated, 1 if source files has been updated"
 runs:
   using: "docker"
-  image: "docker://{ML_DOCKER_IMAGE}-{flavor}:{image_release}"
+  image: "docker://{ML_DOCKER_IMAGE_WITH_HOST}-{flavor}:{image_release}"
   args:
     - "-v"
     - "/var/run/docker.sock:/var/run/docker.sock:rw"
@@ -749,7 +753,7 @@ def generate_linter_dockerfiles():
     linters_md += "| Linter key | Docker image | Size |\n"
     linters_md += "| :----------| :----------- | :--: |\n"
     descriptor_files = megalinter.linter_factory.list_descriptor_files()
-    gha_workflow_yml = ["        linter:", "          ["]
+    active_linter_list_lower = []
     for descriptor_file in descriptor_files:
         descriptor_items = []
         with open(descriptor_file, "r", encoding="utf-8") as f:
@@ -807,36 +811,26 @@ def generate_linter_dockerfiles():
             build_dockerfile(
                 dockerfile, descriptor_and_linter, requires_docker, "none", extra_lines
             )
-            gha_workflow_yml += [f'            "{linter_lower_name}",']
-            docker_image = f"{ML_DOCKER_IMAGE}-only-{linter_lower_name}:{VERSION_V}"
+            docker_image = (
+                f"{ML_DOCKER_IMAGE_WITH_HOST}-only-{linter_lower_name}:{VERSION_V}"
+            )
             docker_image_badge = (
                 f"![Docker Image Size (tag)]({BASE_SHIELD_IMAGE_LINK}/"
                 f"{ML_DOCKER_IMAGE}-only-{linter_lower_name}/{VERSION_V})"
             )
             linters_md += (
                 f"| {linter.name} | {docker_image} | {docker_image_badge}  |\n"
             )
+            if not (hasattr(linter, "disabled") and linter.disabled is True):
+                active_linter_list_lower += [linter_lower_name]
+
+    # Write linter_list_lower in .automation/generated/linters_matrix.json
+    linters_matrix_file = f"{REPO_HOME}/.automation/generated/linters_matrix.json"
+    with open(linters_matrix_file, "w", encoding="utf-8") as file:
+        json.dump(active_linter_list_lower, file, indent=2, sort_keys=True)
+        file.write("\n")
+        logging.info(f"Updated {linters_matrix_file}")
 
-    # Update github action workflow
-    gha_workflow_yml += ["          ]"]
-    replace_in_file(
-        f"{REPO_HOME}/.github/workflows/deploy-DEV-linters.yml",
-        "# linters-start",
-        "# linters-end",
-        "\n".join(gha_workflow_yml),
-    )
-    replace_in_file(
-        f"{REPO_HOME}/.github/workflows/deploy-BETA-linters.yml",
-        "# linters-start",
-        "# linters-end",
-        "\n".join(gha_workflow_yml),
-    )
-    replace_in_file(
-        f"{REPO_HOME}/.github/workflows/deploy-RELEASE-linters.yml",
-        "# linters-start",
-        "# linters-end",
-        "\n".join(gha_workflow_yml),
-    )
     # Write MD file
     file = open(f"{REPO_HOME}/docs/standalone-linters.md", "w", encoding="utf-8")
     file.write(linters_md + "\n")
@@ -1161,13 +1155,16 @@ def generate_descriptor_documentation(descriptor):
 
 def generate_flavor_documentation(flavor_id, flavor, linters_tables_md):
     flavor_github_action = f"{ML_REPO}/flavors/{flavor_id}@{VERSION_V}"
-    flavor_docker_image = f"{ML_DOCKER_IMAGE}-{flavor_id}:{VERSION_V}"
+    flavor_docker_image = f"{ML_DOCKER_IMAGE_WITH_HOST}-{flavor_id}:{VERSION_V}"
+    flavor_docker_image_dockerhub = (
+        f"docker.io/{ML_DOCKER_IMAGE}-{flavor_id}:{VERSION_V}"
+    )
     docker_image_badge = (
         f"![Docker Image Size (tag)]({BASE_SHIELD_IMAGE_LINK}/"
         f"{ML_DOCKER_IMAGE}-{flavor_id}/{VERSION_V})"
     )
     docker_pulls_badge = (
-        f"![Docker Pulls]({BASE_SHIELD_COUNT_LINK}/" f"{ML_DOCKER_IMAGE}-{flavor_id})"
+        f"![Docker Pulls]({BASE_SHIELD_COUNT_LINK}/{ML_DOCKER_IMAGE}-{flavor_id})"
     )
     flavor_doc_md = [
         "---",
@@ -1187,7 +1184,12 @@ def generate_flavor_documentation(flavor_id, flavor, linters_tables_md):
         "## Usage",
         "",
         f"- [GitHub Action]({MKDOCS_URL_ROOT}/installation/#github-action): **{flavor_github_action}**",
-        f"- Docker image: **{flavor_docker_image}**",
+        "",
+        "- Docker images:",
+        "",
+        f"  - GitHub Packages: **{flavor_docker_image}**",
+        f"  - Docker Hub: **{flavor_docker_image_dockerhub}**",
+        "",
         f"- [mega-linter-runner]({MKDOCS_URL_ROOT}/mega-linter-runner/): `mega-linter-runner --flavor {flavor_id}`",
         "",
         "## Embedded linters",
@@ -2003,9 +2005,7 @@ def build_flavors_md_table(filter_linter_name=None, replace_link=False):
         + +len(linters_by_type["other"])
     )
     docker_image_badge = f"![Docker Image Size (tag)]({BASE_SHIELD_IMAGE_LINK}/{ML_DOCKER_IMAGE}/{VERSION_V})"
-    docker_pulls_badge = (
-        f"![Docker Pulls]({BASE_SHIELD_COUNT_LINK}/" f"{ML_DOCKER_IMAGE})"
-    )
+    docker_pulls_badge = f"![Docker Pulls]({BASE_SHIELD_COUNT_LINK}/{ML_DOCKER_IMAGE})"
     md_line_all = (
         f"| {icon_html} | [all]({MKDOCS_URL_ROOT}/supported-linters/) | "
         f"Default MegaLinter Flavor | {str(linters_number)} | {docker_image_badge} {docker_pulls_badge} |"
@@ -2106,29 +2106,32 @@ def update_docker_pulls_counter():
     now_str = datetime.now().replace(microsecond=0).isoformat()
     for flavor_id in all_flavors_ids:
         if flavor_id == "all":
-            docker_image_url = (
-                f"https://hub.docker.com/v2/repositories/{ML_DOCKER_IMAGE}"
-            )
+            ghcr_image_url = f"{GHCR_PACKAGES_ROOT_URL}/{ML_DOCKER_NAME}"
+            docker_image_url = f"{DOCKER_PACKAGES_ROOT_URL}/{ML_DOCKER_IMAGE}"
             legacy_docker_image_url = (
-                f"https://hub.docker.com/v2/repositories/{ML_DOCKER_IMAGE_LEGACY}"
+                f"{DOCKER_PACKAGES_ROOT_URL}/{ML_DOCKER_IMAGE_LEGACY}"
             )
             legacy_v5_docker_image_url = (
-                f"https://hub.docker.com/v2/repositories/{ML_DOCKER_IMAGE_LEGACY_V5}"
+                f"{DOCKER_PACKAGES_ROOT_URL}/{ML_DOCKER_IMAGE_LEGACY_V5}"
             )
         else:
+            ghcr_image_url = f"{GHCR_PACKAGES_ROOT_URL}/{ML_DOCKER_NAME}-{flavor_id}"
             docker_image_url = (
-                f"https://hub.docker.com/v2/repositories/{ML_DOCKER_IMAGE}-{flavor_id}"
+                f"{DOCKER_PACKAGES_ROOT_URL}/{ML_DOCKER_IMAGE}-{flavor_id}"
+            )
+            legacy_docker_image_url = (
+                f"{DOCKER_PACKAGES_ROOT_URL}/{ML_DOCKER_IMAGE_LEGACY}-{flavor_id}"
             )
-            legacy_docker_image_url = f"https://hub.docker.com/v2/repositories/{ML_DOCKER_IMAGE_LEGACY}-{flavor_id}"
             legacy_v5_docker_image_url = (
-                "https://hub.docker.com/v2/repositories/"
+                f"{DOCKER_PACKAGES_ROOT_URL}/"
                 + f"{ML_DOCKER_IMAGE_LEGACY_V5}-{flavor_id}"
             )
 
+        flavor_count_0 = perform_count_request(ghcr_image_url)
         flavor_count_1 = perform_count_request(docker_image_url)
         flavor_count_2 = perform_count_request(legacy_docker_image_url)
         flavor_count_3 = perform_count_request(legacy_v5_docker_image_url)
-        flavor_count = flavor_count_1 + flavor_count_2 + flavor_count_3
+        flavor_count = flavor_count_0 + flavor_count_1 + flavor_count_2 + flavor_count_3
         logging.info(f"- docker pulls for {flavor_id}: {flavor_count}")
         total_count = total_count + flavor_count
         flavor_stats = list(docker_stats.get(flavor_id, []))
@@ -3503,29 +3506,6 @@ def update_dependents_info():
     os.system(" ".join(command))
 
 
-def update_workflows_linters():
-    descriptors, _ = list_descriptors_for_build()
-
-    linters = ""
-
-    for descriptor in descriptors:
-        for linter in descriptor["linters"]:
-            if "disabled" in linter and linter["disabled"] is True:
-                continue
-            if "name" in linter:
-                name = linter["name"].lower()
-            else:
-                lang_lower = descriptor["descriptor_id"].lower()
-                linter_name_lower = linter["linter_name"].lower().replace("-", "_")
-                name = f"{lang_lower}_{linter_name_lower}"
-
-            linters += f'            "{name}",\n'
-
-    update_workflow_linters(".github/workflows/deploy-DEV-linters.yml", linters)
-    update_workflow_linters(".github/workflows/deploy-BETA-linters.yml", linters)
-    update_workflow_linters(".github/workflows/deploy-RELEASE-linters.yml", linters)
-
-
 def update_workflow_linters(file_path, linters):
     with open(file_path, "r", encoding="utf-8") as f:
         file_content = f.read()
@@ -3640,7 +3620,6 @@ def build_custom_flavor(dockerfile):
         generate_all_flavors()
         generate_linter_dockerfiles()
         generate_linter_test_classes()
-        update_workflows_linters()
         if UPDATE_DOC is True:
             logging.info("Running documentation generators…")
             # refresh_users_info() # deprecated since now we use github-dependents-info