From 8b27ea9c656ba5e30a6e75a163435a2f7942a4ad Mon Sep 17 00:00:00 2001
From: sushi30 <imri.paran@gmail.com>
Date: Wed, 21 Feb 2024 14:24:35 +0100
Subject: [PATCH 1/3] aligned pre-commit with Makefile. added check json

---
 .pre-commit-config.yaml | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index f26675ce6b08..9b68634eb0dc 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,22 +1,29 @@
 default_language_version:
   python: python3
 repos:
-  - repo: https://github.com/ambv/black
-    rev: 22.3.0
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
     hooks:
-      - id: black
-        exclude: ingestion/src/metadata/generated
+      - id: check-json
+        exclude: vscode
+  - repo: https://github.com/hadialqattan/pycln
+    rev: v2.4.0
+    hooks:
+      - id: pycln
+        files: ^(ingestion|openmetadata-airflow-apis)/
+        args: [ "--config", "ingestion/pyproject.toml" ]
   - repo: https://github.com/timothycrosley/isort
     rev: 5.12.0
     hooks:
       - id: isort
-        args: ["--profile", "black"]
-        exclude: ingestion/src/metadata/generated
-  - repo: https://github.com/hadialqattan/pycln
-    rev: v1.2.4
+        files: ^(ingestion|openmetadata-airflow-apis)/
+        args: [ "--settings-file", "ingestion/pyproject.toml" ]
+  - repo: https://github.com/ambv/black
+    rev: 22.3.0
     hooks:
-      - id: pycln
-        exclude: ingestion/src/metadata/generated
+      - id: black
+        files: ^(ingestion|openmetadata-airflow-apis)/
+        args: [ "--config", "ingestion/pyproject.toml" ]
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v2.5.1
     hooks:

From f09a7ea4444deb18c4f61a3d002458729c811087 Mon Sep 17 00:00:00 2001
From: sushi30 <imri.paran@gmail.com>
Date: Thu, 22 Feb 2024 16:30:14 +0100
Subject: [PATCH 2/3] add check for json/yaml files

---
 .github/workflows/validate-jsons-yamls.yml | 92 ++++++++++++++++++++++
 scripts/validate_json_yaml.sh              | 10 +++
 scripts/validate_yaml.py                   |  9 +++
 3 files changed, 111 insertions(+)
 create mode 100644 .github/workflows/validate-jsons-yamls.yml
 create mode 100755 scripts/validate_json_yaml.sh
 create mode 100644 scripts/validate_yaml.py

diff --git a/.github/workflows/validate-jsons-yamls.yml b/.github/workflows/validate-jsons-yamls.yml
new file mode 100644
index 000000000000..58b2707b0426
--- /dev/null
+++ b/.github/workflows/validate-jsons-yamls.yml
@@ -0,0 +1,92 @@
+#  Copyright 2024 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+name: Validate JSON/YAML
+
+# read-write repo token
+# access to secrets
+on:
+  pull_request_target:
+    types: [labeled, opened, synchronize, reopened]
+
+permissions:
+  contents: read
+
+jobs:
+  validate-json-yaml:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Wait for the labeler
+        uses: lewagon/wait-on-check-action@v1.3.3
+        if: ${{ github.event_name == 'pull_request_target' }}
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          check-name: Team Label
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          wait-interval: 90
+
+      - name: Verify PR labels
+        uses: jesusvasquez333/verify-pr-label-action@v1.4.0
+        if: ${{ github.event_name == 'pull_request_target' }}
+        with:
+          github-token: '${{ secrets.GITHUB_TOKEN }}'
+          valid-labels: 'safe to test'
+          pull-request-number: '${{ github.event.pull_request.number }}'
+          disable-reviews: true  # To not auto approve changes
+
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Install Ubuntu related dependencies
+        run: |
+          sudo apt-get update  && sudo apt-get install -y libsasl2-dev unixodbc-dev python3-venv
+
+      - name: Install Python & Openmetadata related dependencies
+        run: |
+          python3 -m venv env
+          source env/bin/activate
+          pip install pyyaml
+
+      # Add back linting once we have 10/10 on main
+      - name: Code style check
+        id: style
+        continue-on-error: true
+        run: |
+          source env/bin/activate
+          pip install pyyaml
+          ./scripts/validate_json_yaml.sh
+
+      - name: Create a comment in the PR with the instructions
+        if: steps.style.outcome != 'success'
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          body: |
+            **JSON/YAML Validations failed.**
+
+            Please run `./scripts/validate_json_yaml.sh` to find any bad json/yaml files and fix them.
+            You can also use [pre-commit](https://pre-commit.com/) to automate the Python code formatting.
+
+            You can install the pre-commit hooks with `make install_test precommit_install`.
+
+      - name: Python checkstyle failed, check the comment in the PR
+        if: steps.style.outcome != 'success'
+        run: |
+          exit 1
diff --git a/scripts/validate_json_yaml.sh b/scripts/validate_json_yaml.sh
new file mode 100755
index 000000000000..011031890ad4
--- /dev/null
+++ b/scripts/validate_json_yaml.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# bash strict mode
+set -eup pipefail
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+EXCLUDED_DIRS="vscode"
+echo "Validating JSON files..."
+git ls-files | grep "\.json$" | grep -vE "^($EXCLUDED_DIRS)/" | while read file; do jq . "$file" >/dev/null 2>&1 || { echo "Invalid JSON in $file"; exit 1; }; done
+echo "Validating YAML files..."
+git ls-files | grep -E "\.ya?ml$" | grep -vE "^($EXCLUDED_DIRS)/" | while read file; do python ${SCRIPT_DIR}/validate_yaml.py "$file" >/dev/null 2>&1 || { echo "Invalid YAML in $file"; exit 1; }; done
diff --git a/scripts/validate_yaml.py b/scripts/validate_yaml.py
new file mode 100644
index 000000000000..fd43fac50f29
--- /dev/null
+++ b/scripts/validate_yaml.py
@@ -0,0 +1,9 @@
+import os
+import sys
+
+import yaml
+
+path = sys.argv[1]
+if os.path.islink(path):
+    exit()
+yaml.safe_load(open(path, "r"))

From f4f55023972bfdb7ea9fa9d21de25a3c80cfdf09 Mon Sep 17 00:00:00 2001
From: sushi30 <imri.paran@gmail.com>
Date: Thu, 22 Feb 2024 17:30:54 +0100
Subject: [PATCH 3/3] add path checks for json/yaml validation

---
 .github/workflows/validate-jsons-yamls.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/validate-jsons-yamls.yml b/.github/workflows/validate-jsons-yamls.yml
index 58b2707b0426..af7cc27a1317 100644
--- a/.github/workflows/validate-jsons-yamls.yml
+++ b/.github/workflows/validate-jsons-yamls.yml
@@ -16,6 +16,10 @@ name: Validate JSON/YAML
 on:
   pull_request_target:
     types: [labeled, opened, synchronize, reopened]
+    paths:
+      - "**.json"
+      - "**.yaml"
+      - "**.yml"
 
 permissions:
   contents: read