Skip to content

Commit b8dc12b

Browse files
sushi30sushi30
authored
aligned pre-commit with Makefile. added check json (#15295)
* aligned pre-commit with Makefile. added check json * add check for json/yaml files * add path checks for json/yaml validation
1 parent 6b3096f commit b8dc12b

File tree

4 files changed

+132
-10
lines changed

4 files changed

+132
-10
lines changed

‎.github/workflows/validate-jsons-yamls.yml

+96
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
# Copyright 2024 Collate
2+
# Licensed under the Apache License, Version 2.0 (the "License");
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
# http://www.apache.org/licenses/LICENSE-2.0
6+
# Unless required by applicable law or agreed to in writing, software
7+
# distributed under the License is distributed on an "AS IS" BASIS,
8+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9+
# See the License for the specific language governing permissions and
10+
# limitations under the License.
11+
12+
name: Validate JSON/YAML
13+
14+
# read-write repo token
15+
# access to secrets
16+
on:
17+
pull_request_target:
18+
types: [labeled, opened, synchronize, reopened]
19+
paths:
20+
- "**.json"
21+
- "**.yaml"
22+
- "**.yml"
23+
24+
permissions:
25+
contents: read
26+
27+
jobs:
28+
validate-json-yaml:
29+
runs-on: ubuntu-latest
30+
permissions:
31+
pull-requests: write
32+
steps:
33+
- name: Wait for the labeler
34+
uses: lewagon/wait-on-check-action@v1.3.3
35+
if: ${{ github.event_name == 'pull_request_target' }}
36+
with:
37+
ref: ${{ github.event.pull_request.head.sha }}
38+
check-name: Team Label
39+
repo-token: ${{ secrets.GITHUB_TOKEN }}
40+
wait-interval: 90
41+
42+
- name: Verify PR labels
43+
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
44+
if: ${{ github.event_name == 'pull_request_target' }}
45+
with:
46+
github-token: '${{ secrets.GITHUB_TOKEN }}'
47+
valid-labels: 'safe to test'
48+
pull-request-number: '${{ github.event.pull_request.number }}'
49+
disable-reviews: true # To not auto approve changes
50+
51+
- name: Checkout
52+
uses: actions/checkout@v3
53+
with:
54+
ref: ${{ github.event.pull_request.head.sha }}
55+
56+
- name: Set up Python 3.9
57+
uses: actions/setup-python@v4
58+
with:
59+
python-version: 3.9
60+
61+
- name: Install Ubuntu related dependencies
62+
run: |
63+
sudo apt-get update && sudo apt-get install -y libsasl2-dev unixodbc-dev python3-venv
64+
65+
- name: Install Python & Openmetadata related dependencies
66+
run: |
67+
python3 -m venv env
68+
source env/bin/activate
69+
pip install pyyaml
70+
71+
# Add back linting once we have 10/10 on main
72+
- name: Code style check
73+
id: style
74+
continue-on-error: true
75+
run: |
76+
source env/bin/activate
77+
pip install pyyaml
78+
./scripts/validate_json_yaml.sh
79+
80+
- name: Create a comment in the PR with the instructions
81+
if: steps.style.outcome != 'success'
82+
uses: peter-evans/create-or-update-comment@v1
83+
with:
84+
issue-number: ${{ github.event.pull_request.number }}
85+
body: |
86+
**JSON/YAML Validations failed.**
87+
88+
Please run `./scripts/validate_json_yaml.sh` to find any bad json/yaml files and fix them.
89+
You can also use [pre-commit](https://pre-commit.com/) to automate the Python code formatting.
90+
91+
You can install the pre-commit hooks with `make install_test precommit_install`.
92+
93+
- name: Python checkstyle failed, check the comment in the PR
94+
if: steps.style.outcome != 'success'
95+
run: |
96+
exit 1

‎.pre-commit-config.yaml

+17-10
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,29 @@
11
default_language_version:
22
python: python3
33
repos:
4-
- repo: https://github.com/ambv/black
5-
rev: 22.3.0
4+
- repo: https://github.com/pre-commit/pre-commit-hooks
5+
rev: v2.3.0
66
hooks:
7-
- id: black
8-
exclude: ingestion/src/metadata/generated
7+
- id: check-json
8+
exclude: vscode
9+
- repo: https://github.com/hadialqattan/pycln
10+
rev: v2.4.0
11+
hooks:
12+
- id: pycln
13+
files: ^(ingestion|openmetadata-airflow-apis)/
14+
args: [ "--config", "ingestion/pyproject.toml" ]
915
- repo: https://github.com/timothycrosley/isort
1016
rev: 5.12.0
1117
hooks:
1218
- id: isort
13-
args: ["--profile", "black"]
14-
exclude: ingestion/src/metadata/generated
15-
- repo: https://github.com/hadialqattan/pycln
16-
rev: v1.2.4
19+
files: ^(ingestion|openmetadata-airflow-apis)/
20+
args: [ "--settings-file", "ingestion/pyproject.toml" ]
21+
- repo: https://github.com/ambv/black
22+
rev: 22.3.0
1723
hooks:
18-
- id: pycln
19-
exclude: ingestion/src/metadata/generated
24+
- id: black
25+
files: ^(ingestion|openmetadata-airflow-apis)/
26+
args: [ "--config", "ingestion/pyproject.toml" ]
2027
- repo: https://github.com/pre-commit/mirrors-prettier
2128
rev: v2.5.1
2229
hooks:

‎scripts/validate_json_yaml.sh

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/usr/bin/env bash
2+
# bash strict mode
3+
set -eup pipefail
4+
5+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
6+
EXCLUDED_DIRS="vscode"
7+
echo "Validating JSON files..."
8+
git ls-files | grep "\.json$" | grep -vE "^($EXCLUDED_DIRS)/" | while read file; do jq . "$file" >/dev/null 2>&1 || { echo "Invalid JSON in $file"; exit 1; }; done
9+
echo "Validating YAML files..."
10+
git ls-files | grep -E "\.ya?ml$" | grep -vE "^($EXCLUDED_DIRS)/" | while read file; do python ${SCRIPT_DIR}/validate_yaml.py "$file" >/dev/null 2>&1 || { echo "Invalid YAML in $file"; exit 1; }; done

‎scripts/validate_yaml.py

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
import os
2+
import sys
3+
4+
import yaml
5+
6+
path = sys.argv[1]
7+
if os.path.islink(path):
8+
exit()
9+
yaml.safe_load(open(path, "r"))

0 commit comments

Comments
 (0)