From f4a5929c87ed58c15de6d2374d47325df7f641d3 Mon Sep 17 00:00:00 2001 From: Imri Paran Date: Thu, 18 Apr 2024 15:32:38 +0200 Subject: [PATCH] aligned pre-commit with Makefile. added check json (#15295) * aligned pre-commit with Makefile. added check json * add check for json/yaml files * add path checks for json/yaml validation --- .github/workflows/validate-jsons-yamls.yml | 96 ++++++++++++++++++++++ .pre-commit-config.yaml | 27 +++--- scripts/validate_json_yaml.sh | 10 +++ scripts/validate_yaml.py | 9 ++ 4 files changed, 132 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/validate-jsons-yamls.yml create mode 100755 scripts/validate_json_yaml.sh create mode 100644 scripts/validate_yaml.py diff --git a/.github/workflows/validate-jsons-yamls.yml b/.github/workflows/validate-jsons-yamls.yml new file mode 100644 index 000000000000..af7cc27a1317 --- /dev/null +++ b/.github/workflows/validate-jsons-yamls.yml @@ -0,0 +1,96 @@ +# Copyright 2024 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Validate JSON/YAML + +# read-write repo token +# access to secrets +on: + pull_request_target: + types: [labeled, opened, synchronize, reopened] + paths: + - "**.json" + - "**.yaml" + - "**.yml" + +permissions: + contents: read + +jobs: + validate-json-yaml: + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Wait for the labeler + uses: lewagon/wait-on-check-action@v1.3.3 + if: ${{ github.event_name == 'pull_request_target' }} + with: + ref: ${{ github.event.pull_request.head.sha }} + check-name: Team Label + repo-token: ${{ secrets.GITHUB_TOKEN }} + wait-interval: 90 + + - name: Verify PR labels + uses: jesusvasquez333/verify-pr-label-action@v1.4.0 + if: ${{ github.event_name == 'pull_request_target' }} + with: + github-token: '${{ secrets.GITHUB_TOKEN }}' + valid-labels: 'safe to test' + pull-request-number: '${{ github.event.pull_request.number }}' + disable-reviews: true # To not auto approve changes + + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up Python 3.9 + uses: actions/setup-python@v4 + with: + python-version: 3.9 + + - name: Install Ubuntu related dependencies + run: | + sudo apt-get update && sudo apt-get install -y libsasl2-dev unixodbc-dev python3-venv + + - name: Install Python & Openmetadata related dependencies + run: | + python3 -m venv env + source env/bin/activate + pip install pyyaml + + # Add back linting once we have 10/10 on main + - name: Code style check + id: style + continue-on-error: true + run: | + source env/bin/activate + pip install pyyaml + ./scripts/validate_json_yaml.sh + + - name: Create a comment in the PR with the instructions + if: steps.style.outcome != 'success' + uses: peter-evans/create-or-update-comment@v1 + with: + issue-number: ${{ github.event.pull_request.number }} + body: | + **JSON/YAML Validations failed.** + + Please run `./scripts/validate_json_yaml.sh` to find any bad json/yaml files and fix them. + You can also use [pre-commit](https://pre-commit.com/) to automate the Python code formatting. + + You can install the pre-commit hooks with `make install_test precommit_install`. + + - name: Python checkstyle failed, check the comment in the PR + if: steps.style.outcome != 'success' + run: | + exit 1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f26675ce6b08..9b68634eb0dc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,22 +1,29 @@ default_language_version: python: python3 repos: - - repo: https://github.com/ambv/black - rev: 22.3.0 + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 hooks: - - id: black - exclude: ingestion/src/metadata/generated + - id: check-json + exclude: vscode + - repo: https://github.com/hadialqattan/pycln + rev: v2.4.0 + hooks: + - id: pycln + files: ^(ingestion|openmetadata-airflow-apis)/ + args: [ "--config", "ingestion/pyproject.toml" ] - repo: https://github.com/timothycrosley/isort rev: 5.12.0 hooks: - id: isort - args: ["--profile", "black"] - exclude: ingestion/src/metadata/generated - - repo: https://github.com/hadialqattan/pycln - rev: v1.2.4 + files: ^(ingestion|openmetadata-airflow-apis)/ + args: [ "--settings-file", "ingestion/pyproject.toml" ] + - repo: https://github.com/ambv/black + rev: 22.3.0 hooks: - - id: pycln - exclude: ingestion/src/metadata/generated + - id: black + files: ^(ingestion|openmetadata-airflow-apis)/ + args: [ "--config", "ingestion/pyproject.toml" ] - repo: https://github.com/pre-commit/mirrors-prettier rev: v2.5.1 hooks: diff --git a/scripts/validate_json_yaml.sh b/scripts/validate_json_yaml.sh new file mode 100755 index 000000000000..011031890ad4 --- /dev/null +++ b/scripts/validate_json_yaml.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# bash strict mode +set -eup pipefail + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +EXCLUDED_DIRS="vscode" +echo "Validating JSON files..." +git ls-files | grep "\.json$" | grep -vE "^($EXCLUDED_DIRS)/" | while read file; do jq . "$file" >/dev/null 2>&1 || { echo "Invalid JSON in $file"; exit 1; }; done +echo "Validating YAML files..." +git ls-files | grep -E "\.ya?ml$" | grep -vE "^($EXCLUDED_DIRS)/" | while read file; do python ${SCRIPT_DIR}/validate_yaml.py "$file" >/dev/null 2>&1 || { echo "Invalid YAML in $file"; exit 1; }; done diff --git a/scripts/validate_yaml.py b/scripts/validate_yaml.py new file mode 100644 index 000000000000..fd43fac50f29 --- /dev/null +++ b/scripts/validate_yaml.py @@ -0,0 +1,9 @@ +import os +import sys + +import yaml + +path = sys.argv[1] +if os.path.islink(path): + exit() +yaml.safe_load(open(path, "r"))