From 4dfe41b1d1aba5d7f0db20bea34a71a87783b9ac Mon Sep 17 00:00:00 2001 From: sushi30 Date: Fri, 2 Feb 2024 20:14:38 +0100 Subject: [PATCH 1/5] docs(check-prerequisites): prerequisites recipe - added scripts/check_prerequisites.sh. - added usage in docs. - added prerequisites to Makefile. --- Makefile | 4 + .../build-code-and-run-tests/prerequisites.md | 8 +- scripts/check_prerequisites.sh | 107 ++++++++++++++++++ 3 files changed, 118 insertions(+), 1 deletion(-) create mode 100755 scripts/check_prerequisites.sh diff --git a/Makefile b/Makefile index 61623e846bd4..7a25de922f9e 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,10 @@ include ingestion/Makefile help: @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":"}; {printf "\033[35m%-35s\033[0m %s\n", $$2, $$3}' +.PHONY: prerequisites +prerequisites: + ./scripts/check_prerequisites.sh + .PHONY: install_e2e_tests install_e2e_tests: ## Install the ingestion module with e2e test dependencies (playwright) python -m pip install "ingestion[e2e_test]/" diff --git a/openmetadata-docs/content/v1.3.x/developers/contribute/build-code-and-run-tests/prerequisites.md b/openmetadata-docs/content/v1.3.x/developers/contribute/build-code-and-run-tests/prerequisites.md index 16128315f679..be0699f808d7 100644 --- a/openmetadata-docs/content/v1.3.x/developers/contribute/build-code-and-run-tests/prerequisites.md +++ b/openmetadata-docs/content/v1.3.x/developers/contribute/build-code-and-run-tests/prerequisites.md @@ -18,7 +18,13 @@ OpenMetadata being a full stack project, we use the following for development: - [Yarn ^1.22.0](https://classic.yarnpkg.com/lang/en/docs/install/) - [Rpm (Optional, only to run RPM profile with maven)](https://macappstore.org/rpm/) -- Here is a snapshot of a working environment on a Macbook. +To validate the installation of the above tools, you can run: + +```shell +make prerequisites +``` + +### Example Snapshot on a Macbook ```shell > docker --version diff --git a/scripts/check_prerequisites.sh b/scripts/check_prerequisites.sh new file mode 100755 index 000000000000..c2b17986fe52 --- /dev/null +++ b/scripts/check_prerequisites.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +declare -A python +python["name"]="Python" +python["version_command"]="python --version 2>&1 | awk '{print \$2}'" +python["required_version"]="3.9 3.10 3.11" + +declare -A docker +docker["name"]="Docker" +docker["version_command"]="docker --version | awk '{print \$3}'" +docker["required_version"]="20 21 22 23 24" + +declare -A maven +maven["name"]="Maven" +maven["version_command"]="mvn --version | head -n1 | awk '{print \$3}'" +maven["required_version"]="3.6 3.7 3.8 3.9" + +declare -A java +java["name"]="Java" +java["version_command"]="java -version 2>&1 | awk -F'\"' '/version/ {print \$2}'" +java["required_version"]="11 16 17" + +declare -A jq +jq["name"]="jq" +jq["version_command"]="jq --version | awk -F- '{print \$2}'" +jq["required_version"]="any" + +declare -A node +node["name"]="Node" +node["version_command"]="node --version" +node["required_version"]="16 17 18" + +declare -A yarn +yarn["name"]="Yarn" +yarn["version_command"]="yarn --version" +yarn["required_version"]="1.22 1.23 1.24" + +declare -A antlr +antlr["name"]="ANTLR" +antlr["version_command"]="antlr4 | head -n1 | awk 'NF>1{print \$NF}'" +antlr["required_version"]="4.9" + + +code=0 + +function print_error() { + >&2 echo "✗ ERROR: $1" +} + +check_command_existence() { + which "$1" >/dev/null 2>&1 + res=$? + if [[ $res -ne 0 ]]; then + print_error "$command is not installed." + code=2 + fi + echo $res +} + +check_version() { + local tool_name=$1 + local current=$2 + local required=$3 + IFS=' ' read -r -a required_versions <<< "$required" + if [[ "$required" == "any" ]]; then + echo "✓ $tool_name version $current is supported." + return + fi + for v in "${required_versions[@]}"; do + if [[ "$current" =~ $v.* ]]; then + echo "✓ $tool_name version $version is supported." + return + fi + done + print_error "$tool_name version $version is not supported. Supported versions are: $required" + code=1 +} + +declare -n dependency +for dependency in python docker java maven jq node yarn antlr; do + command=$(echo "${dependency["version_command"]}" | awk '{print $1}') + if [[ $(check_command_existence "$command") -ne 0 ]]; then + continue + fi + tool_name=${dependency["name"]} + version=$(eval ${dependency["version_command"]}) + required_version=${dependency["required_version"]} + check_version $tool_name "$version" "$required_version" +done +if [[ $code -eq 0 ]]; then + echo "✓ All prerequisites are met." +else + print_error "Some prerequisites are not met." +fi +exit $code From 4c58b0e82532642de3975b0fd587cfb89e98c234 Mon Sep 17 00:00:00 2001 From: sushi30 Date: Fri, 2 Feb 2024 20:22:55 +0100 Subject: [PATCH 2/5] docs(check-prerequisites): fix for docker version --- scripts/check_prerequisites.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check_prerequisites.sh b/scripts/check_prerequisites.sh index c2b17986fe52..a03ce9a3a64d 100755 --- a/scripts/check_prerequisites.sh +++ b/scripts/check_prerequisites.sh @@ -19,7 +19,7 @@ python["required_version"]="3.9 3.10 3.11" declare -A docker docker["name"]="Docker" -docker["version_command"]="docker --version | awk '{print \$3}'" +docker["version_command"]="docker --version | awk '{print $3}' | sed 's/,//'" docker["required_version"]="20 21 22 23 24" declare -A maven From e03477ce3b1c2c66055594d9e7cdec61e43fc32e Mon Sep 17 00:00:00 2001 From: sushi30 Date: Fri, 2 Feb 2024 20:23:31 +0100 Subject: [PATCH 3/5] docs(check-prerequisites): fix for docker version --- docker/development/docker-compose.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml index f2063295a7bd..8f2a088c1168 100644 --- a/docker/development/docker-compose.yml +++ b/docker/development/docker-compose.yml @@ -255,6 +255,8 @@ services: - local_app_net openmetadata-server: + deploy: + replicas: 0 build: context: ../../. dockerfile: docker/development/Dockerfile @@ -465,8 +467,8 @@ services: condition: service_started mysql: condition: service_healthy - openmetadata-server: - condition: service_started +# openmetadata-server: +# condition: service_started expose: - 8080 ports: @@ -475,7 +477,7 @@ services: - local_app_net volumes: - ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs - - ingestion-volume-dags:/opt/airflow/dags +# - ingestion-volume-dags:/opt/airflow/dags - ingestion-volume-tmp:/tmp - /var/run/docker.sock:/var/run/docker.sock:z # Need 600 permissions to run DockerOperator From 1d4b355bc3e80f088698b269139025de393c35ee Mon Sep 17 00:00:00 2001 From: sushi30 Date: Fri, 2 Feb 2024 20:31:59 +0100 Subject: [PATCH 4/5] docs(check-prerequisites): fix for docker version --- docker/development/docker-compose.yml | 4 +--- scripts/check_prerequisites.sh | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml index 8f2a088c1168..421b049d4103 100644 --- a/docker/development/docker-compose.yml +++ b/docker/development/docker-compose.yml @@ -255,8 +255,6 @@ services: - local_app_net openmetadata-server: - deploy: - replicas: 0 build: context: ../../. dockerfile: docker/development/Dockerfile @@ -477,7 +475,7 @@ services: - local_app_net volumes: - ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs -# - ingestion-volume-dags:/opt/airflow/dags + - ingestion-volume-dags:/opt/airflow/dags - ingestion-volume-tmp:/tmp - /var/run/docker.sock:/var/run/docker.sock:z # Need 600 permissions to run DockerOperator diff --git a/scripts/check_prerequisites.sh b/scripts/check_prerequisites.sh index a03ce9a3a64d..20b39b26a939 100755 --- a/scripts/check_prerequisites.sh +++ b/scripts/check_prerequisites.sh @@ -19,7 +19,7 @@ python["required_version"]="3.9 3.10 3.11" declare -A docker docker["name"]="Docker" -docker["version_command"]="docker --version | awk '{print $3}' | sed 's/,//'" +docker["version_command"]="docker --version | awk '{print \$3}' | sed 's/,//'" docker["required_version"]="20 21 22 23 24" declare -A maven From 6a55cc4d36f4091bc72eb567dc71adbcd2bdb66a Mon Sep 17 00:00:00 2001 From: sushi30 Date: Fri, 2 Feb 2024 20:32:42 +0100 Subject: [PATCH 5/5] docs(check-prerequisites): revert docker-compose.yml --- docker/development/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml index 421b049d4103..f2063295a7bd 100644 --- a/docker/development/docker-compose.yml +++ b/docker/development/docker-compose.yml @@ -465,8 +465,8 @@ services: condition: service_started mysql: condition: service_healthy -# openmetadata-server: -# condition: service_started + openmetadata-server: + condition: service_started expose: - 8080 ports: