Merge branch 'fix_ingestion_depency_script' of https://github.com/all…

…enkallz/OpenMetadata into fix_ingestion_depency_script

.github/workflows/cypress-integration-tests-postgresql.yml

@@ -32,10 +32,6 @@ concurrency:
 jobs:
   cypress-ci-postgresql:
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        job: [0, 1, 2]
     environment: test
     steps:
       - name: Free Disk Space (Ubuntu)

.github/workflows/openmetadata-airflow-apis.yml

@@ -34,6 +34,6 @@ jobs:
         run: |
           make install_dev install_apis
           cd openmetadata-airflow-apis; \
-            python setup.py build sdist bdist_wheel; \
+            python -m build; \
             twine check dist/*; \
             twine upload dist/* --verbose

.github/workflows/py-cli-e2e-tests.yml

@@ -25,7 +25,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        e2e-test: ['bigquery', 'dbt_redshift', 'metabase', 'mssql', 'mysql', 'redash', 'snowflake', 'tableau', 'powerbi', 'vertica', 'python', 'redshift', 'hive', 'quicksight', 'datalake_s3', 'postgres', 'oracle', 'athena', 'bigquery_multiple_project']
+        e2e-test: ['bigquery', 'dbt_redshift', 'metabase', 'mssql', 'mysql', 'redash', 'snowflake', 'tableau', 'powerbi', 'vertica', 'python', 'redshift', 'quicksight', 'datalake_s3', 'postgres', 'oracle', 'athena', 'bigquery_multiple_project']
     environment: test
 
     steps:

.github/workflows/py-ingestion-publish.yml

@@ -37,6 +37,6 @@ jobs:
           sudo make install_antlr_cli
           make install_dev generate install
           cd ingestion; \
-            python setup.py build sdist bdist_wheel; \
+            python -m build; \
             twine check dist/*; \
             twine upload dist/* --verbose

Makefile

@@ -4,7 +4,7 @@ include ingestion/Makefile
 
 .PHONY: help
 help:
-	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[35m%-30s\033[0m %s\n", $$1, $$2}'
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":"}; {printf "\033[35m%-35s\033[0m %s\n", $$2, $$3}'
 
 .PHONY: install_e2e_tests
 install_e2e_tests:  ## Install the ingestion module with e2e test dependencies (playwright)
@@ -24,40 +24,6 @@ yarn_install_cache:  ## Use Yarn to install UI dependencies
 yarn_start_dev_ui:  ## Run the UI locally with Yarn
 	cd openmetadata-ui/src/main/resources/ui && yarn start
 
-## Ingestion Core
-.PHONY: core_install_dev
-core_install_dev:  ## Prepare a venv for the ingestion-core module
-	cd ingestion-core; \
-		rm -rf venv; \
-		python3 -m venv venv; \
-		. venv/bin/activate; \
-		python3 -m pip install ".[dev]"
-
-.PHONY: core_clean
-core_clean:  ## Clean the ingestion-core generated files
-	rm -rf ingestion-core/src/metadata/generated
-	rm -rf ingestion-core/build
-	rm -rf ingestion-core/dist
-
-.PHONY: core_generate
-core_generate:  ## Generate the pydantic models from the JSON Schemas to the ingestion-core module
-	$(MAKE) core_install_dev
-	mkdir -p ingestion-core/src/metadata/generated; \
-	. ingestion-core/venv/bin/activate; \
-	datamodel-codegen --input openmetadata-spec/src/main/resources/json/schema  --input-file-type jsonschema --output ingestion-core/src/metadata/generated/schema
-	$(MAKE) core_py_antlr
-
-.PHONY: core_bump_version_dev
-core_bump_version_dev:  ## Bump a `dev` version to the ingestion-core module. To be used when schemas are updated
-	$(MAKE) core_install_dev
-	cd ingestion-core; \
-		. venv/bin/activate; \
-		python -m incremental.update metadata --dev
-
-.PHONY: core_py_antlr
-core_py_antlr:  ## Generate the Python core code for parsing FQNs under ingestion-core
-	antlr4 -Dlanguage=Python3 -o ingestion-core/src/metadata/generated/antlr ${PWD}/openmetadata-spec/src/main/antlr4/org/openmetadata/schema/*.g4
-
 .PHONY: py_antlr
 py_antlr:  ## Generate the Python code for parsing FQNs
 	antlr4 -Dlanguage=Python3 -o ingestion/src/metadata/generated/antlr ${PWD}/openmetadata-spec/src/main/antlr4/org/openmetadata/schema/*.g4
@@ -103,8 +69,8 @@ SNYK_ARGS := --severity-threshold=high
 .PHONY: snyk-ingestion-report
 snyk-ingestion-report:  ## Uses Snyk CLI to validate the ingestion code and container. Don't stop the execution
 	@echo "Validating Ingestion container..."
-	docker build -t openmetadata-ingestion:scan -f ingestion/Dockerfile .
-	snyk container test openmetadata-ingestion:scan --file=ingestion/Dockerfile $(SNYK_ARGS) --json > security-report/ingestion-docker-scan.json | true;
+	docker build -t openmetadata-ingestion:scan -f ingestion/Dockerfile.ci .
+	snyk container test openmetadata-ingestion:scan --file=ingestion/Dockerfile.ci $(SNYK_ARGS) --json > security-report/ingestion-docker-scan.json | true;
 	@echo "Validating ALL ingestion dependencies. Make sure the venv is activated."
 	cd ingestion; \
 		pip freeze > scan-requirements.txt; \
@@ -171,7 +137,7 @@ generate-schema-docs:  ## Generates markdown files for documenting the JSON Sche
 
 #Upgrade release automation scripts below	
 .PHONY: update_all
-update_all: ## To update all the release related files run make update_all RELEASE_VERSION=2.2.2 PY_RELEASE_VERSION=2.2.2.2
+update_all:  ## To update all the release related files run make update_all RELEASE_VERSION=2.2.2 PY_RELEASE_VERSION=2.2.2.2
 	@echo "The release version is: $(RELEASE_VERSION)" ; \
 	echo "The python metadata release version: $(PY_RELEASE_VERSION)" ; \
 	$(MAKE) update_maven ; \
@@ -184,15 +150,15 @@ update_all: ## To update all the release related files run make update_all RELEA
 #make update_all RELEASE_VERSION=2.2.2 PY_RELEASE_VERSION=2.2.2.2
 
 .PHONY: update_maven
-update_maven: ## To update the common and pom.xml maven version
+update_maven:  ## To update the common and pom.xml maven version
 	@echo "Updating Maven projects to version $(RELEASE_VERSION)..."; \
 	mvn versions:set -DnewVersion=$(RELEASE_VERSION)
 #remove comment and use the below section when want to use this sub module "update_maven" independently to update github actions
 #make update_maven RELEASE_VERSION=2.2.2
 
 
 .PHONY: update_github_action_paths
-update_github_action_paths: ## To update the github action ci docker files
+update_github_action_paths:  ## To update the github action ci docker files
 	@echo "Updating docker github action release version to $(RELEASE_VERSION)... "; \
 	file_paths="docker/docker-compose-quickstart/Dockerfile \
 	            .github/workflows/docker-openmetadata-db.yml \
@@ -212,7 +178,7 @@ update_github_action_paths: ## To update the github action ci docker files
 #make update_github_action_paths RELEASE_VERSION=2.2.2
 
 .PHONY: update_python_release_paths
-update_python_release_paths: ## To update the setup.py files
+update_python_release_paths:  ## To update the setup.py files
 	file_paths="ingestion/setup.py \
 				openmetadata-airflow-apis/setup.py"; \
 	echo "Updating Python setup file versions to $(PY_RELEASE_VERSION)... "; \
@@ -223,7 +189,7 @@ update_python_release_paths: ## To update the setup.py files
 #make update_python_release_paths PY_RELEASE_VERSION=2.2.2.2
 
 .PHONY: update_dockerfile_version
-update_dockerfile_version: ## To update the dockerfiles version
+update_dockerfile_version:  ## To update the dockerfiles version
 	@file_paths="docker/docker-compose-ingestion/docker-compose-ingestion.yml \
 		     docker/docker-compose-openmetadata/docker-compose-openmetadata.yml \
 		     docker/docker-compose-quickstart/docker-compose-postgres.yml \
@@ -236,7 +202,7 @@ update_dockerfile_version: ## To update the dockerfiles version
 #make update_dockerfile_version RELEASE_VERSION=2.2.2
 
 .PHONY: update_ingestion_dockerfile_version
-update_ingestion_dockerfile_version: ## To update the ingestion dockerfiles version
+update_ingestion_dockerfile_version:  ## To update the ingestion dockerfiles version
 	@file_paths="ingestion/Dockerfile \
 	             ingestion/operators/docker/Dockerfile"; \
 	echo "Updating ingestion dockerfile release version to $(PY_RELEASE_VERSION)... "; \

bootstrap/sql/migrations/native/1.2.3/postgres/schemaChanges.sql

@@ -0,0 +1,12 @@
+
+-- fixed Query for updating viewParsingTimeoutLimit
+UPDATE ingestion_pipeline_entity
+SET json = jsonb_set(
+  json::jsonb #- '{sourceConfig,config,viewParsingTimeoutLimit}',
+  '{sourceConfig,config,queryParsingTimeoutLimit}',
+  (json #> '{sourceConfig,config,viewParsingTimeoutLimit}')::jsonb,
+  true
+)
+WHERE json #>> '{pipelineType}' = 'metadata'
+AND json #>> '{sourceConfig,config,type}' = 'DatabaseMetadata'
+AND json #>> '{sourceConfig,config,viewParsingTimeoutLimit}' is not null;

bootstrap/sql/migrations/native/1.3.0/mysql/postDataMigrationSQLScript.sql

@@ -0,0 +1,8 @@
+-- Rename customMetricsProfile to customMetrics
+UPDATE profiler_data_time_series
+SET json = REPLACE(json, '"customMetricsProfile"', '"customMetrics"');
+
+-- Delete customMetricsProfile from entity_extension
+-- This was not supported on the processing side before 1.3.
+DELETE FROM openmetadata_db.entity_extension ee   
+where extension  like '%customMetrics';

bootstrap/sql/migrations/native/1.3.0/postgres/postDataMigrationSQLScript.sql

@@ -0,0 +1,8 @@
+-- Rename customMetricsProfile to customMetrics
+UPDATE profiler_data_time_series
+SET json = REPLACE(json::text, '"customMetricsProfile"', '"customMetrics"')::jsonb;
+
+-- Delete customMetricsProfile from entity_extension
+-- This was not supported on the processing side before 1.3.
+DELETE FROM entity_extension ee   
+where extension  like '%customMetrics';

conf/openmetadata.yaml

@@ -144,6 +144,8 @@ authorizerConfiguration:
 
 authenticationConfiguration:
   provider: ${AUTHENTICATION_PROVIDER:-basic}
+  # This is used by auth provider provide response as either id_token or code
+  responseType: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
   # This will only be valid when provider type specified is customOidc
   providerName: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
   publicKeyUrls: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}

docker/development/docker-compose-postgres.yml

@@ -85,6 +85,7 @@ services:
       AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
       AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
       AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
+      AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
       CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
       AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
       AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
@@ -230,6 +231,7 @@ services:
       AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
       AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
       AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
+      AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
       CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
       AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
       AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}

docker/development/docker-compose.yml

@@ -84,6 +84,7 @@ services:
       AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
       AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
       AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
+      AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
       CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
       AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
       AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
@@ -231,6 +232,7 @@ services:
       AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
       AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
       AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
+      AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
       CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
       AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
       AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}

docker/docker-compose-openmetadata/docker-compose-openmetadata.yml

@@ -34,6 +34,7 @@ services:
       AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
       AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
       AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
+      AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
       CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
       AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
       AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
@@ -175,6 +176,7 @@ services:
       AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
       AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
       AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
+      AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
       CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
       AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
       AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}

docker/docker-compose-openmetadata/env-mysql

@@ -16,6 +16,7 @@ AUTHORIZER_PRINCIPAL_DOMAIN="openmetadata.org"
 AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN="false"
 AUTHORIZER_ENABLE_SECURE_SOCKET="false"
 AUTHENTICATION_PROVIDER="basic"
+AUTHENTICATION_RESPONSE_TYPE="id_token"
 CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME=""
 AUTHENTICATION_PUBLIC_KEYS=[http://localhost:8585/api/v1/system/config/jwks]
 AUTHENTICATION_AUTHORITY="https://accounts.google.com"

docker/docker-compose-openmetadata/env-postgres

@@ -16,6 +16,7 @@ AUTHORIZER_PRINCIPAL_DOMAIN="openmetadata.org"
 AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN="false"
 AUTHORIZER_ENABLE_SECURE_SOCKET="false"
 AUTHENTICATION_PROVIDER="basic"
+AUTHENTICATION_RESPONSE_TYPE:"id_token"
 CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME=""
 AUTHENTICATION_PUBLIC_KEYS=[http://localhost:8585/api/v1/system/config/jwks]
 AUTHENTICATION_AUTHORITY="https://accounts.google.com"

docker/docker-compose-quickstart/docker-compose-postgres.yml

@@ -77,6 +77,7 @@ services:
       AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
       AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
       AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
+      AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
       CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
       AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
       AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
@@ -219,6 +220,7 @@ services:
       AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
       AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
       AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
+      AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
       CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
       AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
       AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}

docker/docker-compose-quickstart/docker-compose.yml

@@ -75,6 +75,7 @@ services:
       AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
       AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
       AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
+      AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
       CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
       AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
       AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
@@ -217,6 +218,7 @@ services:
       AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
       AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
       AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
+      AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
       CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
       AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
       AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}