From 1a47d4dc8dab460078c9bf0f4966fee9bc76d19e Mon Sep 17 00:00:00 2001 From: Harpreet Date: Sun, 19 Nov 2023 23:29:41 +0100 Subject: [PATCH 1/3] fix the sed command issue in ingestion dep script --- ingestion/ingestion_dependency.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingestion/ingestion_dependency.sh b/ingestion/ingestion_dependency.sh index 5b2b78d0d296..22cc1d8390ac 100755 --- a/ingestion/ingestion_dependency.sh +++ b/ingestion/ingestion_dependency.sh @@ -25,7 +25,7 @@ AIRFLOW_ADMIN_PASSWORD=${AIRFLOW_ADMIN_PASSWORD:-admin} OPENMETADATA_SERVER=${OPENMETADATA_SERVER:-"http://openmetadata-server:8585"} -sed -i "s#\(sql_alchemy_conn = \).*#\1${DB_CONN}#" /opt/airflow/airflow.cfg +sed -i 's,^#*[ ]*\(sql_alchemy_conn[ ]*=\).*,\1 '${DB_CONN}',g' /opt/airflow/airflow.cfg airflow db init From 359654e42c47eabad9d138b8e671e9b24e634fad Mon Sep 17 00:00:00 2001 From: Harpreet Date: Tue, 28 Nov 2023 08:58:57 +0100 Subject: [PATCH 2/3] remove sql_alchemy_conn from airflow.cfg and use env variable --- docker/development/docker-compose-postgres.yml | 2 +- docker/development/docker-compose.yml | 2 +- docker/docker-compose-ingestion/docker-compose-ingestion.yml | 2 +- docker/docker-compose-quickstart/docker-compose-postgres.yml | 2 +- docker/docker-compose-quickstart/docker-compose.yml | 2 +- ingestion/ingestion_dependency.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/development/docker-compose-postgres.yml b/docker/development/docker-compose-postgres.yml index 10c96d79a3d4..ded3943fc559 100644 --- a/docker/development/docker-compose-postgres.yml +++ b/docker/development/docker-compose-postgres.yml @@ -357,7 +357,7 @@ services: DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass} entrypoint: /bin/bash command: - - "/opt/airflow/ingestion_dependency.sh" + - "source /opt/airflow/ingestion_dependency.sh" depends_on: opensearch: condition: service_started diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml index 18216a92ba91..42840e2ba9f0 100644 --- a/docker/development/docker-compose.yml +++ b/docker/development/docker-compose.yml @@ -358,7 +358,7 @@ services: DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass} entrypoint: /bin/bash command: - - "/opt/airflow/ingestion_dependency.sh" + - "source /opt/airflow/ingestion_dependency.sh" depends_on: elasticsearch: condition: service_started diff --git a/docker/docker-compose-ingestion/docker-compose-ingestion.yml b/docker/docker-compose-ingestion/docker-compose-ingestion.yml index 74a32326bea8..f85585173b99 100644 --- a/docker/docker-compose-ingestion/docker-compose-ingestion.yml +++ b/docker/docker-compose-ingestion/docker-compose-ingestion.yml @@ -31,7 +31,7 @@ services: DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass} entrypoint: /bin/bash command: - - "/opt/airflow/ingestion_dependency.sh" + - "source /opt/airflow/ingestion_dependency.sh" expose: - 8080 ports: diff --git a/docker/docker-compose-quickstart/docker-compose-postgres.yml b/docker/docker-compose-quickstart/docker-compose-postgres.yml index 1e25d031502f..6e5c76fa6b85 100644 --- a/docker/docker-compose-quickstart/docker-compose-postgres.yml +++ b/docker/docker-compose-quickstart/docker-compose-postgres.yml @@ -368,7 +368,7 @@ services: DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass} entrypoint: /bin/bash command: - - "/opt/airflow/ingestion_dependency.sh" + - "source /opt/airflow/ingestion_dependency.sh" expose: - 8080 ports: diff --git a/docker/docker-compose-quickstart/docker-compose.yml b/docker/docker-compose-quickstart/docker-compose.yml index 68867bbdbb44..1c6ebc413062 100644 --- a/docker/docker-compose-quickstart/docker-compose.yml +++ b/docker/docker-compose-quickstart/docker-compose.yml @@ -366,7 +366,7 @@ services: DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass} entrypoint: /bin/bash command: - - "/opt/airflow/ingestion_dependency.sh" + - "source /opt/airflow/ingestion_dependency.sh" expose: - 8080 ports: diff --git a/ingestion/ingestion_dependency.sh b/ingestion/ingestion_dependency.sh index 22cc1d8390ac..56f03e097ad2 100755 --- a/ingestion/ingestion_dependency.sh +++ b/ingestion/ingestion_dependency.sh @@ -25,7 +25,7 @@ AIRFLOW_ADMIN_PASSWORD=${AIRFLOW_ADMIN_PASSWORD:-admin} OPENMETADATA_SERVER=${OPENMETADATA_SERVER:-"http://openmetadata-server:8585"} -sed -i 's,^#*[ ]*\(sql_alchemy_conn[ ]*=\).*,\1 '${DB_CONN}',g' /opt/airflow/airflow.cfg +export AIRFLOW__DATABASE__SQL_ALCHEMY_CONN="${DB_CONN}" airflow db init From 5f34558ca65885b5cc338782c8d9cc28b038ea40 Mon Sep 17 00:00:00 2001 From: Harpreet Date: Sun, 3 Dec 2023 18:54:14 +0100 Subject: [PATCH 3/3] try execute ingestion dep script without source --- docker/development/docker-compose-postgres.yml | 2 +- docker/development/docker-compose.yml | 2 +- docker/docker-compose-ingestion/docker-compose-ingestion.yml | 2 +- docker/docker-compose-quickstart/docker-compose-postgres.yml | 2 +- docker/docker-compose-quickstart/docker-compose.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/development/docker-compose-postgres.yml b/docker/development/docker-compose-postgres.yml index cb3cab7a02d2..6c227f400d72 100644 --- a/docker/development/docker-compose-postgres.yml +++ b/docker/development/docker-compose-postgres.yml @@ -359,7 +359,7 @@ services: DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass} entrypoint: /bin/bash command: - - "source /opt/airflow/ingestion_dependency.sh" + - "/opt/airflow/ingestion_dependency.sh" depends_on: opensearch: condition: service_started diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml index 60c6bae46434..5107c6ab9c8b 100644 --- a/docker/development/docker-compose.yml +++ b/docker/development/docker-compose.yml @@ -367,7 +367,7 @@ services: entrypoint: /bin/bash command: - - "source /opt/airflow/ingestion_dependency.sh" + - "/opt/airflow/ingestion_dependency.sh" depends_on: elasticsearch: condition: service_started diff --git a/docker/docker-compose-ingestion/docker-compose-ingestion.yml b/docker/docker-compose-ingestion/docker-compose-ingestion.yml index f85585173b99..74a32326bea8 100644 --- a/docker/docker-compose-ingestion/docker-compose-ingestion.yml +++ b/docker/docker-compose-ingestion/docker-compose-ingestion.yml @@ -31,7 +31,7 @@ services: DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass} entrypoint: /bin/bash command: - - "source /opt/airflow/ingestion_dependency.sh" + - "/opt/airflow/ingestion_dependency.sh" expose: - 8080 ports: diff --git a/docker/docker-compose-quickstart/docker-compose-postgres.yml b/docker/docker-compose-quickstart/docker-compose-postgres.yml index 97da1f7632f5..2ccbf93a7fa6 100644 --- a/docker/docker-compose-quickstart/docker-compose-postgres.yml +++ b/docker/docker-compose-quickstart/docker-compose-postgres.yml @@ -370,7 +370,7 @@ services: DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass} entrypoint: /bin/bash command: - - "source /opt/airflow/ingestion_dependency.sh" + - "/opt/airflow/ingestion_dependency.sh" expose: - 8080 ports: diff --git a/docker/docker-compose-quickstart/docker-compose.yml b/docker/docker-compose-quickstart/docker-compose.yml index 8fa6c340f895..75e09d5a6999 100644 --- a/docker/docker-compose-quickstart/docker-compose.yml +++ b/docker/docker-compose-quickstart/docker-compose.yml @@ -368,7 +368,7 @@ services: DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass} entrypoint: /bin/bash command: - - "source /opt/airflow/ingestion_dependency.sh" + - "/opt/airflow/ingestion_dependency.sh" expose: - 8080 ports: