From 9e865b63feb5f1d1e1de88f3f09dd6abe73dc787 Mon Sep 17 00:00:00 2001 From: TeoCalvo Date: Mon, 15 Jul 2024 13:11:44 +0000 Subject: [PATCH 1/2] =?UTF-8?q?Feat:=20Ingest=C3=A3o=20em=20silver?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - cliente - produtos - transacoes - transacao_produto --- src/silver/cliente.sql | 6 ++++++ src/silver/ingestao.py | 15 +++++++++++++++ src/silver/produtos.sql | 3 +++ src/silver/transacao_produto.sql | 7 +++++++ src/silver/transacoes.sql | 6 ++++++ 5 files changed, 37 insertions(+) create mode 100644 src/silver/cliente.sql create mode 100644 src/silver/ingestao.py create mode 100644 src/silver/produtos.sql create mode 100644 src/silver/transacao_produto.sql create mode 100644 src/silver/transacoes.sql diff --git a/src/silver/cliente.sql b/src/silver/cliente.sql new file mode 100644 index 0000000..7082809 --- /dev/null +++ b/src/silver/cliente.sql @@ -0,0 +1,6 @@ +SELECT + idCustomer AS idCliente, + PointsCustomer AS nrPontosCliente, + flEmail AS flEmailCliente + +FROM bronze.upsell.customers \ No newline at end of file diff --git a/src/silver/ingestao.py b/src/silver/ingestao.py new file mode 100644 index 0000000..a61bbe3 --- /dev/null +++ b/src/silver/ingestao.py @@ -0,0 +1,15 @@ +# Databricks notebook source +def import_query(path): + with open(path, "r") as open_file: + return open_file.read() + +tablename = dbutils.widgets.get("tablename") + +query = import_query(f"{tablename}.sql") + +(spark.sql(query) + .write + .format("delta") + .mode("overwrite") + .option("overwriteSchema", "true") + .saveAsTable(f"silver.upsell.{tablename}")) diff --git a/src/silver/produtos.sql b/src/silver/produtos.sql new file mode 100644 index 0000000..793f4eb --- /dev/null +++ b/src/silver/produtos.sql @@ -0,0 +1,3 @@ +SELECT DISTINCT NameProduct +FROM bronze.upsell.transactions_product +ORDER BY 1 \ No newline at end of file diff --git a/src/silver/transacao_produto.sql b/src/silver/transacao_produto.sql new file mode 100644 index 0000000..664cdf1 --- /dev/null +++ b/src/silver/transacao_produto.sql @@ -0,0 +1,7 @@ +SELECT + idTransactionCart AS idTransacaoProduto, + idTransaction AS idTransacao, + NameProduct AS descNomeProduto, + QuantityProduct AS nrQuantidadeProduto + +FROM bronze.upsell.transactions_product \ No newline at end of file diff --git a/src/silver/transacoes.sql b/src/silver/transacoes.sql new file mode 100644 index 0000000..c77f5f1 --- /dev/null +++ b/src/silver/transacoes.sql @@ -0,0 +1,6 @@ +SELECT idTransaction AS idTransacao, + idCustomer AS idCliente, + dtTransaction AS dtTransacao, + pointsTransaction AS nrPontosTransacao + +FROM bronze.upsell.transactions \ No newline at end of file From 3d7e85b03293a2e57ce75930269641829e3d95a1 Mon Sep 17 00:00:00 2001 From: TeoCalvo Date: Mon, 15 Jul 2024 13:18:41 +0000 Subject: [PATCH 2/2] Feat: workflow atualizado com silver --- src/workflows/upsell.json | 84 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 4 deletions(-) diff --git a/src/workflows/upsell.json b/src/workflows/upsell.json index 819216f..40157f7 100644 --- a/src/workflows/upsell.json +++ b/src/workflows/upsell.json @@ -11,7 +11,7 @@ "quartz_cron_expression": "00 00 7 * * ?", "timezone_id": "America/Sao_Paulo", "pause_status": "UNPAUSED" - }, + }, "max_concurrent_runs": 1, "tasks": [ { @@ -26,7 +26,7 @@ }, "source": "GIT" }, - "existing_cluster_id": "0809-155233-nc569ju7", + "existing_cluster_id": "0704-112023-u2kusxyv", "timeout_seconds": 0, "email_notifications": {}, "notification_settings": { @@ -48,7 +48,7 @@ }, "source": "GIT" }, - "existing_cluster_id": "0809-155233-nc569ju7", + "existing_cluster_id": "0704-112023-u2kusxyv", "timeout_seconds": 0, "email_notifications": {}, "notification_settings": { @@ -70,7 +70,7 @@ }, "source": "GIT" }, - "existing_cluster_id": "0809-155233-nc569ju7", + "existing_cluster_id": "0704-112023-u2kusxyv", "timeout_seconds": 0, "email_notifications": {}, "notification_settings": { @@ -79,6 +79,82 @@ "alert_on_last_attempt": false }, "webhook_notifications": {} + }, + { + "task_key": "silver_cliente", + "depends_on": [ + { + "task_key": "bronze_customers" + } + ], + "run_if": "ALL_SUCCESS", + "notebook_task": { + "notebook_path": "src/silver/ingestao", + "base_parameters": { + "tablename": "cliente" + }, + "source": "GIT" + }, + "existing_cluster_id": "0704-112023-u2kusxyv", + "timeout_seconds": 0, + "email_notifications": {} + }, + { + "task_key": "silver_produtos", + "depends_on": [ + { + "task_key": "bronze_transactions_product" + } + ], + "run_if": "ALL_SUCCESS", + "notebook_task": { + "notebook_path": "src/silver/ingestao", + "base_parameters": { + "tablename": "produtos" + }, + "source": "GIT" + }, + "existing_cluster_id": "0704-112023-u2kusxyv", + "timeout_seconds": 0, + "email_notifications": {} + }, + { + "task_key": "silver_transacao_produto", + "depends_on": [ + { + "task_key": "bronze_transactions_product" + } + ], + "run_if": "ALL_SUCCESS", + "notebook_task": { + "notebook_path": "src/silver/ingestao", + "base_parameters": { + "tablename": "transacao_produto" + }, + "source": "GIT" + }, + "existing_cluster_id": "0704-112023-u2kusxyv", + "timeout_seconds": 0, + "email_notifications": {} + }, + { + "task_key": "silver_transacoes", + "depends_on": [ + { + "task_key": "bronze_transactions" + } + ], + "run_if": "ALL_SUCCESS", + "notebook_task": { + "notebook_path": "src/silver/ingestao", + "base_parameters": { + "tablename": "transacoes" + }, + "source": "GIT" + }, + "existing_cluster_id": "0704-112023-u2kusxyv", + "timeout_seconds": 0, + "email_notifications": {} } ], "git_source": {