Merge pull request #9 from TeoMeWhy/feat/silver

Feat/silver
TeoMeWhy · Jul 15, 2024 · 264b4a5 · 264b4a5
2 parents 2d532fd + 3d7e85b
commit 264b4a5
Show file tree

Hide file tree

Showing 6 changed files with 117 additions and 4 deletions.
diff --git a/src/silver/cliente.sql b/src/silver/cliente.sql
@@ -0,0 +1,6 @@
+SELECT
+    idCustomer AS idCliente,
+    PointsCustomer AS nrPontosCliente,
+    flEmail AS flEmailCliente
+
+FROM bronze.upsell.customers
diff --git a/src/silver/ingestao.py b/src/silver/ingestao.py
@@ -0,0 +1,15 @@
+# Databricks notebook source
+def import_query(path):
+    with open(path, "r") as open_file:
+        return open_file.read()
+
+tablename = dbutils.widgets.get("tablename")
+
+query = import_query(f"{tablename}.sql")
+
+(spark.sql(query)
+      .write
+      .format("delta")
+      .mode("overwrite")
+      .option("overwriteSchema", "true")
+      .saveAsTable(f"silver.upsell.{tablename}"))
diff --git a/src/silver/produtos.sql b/src/silver/produtos.sql
@@ -0,0 +1,3 @@
+SELECT DISTINCT NameProduct
+FROM bronze.upsell.transactions_product
+ORDER BY 1
diff --git a/src/silver/transacao_produto.sql b/src/silver/transacao_produto.sql
@@ -0,0 +1,7 @@
+SELECT
+    idTransactionCart AS idTransacaoProduto,
+    idTransaction AS idTransacao,
+    NameProduct AS descNomeProduto,
+    QuantityProduct AS nrQuantidadeProduto
+
+FROM bronze.upsell.transactions_product
diff --git a/src/silver/transacoes.sql b/src/silver/transacoes.sql
@@ -0,0 +1,6 @@
+SELECT idTransaction AS idTransacao,
+       idCustomer AS idCliente,
+       dtTransaction AS dtTransacao,
+       pointsTransaction AS nrPontosTransacao
+
+FROM bronze.upsell.transactions
diff --git a/src/workflows/upsell.json b/src/workflows/upsell.json
@@ -11,7 +11,7 @@
             "quartz_cron_expression": "00 00 7 * * ?",
             "timezone_id": "America/Sao_Paulo",
             "pause_status": "UNPAUSED"
-          },
+        },
         "max_concurrent_runs": 1,
         "tasks": [
             {
@@ -26,7 +26,7 @@
                     },
                     "source": "GIT"
                 },
-                "existing_cluster_id": "0809-155233-nc569ju7",
+                "existing_cluster_id": "0704-112023-u2kusxyv",
                 "timeout_seconds": 0,
                 "email_notifications": {},
                 "notification_settings": {
@@ -48,7 +48,7 @@
                     },
                     "source": "GIT"
                 },
-                "existing_cluster_id": "0809-155233-nc569ju7",
+                "existing_cluster_id": "0704-112023-u2kusxyv",
                 "timeout_seconds": 0,
                 "email_notifications": {},
                 "notification_settings": {
@@ -70,7 +70,7 @@
                     },
                     "source": "GIT"
                 },
-                "existing_cluster_id": "0809-155233-nc569ju7",
+                "existing_cluster_id": "0704-112023-u2kusxyv",
                 "timeout_seconds": 0,
                 "email_notifications": {},
                 "notification_settings": {
@@ -79,6 +79,82 @@
                     "alert_on_last_attempt": false
                 },
                 "webhook_notifications": {}
+            },
+            {
+                "task_key": "silver_cliente",
+                "depends_on": [
+                    {
+                        "task_key": "bronze_customers"
+                    }
+                ],
+                "run_if": "ALL_SUCCESS",
+                "notebook_task": {
+                    "notebook_path": "src/silver/ingestao",
+                    "base_parameters": {
+                        "tablename": "cliente"
+                    },
+                    "source": "GIT"
+                },
+                "existing_cluster_id": "0704-112023-u2kusxyv",
+                "timeout_seconds": 0,
+                "email_notifications": {}
+            },
+            {
+                "task_key": "silver_produtos",
+                "depends_on": [
+                    {
+                        "task_key": "bronze_transactions_product"
+                    }
+                ],
+                "run_if": "ALL_SUCCESS",
+                "notebook_task": {
+                    "notebook_path": "src/silver/ingestao",
+                    "base_parameters": {
+                        "tablename": "produtos"
+                    },
+                    "source": "GIT"
+                },
+                "existing_cluster_id": "0704-112023-u2kusxyv",
+                "timeout_seconds": 0,
+                "email_notifications": {}
+            },
+            {
+                "task_key": "silver_transacao_produto",
+                "depends_on": [
+                    {
+                        "task_key": "bronze_transactions_product"
+                    }
+                ],
+                "run_if": "ALL_SUCCESS",
+                "notebook_task": {
+                    "notebook_path": "src/silver/ingestao",
+                    "base_parameters": {
+                        "tablename": "transacao_produto"
+                    },
+                    "source": "GIT"
+                },
+                "existing_cluster_id": "0704-112023-u2kusxyv",
+                "timeout_seconds": 0,
+                "email_notifications": {}
+            },
+            {
+                "task_key": "silver_transacoes",
+                "depends_on": [
+                    {
+                        "task_key": "bronze_transactions"
+                    }
+                ],
+                "run_if": "ALL_SUCCESS",
+                "notebook_task": {
+                    "notebook_path": "src/silver/ingestao",
+                    "base_parameters": {
+                        "tablename": "transacoes"
+                    },
+                    "source": "GIT"
+                },
+                "existing_cluster_id": "0704-112023-u2kusxyv",
+                "timeout_seconds": 0,
+                "email_notifications": {}
             }
         ],
         "git_source": {