Updates SQLMesh to the latest version (#2679)

* Upgrade sqlmesh to the latest * more fixes More refactoring fix tests fix tested fixes * ensure trino install * lock fix
opensource-observer · Dec 27, 2024 · 10b5e92 · 10b5e92
1 parent ba66152
commit 10b5e92
Show file tree

Hide file tree

Showing 18 changed files with 721 additions and 809 deletions.
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -59,7 +59,7 @@ sqlalchemy = "^2.0.25"
 textual = "^0.52.1"
 redis = "^5.0.7"
 githubkit = "^0.11.6"
-sqlmesh = { extras = ["trino"], version = "^0.129.0" }
+sqlmesh = { extras = ["trino"], version = "^0.141.1" }
 dagster-duckdb = "^0.24.0"
 dagster-duckdb-polars = "^0.24.0"
 google-cloud-bigquery-storage = "^2.25.0"

diff --git a/warehouse/metrics_mesh/config.py b/warehouse/metrics_mesh/config.py
@@ -19,7 +19,8 @@
     gateways={
         "local": GatewayConfig(
             connection=DuckDBConnectionConfig(
-                database=os.environ.get("SQLMESH_DUCKDB_LOCAL_PATH")
+                concurrent_tasks=1,
+                database=os.environ.get("SQLMESH_DUCKDB_LOCAL_PATH"),
             ),
             variables={
                 "oso_source_db": "sources",

diff --git a/warehouse/metrics_mesh/models/events_daily_to_artifact.sql b/warehouse/metrics_mesh/models/events_daily_to_artifact.sql
@@ -36,4 +36,4 @@ GROUP BY
   to_artifact_id,
   event_source,
   event_type,
-  DATE_TRUNC('DAY', time::DATE)
+  DATE_TRUNC('DAY', time::DATE)
diff --git a/warehouse/metrics_mesh/tests/test_events_daily_to_artifact.yml b/warehouse/metrics_mesh/tests/test_events_daily_to_artifact.yml
@@ -11,43 +11,43 @@ test_events_daily_to_artifact:
         from_artifact_id: user_0
         event_source: BLOCKCHAIN
         event_type: CONTRACT_INVOCATION_SUCCESS_DAILY_COUNT
-        time: 2024-01-01T00:00:00Z
+        time: "2024-01-01 00:00:00+00:00"
         amount: 10
       - to_artifact_id: contract_1
         from_artifact_id: user_0
         event_source: BLOCKCHAIN
         event_type: CONTRACT_INVOCATION_SUCCESS_DAILY_COUNT
-        time: 2024-01-01T00:00:00Z
+        time: "2024-01-01 00:00:00+00:00"
         amount: 10
       - to_artifact_id: repo_0
         from_artifact_id: dev_0
         event_source: SOURCE_PROVIDER
         event_type: COMMIT_CODE
-        time: 2024-01-01T02:00:00Z
+        time: "2024-01-01 02:00:00+00:00"
         amount: 1
       - to_artifact_id: repo_0
         from_artifact_id: dev_0
         event_source: SOURCE_PROVIDER
         event_type: COMMIT_CODE
-        time: 2024-01-01T03:00:00Z
+        time: "2024-01-01 03:00:00+00:00"
         amount: 1
       - to_artifact_id: repo_0
         from_artifact_id: dev_0
         event_source: SOURCE_PROVIDER
         event_type: COMMIT_CODE
-        time: 2024-01-01T04:00:00Z
+        time: "2024-01-01 04:00:00+00:00"
         amount: 1
       - to_artifact_id: repo_0
         from_artifact_id: dev_1
         event_source: SOURCE_PROVIDER
         event_type: COMMIT_CODE
-        time: 2024-01-01T04:00:00Z
+        time: "2024-01-01 04:00:00+00:00"
         amount: 1
       - to_artifact_id: repo_0
         from_artifact_id: dev_1
         event_source: SOURCE_PROVIDER
         event_type: COMMIT_CODE
-        time: 2024-01-02T04:00:00Z
+        time: "2024-01-02 04:00:00+00:00"
         amount: 1
   outputs:
     query:

diff --git a/warehouse/metrics_tools/factory/__init__.py b/warehouse/metrics_tools/factory/__init__.py
@@ -1,4 +1,4 @@
 # ruff: noqa: F403
 
-from .factory import *
 from ..definition import *
+from .factory import *
diff --git a/warehouse/metrics_tools/factory/constants.py b/warehouse/metrics_tools/factory/constants.py
@@ -0,0 +1,35 @@
+import typing as t
+
+from sqlglot import exp
+
+TIME_AGGREGATION_TO_CRON = {
+    "daily": "@daily",
+    "monthly": "@monthly",
+    "weekly": "@weekly",
+}
+METRICS_COLUMNS_BY_ENTITY: t.Dict[str, t.Dict[str, exp.DataType]] = {
+    "artifact": {
+        "metrics_sample_date": exp.DataType.build("DATE", dialect="duckdb"),
+        "event_source": exp.DataType.build("STRING", dialect="duckdb"),
+        "to_artifact_id": exp.DataType.build("STRING", dialect="duckdb"),
+        "from_artifact_id": exp.DataType.build("STRING", dialect="duckdb"),
+        "metric": exp.DataType.build("STRING", dialect="duckdb"),
+        "amount": exp.DataType.build("DOUBLE", dialect="duckdb"),
+    },
+    "project": {
+        "metrics_sample_date": exp.DataType.build("DATE", dialect="duckdb"),
+        "event_source": exp.DataType.build("STRING", dialect="duckdb"),
+        "to_project_id": exp.DataType.build("STRING", dialect="duckdb"),
+        "from_artifact_id": exp.DataType.build("STRING", dialect="duckdb"),
+        "metric": exp.DataType.build("STRING", dialect="duckdb"),
+        "amount": exp.DataType.build("DOUBLE", dialect="duckdb"),
+    },
+    "collection": {
+        "metrics_sample_date": exp.DataType.build("DATE", dialect="duckdb"),
+        "event_source": exp.DataType.build("STRING", dialect="duckdb"),
+        "to_collection_id": exp.DataType.build("STRING", dialect="duckdb"),
+        "from_artifact_id": exp.DataType.build("STRING", dialect="duckdb"),
+        "metric": exp.DataType.build("STRING", dialect="duckdb"),
+        "amount": exp.DataType.build("DOUBLE", dialect="duckdb"),
+    },
+}