From c48ea494d32874ecf0658ca6a2c4605d77b30f56 Mon Sep 17 00:00:00 2001 From: Paul Rogalinski-Pinter Date: Thu, 25 Apr 2024 16:47:35 +0200 Subject: [PATCH 01/14] fix(metadata-ingestion): ignore cards without id (text, heading) fix #103080 --- .../src/datahub/ingestion/source/metabase.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/metabase.py b/metadata-ingestion/src/datahub/ingestion/source/metabase.py index 6f8f5097b61497..e8278a2ca19146 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/metabase.py +++ b/metadata-ingestion/src/datahub/ingestion/source/metabase.py @@ -309,9 +309,10 @@ def construct_dashboard_from_api_data( chart_urns = [] cards_data = dashboard_details.get("dashcards", {}) for card_info in cards_data: - chart_urn = builder.make_chart_urn( - self.platform, card_info.get("card").get("id", "") - ) + card_id = card_info.get("card").get("id", "") + if not card_id: + continue # most likely a virtual card without an id (text or heading), not relevant. + chart_urn = builder.make_chart_urn(self.platform, card_id) chart_urns.append(chart_urn) dashboard_info_class = DashboardInfoClass( From 1c3afa4e2b184a04f244c9774749ee47a9a83957 Mon Sep 17 00:00:00 2001 From: Paul Rogalinski-Pinter Date: Thu, 25 Apr 2024 17:07:49 +0200 Subject: [PATCH 02/14] fix(metadata-ingestion): remove/replace filter expressions in native query fixes #9767 --- .../src/datahub/ingestion/source/metabase.py | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/metabase.py b/metadata-ingestion/src/datahub/ingestion/source/metabase.py index e8278a2ca19146..ffa62ea0b2b086 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/metabase.py +++ b/metadata-ingestion/src/datahub/ingestion/source/metabase.py @@ -1,6 +1,6 @@ import json import logging -from dataclasses import dataclass +import re from datetime import datetime, timezone from functools import lru_cache from typing import Dict, Iterable, List, Optional, Tuple, Union @@ -94,7 +94,6 @@ class MetabaseConfig(DatasetLineageProviderConfigBase, StatefulIngestionConfigBa default=False, description="Flag that if true, exclude other user collections", ) - stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = None @validator("connect_uri", "display_uri") def remove_trailing_slash(cls, v): @@ -593,11 +592,12 @@ def get_datasource_urn( ) ] else: - raw_query = ( + raw_query_stripped = self.strip_template_expressions( card_details.get("dataset_query", {}).get("native", {}).get("query", "") ) + result = create_lineage_sql_parsed_result( - query=raw_query, + query=raw_query_stripped, default_db=database_name, default_schema=database_schema or self.config.default_schema, platform=platform, @@ -618,6 +618,24 @@ def get_datasource_urn( return None + @staticmethod + def strip_template_expressions(raw_query) -> str: + """ + Workarounds for metabase raw queries containing most commonly used template expressions: + + - strip conditional expressions "[[ .... ]]" + - replace all {{ filter expressions }} with "1" + + reference: https://www.metabase.com/docs/latest/questions/native-editor/sql-parameters + """ + + # drop [[ WHERE {{FILTER}} ]] + query_patched = re.sub(r"\[\[.+\]\]", r" ", raw_query) + + # replace {{FILTER}} with 1 + query_patched = re.sub(r"\{\{.+\}\}", r"1", query_patched) + return query_patched + @lru_cache(maxsize=None) def get_source_table_from_id( self, table_id: Union[int, str] From cc43525525660b421ad965dceca492b317cde64b Mon Sep 17 00:00:00 2001 From: Paul Rogalinski-Pinter Date: Fri, 17 May 2024 08:18:19 +0200 Subject: [PATCH 03/14] Update metadata-ingestion/src/datahub/ingestion/source/metabase.py Co-authored-by: Harshal Sheth --- metadata-ingestion/src/datahub/ingestion/source/metabase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/metabase.py b/metadata-ingestion/src/datahub/ingestion/source/metabase.py index ffa62ea0b2b086..cefdc9356c7074 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/metabase.py +++ b/metadata-ingestion/src/datahub/ingestion/source/metabase.py @@ -619,7 +619,7 @@ def get_datasource_urn( return None @staticmethod - def strip_template_expressions(raw_query) -> str: + def strip_template_expressions(raw_query: str) -> str: """ Workarounds for metabase raw queries containing most commonly used template expressions: From 0fe1b9eb17bb6bc771da31886d37c1c9afca6970 Mon Sep 17 00:00:00 2001 From: Paul Rogalinski-Pinter Date: Fri, 17 May 2024 08:31:52 +0200 Subject: [PATCH 04/14] PR review: fix rebase / merge mistake --- metadata-ingestion/src/datahub/ingestion/source/metabase.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metadata-ingestion/src/datahub/ingestion/source/metabase.py b/metadata-ingestion/src/datahub/ingestion/source/metabase.py index cefdc9356c7074..cb81529b100890 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/metabase.py +++ b/metadata-ingestion/src/datahub/ingestion/source/metabase.py @@ -94,6 +94,7 @@ class MetabaseConfig(DatasetLineageProviderConfigBase, StatefulIngestionConfigBa default=False, description="Flag that if true, exclude other user collections", ) + stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = None @validator("connect_uri", "display_uri") def remove_trailing_slash(cls, v): From 5d9f399727d26a4ba875fbd5bf2083f200460880 Mon Sep 17 00:00:00 2001 From: Paul Rogalinski-Pinter Date: Fri, 17 May 2024 09:07:14 +0200 Subject: [PATCH 05/14] fix issues reported by linter --- metadata-ingestion/src/datahub/ingestion/source/metabase.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/metabase.py b/metadata-ingestion/src/datahub/ingestion/source/metabase.py index cb81529b100890..cad1b06963f10b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/metabase.py +++ b/metadata-ingestion/src/datahub/ingestion/source/metabase.py @@ -1,6 +1,7 @@ import json import logging import re +from dataclasses import dataclass from datetime import datetime, timezone from functools import lru_cache from typing import Dict, Iterable, List, Optional, Tuple, Union @@ -608,12 +609,12 @@ def get_datasource_urn( ) if result.debug_info.table_error: logger.info( - f"Failed to parse lineage from query {raw_query}: " + f"Failed to parse lineage from query {raw_query_stripped}: " f"{result.debug_info.table_error}" ) self.report.report_warning( key="metabase-query", - reason=f"Unable to retrieve lineage from query: {raw_query}", + reason=f"Unable to retrieve lineage from query: {raw_query_stripped}", ) return result.in_tables From 009d541b501f17119603dd7b099c533f2c41b935 Mon Sep 17 00:00:00 2001 From: Paul Rogalinski-Pinter Date: Sat, 18 May 2024 01:57:42 +0200 Subject: [PATCH 06/14] add test for #9767 --- .../metabase/setup/issue_9767/card_3.json | 159 +++++++++++++ .../metabase/setup/issue_9767/collection.json | 12 + .../issue_9767/collection_dashboard_root.json | 25 ++ .../setup/issue_9767/dashboard_3.json | 215 ++++++++++++++++++ .../metabase/setup/issue_9767/database_1.json | 78 +++++++ .../metabase/setup/issue_9767/table_6.json | 73 ++++++ .../integration/metabase/test_metabase.py | 41 ++++ 7 files changed, 603 insertions(+) create mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/card_3.json create mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection.json create mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection_dashboard_root.json create mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/dashboard_3.json create mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/database_1.json create mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/table_6.json diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/card_3.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/card_3.json new file mode 100644 index 00000000000000..21308d73d8f84a --- /dev/null +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/card_3.json @@ -0,0 +1,159 @@ +{ + "description": null, + "archived": false, + "collection_position": null, + "table_id": 6, + "result_metadata": [ + { + "display_name": "SOURCE", + "field_ref": [ + "field", + "SOURCE", + { + "base-type": "type/Text" + } + ], + "name": "SOURCE", + "base_type": "type/Text", + "effective_type": "type/Text", + "semantic_type": "type/Source", + "fingerprint": { + "global": { + "distinct-count": 5, + "nil%": 0.2 + }, + "type": { + "type/Text": { + "percent-json": 0.0, + "percent-url": 0.0, + "percent-email": 0.0, + "percent-state": 0.0, + "average-length": 5.4 + } + } + } + }, + { + "display_name": "sum", + "field_ref": [ + "field", + "sum", + { + "base-type": "type/BigInteger" + } + ], + "name": "sum", + "base_type": "type/BigInteger", + "effective_type": "type/BigInteger", + "semantic_type": null, + "fingerprint": { + "global": { + "distinct-count": 5, + "nil%": 0.0 + }, + "type": { + "type/Number": { + "min": 5634.0, + "q1": 5650.5, + "q3": 8961.75, + "max": 17178.0, + "sd": 5084.457080161067, + "avg": 8092.6 + } + } + } + } + ], + "creator": { + "email": "pulsar@snowtronix.local", + "first_name": "Johny", + "last_login": "2024-05-17T21:31:25.411414Z", + "is_qbnewb": false, + "is_superuser": true, + "id": 1, + "last_name": "Appleseed", + "date_joined": "2024-05-17T21:31:25.323423Z", + "common_name": "Johny Appleseed" + }, + "initially_published_at": null, + "can_write": true, + "database_id": 1, + "enable_embedding": false, + "collection_id": null, + "query_type": "native", + "name": "Accounts, Sum of Seats, Grouped by Source - Duplicate - Duplicate", + "last_query_start": "2024-05-17T21:51:20.209029Z", + "dashboard_count": 1, + "type": "question", + "average_query_time": 23.76923076923077, + "creator_id": 1, + "moderation_reviews": [], + "updated_at": "2024-05-17T21:51:20.217914Z", + "made_public_by_id": null, + "embedding_params": null, + "cache_ttl": null, + "dataset_query": { + "database": 1, + "type": "native", + "native": { + "template-tags": { + "COUNTRY": { + "type": "text", + "name": "COUNTRY", + "id": "74499e6a-146d-4107-9508-9801deeeae5c", + "display-name": "Country" + } + }, + "collection": "ACCOUNTS", + "query": "SELECT\n \"PUBLIC\".\"ACCOUNTS\".\"SOURCE\" AS \"SOURCE\",\n SUM(\"PUBLIC\".\"ACCOUNTS\".\"SEATS\") AS \"sum\"\nFROM\n \"PUBLIC\".\"ACCOUNTS\"\nGROUP BY\n \"PUBLIC\".\"ACCOUNTS\".\"SOURCE\"\nORDER BY\n \"PUBLIC\".\"ACCOUNTS\".\"SOURCE\" ASC\n[[where PUBLIC.ACCOUNTS.COUNTRY == {{COUNTRY}}]]" + } + }, + "id": 3, + "parameter_mappings": [], + "display": "bar", + "entity_id": "uns9F4BFNsCq8YFMbrCdO", + "collection_preview": true, + "last-edit-info": { + "id": 1, + "email": "pulsar@snowtronix.local", + "first_name": "Johny", + "last_name": "Appleseed", + "timestamp": "2024-05-17T21:51:07.409582Z" + }, + "visualization_settings": { + "graph.dimensions": [ + "SOURCE" + ], + "graph.metrics": [ + "sum" + ] + }, + "collection": { + "metabase.models.collection.root/is-root?": true, + "authority_level": null, + "name": "Our analytics", + "is_personal": false, + "id": "root", + "can_write": true + }, + "metabase_version": "v0.49.11 (b894f2d)", + "parameters": [ + { + "id": "74499e6a-146d-4107-9508-9801deeeae5c", + "type": "category", + "target": [ + "variable", + [ + "template-tag", + "COUNTRY" + ] + ], + "name": "Country", + "slug": "COUNTRY" + } + ], + "dataset": false, + "created_at": "2024-05-17T21:33:56.551981Z", + "parameter_usage_count": 0, + "public_uuid": null +} diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection.json new file mode 100644 index 00000000000000..13ae56d9bf9196 --- /dev/null +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection.json @@ -0,0 +1,12 @@ +[ + { + "authority_level": null, + "can_write": true, + "name": "Our analytics", + "effective_ancestors": [], + "effective_location": null, + "parent_id": null, + "id": "root", + "is_personal": false + } +] diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection_dashboard_root.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection_dashboard_root.json new file mode 100644 index 00000000000000..c711f6d25d532b --- /dev/null +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection_dashboard_root.json @@ -0,0 +1,25 @@ +{ + "total": 3, + "data": [ + { + "description": "test", + "collection_position": null, + "database_id": null, + "name": "test filter", + "id": 3, + "entity_id": "cNXZhyVHAOEONhw5hByKX", + "last-edit-info": { + "id": 1, + "last_name": "Appleseed", + "first_name": "Johny", + "email": "pulsar@snowtronix.local", + "timestamp": "2024-05-17T21:51:20.080515Z" + }, + "model": "dashboard" + }], + "models": [ + "dashboard" + ], + "limit": null, + "offset": null +} diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/dashboard_3.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/dashboard_3.json new file mode 100644 index 00000000000000..b3c4206ce630cc --- /dev/null +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/dashboard_3.json @@ -0,0 +1,215 @@ +{ + "description": "test", + "archived": false, + "collection_position": null, + "dashcards": [ + { + "size_x": 12, + "dashboard_tab_id": null, + "series": [], + "action_id": null, + "collection_authority_level": null, + "card": { + "description": null, + "archived": false, + "collection_position": null, + "table_id": 6, + "result_metadata": [ + { + "display_name": "SOURCE", + "field_ref": [ + "field", + "SOURCE", + { + "base-type": "type/Text" + } + ], + "name": "SOURCE", + "base_type": "type/Text", + "effective_type": "type/Text", + "semantic_type": "type/Source", + "fingerprint": { + "global": { + "distinct-count": 5, + "nil%": 0.2 + }, + "type": { + "type/Text": { + "percent-json": 0.0, + "percent-url": 0.0, + "percent-email": 0.0, + "percent-state": 0.0, + "average-length": 5.4 + } + } + } + }, + { + "display_name": "sum", + "field_ref": [ + "field", + "sum", + { + "base-type": "type/BigInteger" + } + ], + "name": "sum", + "base_type": "type/BigInteger", + "effective_type": "type/BigInteger", + "semantic_type": null, + "fingerprint": { + "global": { + "distinct-count": 5, + "nil%": 0.0 + }, + "type": { + "type/Number": { + "min": 5634.0, + "q1": 5650.5, + "q3": 8961.75, + "max": 17178.0, + "sd": 5084.457080161067, + "avg": 8092.6 + } + } + } + } + ], + "initially_published_at": null, + "can_write": true, + "database_id": 1, + "enable_embedding": false, + "collection_id": null, + "query_type": "native", + "name": "Accounts, Sum of Seats, Grouped by Source - Duplicate - Duplicate", + "type": "question", + "query_average_duration": null, + "creator_id": 1, + "moderation_reviews": [], + "updated_at": "2024-05-17T21:51:20.217914Z", + "made_public_by_id": null, + "embedding_params": null, + "cache_ttl": null, + "dataset_query": { + "database": 1, + "type": "native", + "native": { + "template-tags": { + "COUNTRY": { + "type": "text", + "name": "COUNTRY", + "id": "74499e6a-146d-4107-9508-9801deeeae5c", + "display-name": "Country" + } + }, + "collection": "ACCOUNTS", + "query": "SELECT\n \"PUBLIC\".\"ACCOUNTS\".\"SOURCE\" AS \"SOURCE\",\n SUM(\"PUBLIC\".\"ACCOUNTS\".\"SEATS\") AS \"sum\"\nFROM\n \"PUBLIC\".\"ACCOUNTS\"\nGROUP BY\n \"PUBLIC\".\"ACCOUNTS\".\"SOURCE\"\nORDER BY\n \"PUBLIC\".\"ACCOUNTS\".\"SOURCE\" ASC\n[[where PUBLIC.ACCOUNTS.COUNTRY == {{COUNTRY}}]]" + } + }, + "id": 3, + "parameter_mappings": [], + "display": "bar", + "entity_id": "uns9F4BFNsCq8YFMbrCdO", + "collection_preview": true, + "visualization_settings": { + "graph.dimensions": [ + "SOURCE" + ], + "graph.metrics": [ + "sum" + ] + }, + "metabase_version": "v0.49.11 (b894f2d)", + "parameters": [ + { + "id": "74499e6a-146d-4107-9508-9801deeeae5c", + "type": "category", + "target": [ + "variable", + [ + "template-tag", + "COUNTRY" + ] + ], + "name": "Country", + "slug": "COUNTRY" + } + ], + "dataset": false, + "created_at": "2024-05-17T21:33:56.551981Z", + "public_uuid": null + }, + "updated_at": "2024-05-17T21:51:20.06007Z", + "col": 0, + "id": 3, + "parameter_mappings": [ + { + "parameter_id": "5d1aeef8", + "card_id": 3, + "target": [ + "variable", + [ + "template-tag", + "COUNTRY" + ] + ] + } + ], + "card_id": 3, + "entity_id": "GT9BVzh-D95Kc7-slYhj1", + "visualization_settings": {}, + "size_y": 6, + "dashboard_id": 3, + "created_at": "2024-05-17T21:33:56.551981Z", + "row": 0 + } + ], + "param_values": null, + "initially_published_at": null, + "can_write": true, + "tabs": [], + "enable_embedding": false, + "collection_id": null, + "show_in_getting_started": false, + "name": "test filter", + "width": "fixed", + "caveats": null, + "collection_authority_level": null, + "creator_id": 1, + "updated_at": "2024-05-17T21:51:20.06007Z", + "made_public_by_id": null, + "embedding_params": null, + "cache_ttl": null, + "id": 3, + "position": null, + "entity_id": "cNXZhyVHAOEONhw5hByKX", + "param_fields": null, + "last-edit-info": { + "id": 1, + "email": "pulsar@snowtronix.local", + "first_name": "Johny", + "last_name": "Appleseed", + "timestamp": "2024-05-17T21:51:20.080515Z" + }, + "collection": { + "metabase.models.collection.root/is-root?": true, + "authority_level": null, + "name": "Our analytics", + "is_personal": false, + "id": "root", + "can_write": true + }, + "parameters": [ + { + "name": "Location", + "slug": "location", + "id": "5d1aeef8", + "type": "string/=", + "sectionId": "location" + } + ], + "auto_apply_filters": true, + "created_at": "2024-05-17T21:33:56.551981Z", + "public_uuid": null, + "points_of_interest": null +} diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/database_1.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/database_1.json new file mode 100644 index 00000000000000..851ec8aadcccc0 --- /dev/null +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/database_1.json @@ -0,0 +1,78 @@ +{ + "description": "Some example data for you to play around with.", + "features": [ + "index-info", + "basic-aggregations", + "temporal-extract", + "actions/custom", + "now", + "standard-deviation-aggregations", + "date-arithmetics", + "actions", + "expression-aggregations", + "foreign-keys", + "right-join", + "left-join", + "native-parameters", + "schemas", + "nested-queries", + "expressions", + "uploads", + "regex", + "case-sensitivity-string-filter-options", + "binning", + "datetime-diff", + "upload-with-auto-pk", + "inner-join", + "advanced-math-expressions" + ], + "cache_field_values_schedule": "0 0 17 * * ? *", + "timezone": "GMT", + "auto_run_queries": true, + "metadata_sync_schedule": "0 15 * * * ? *", + "name": "Sample Database", + "settings": null, + "caveats": "You probably don't want to use this for your business-critical analyses, but hey, it's your world, we're just living in it.", + "can-manage": true, + "creator_id": null, + "is_full_sync": true, + "updated_at": "2024-05-17T21:30:27.555139Z", + "cache_ttl": null, + "details": { + "db": "file:/plugins/sample-database.db;USER=GUEST;PASSWORD=guest" + }, + "is_sample": true, + "id": 1, + "is_on_demand": false, + "schedules": { + "metadata_sync": { + "schedule_minute": 15, + "schedule_day": null, + "schedule_frame": null, + "schedule_hour": null, + "schedule_type": "hourly" + }, + "cache_field_values": { + "schedule_minute": 0, + "schedule_day": null, + "schedule_frame": null, + "schedule_hour": 17, + "schedule_type": "daily" + } + }, + "engine": "postgres", + "initial_sync_status": "complete", + "is_audit": false, + "dbms_version": { + "flavor": "H2", + "version": "2.1.214 (2022-06-13)", + "semantic-version": [ + 2, + 1 + ] + }, + "refingerprint": null, + "created_at": "2024-05-17T21:30:26.74786Z", + "points_of_interest": "You can find all sorts of different joinable tables ranging from products to people to reviews here.", + "can_upload": false +} diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/table_6.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/table_6.json new file mode 100644 index 00000000000000..aa2501fde6aebf --- /dev/null +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/table_6.json @@ -0,0 +1,73 @@ +{ + "description": "Information on customer accounts registered with Piespace. Each account represents a new organization signing up for on-demand pies.", + "entity_type": "entity/UserTable", + "schema": "PUBLIC", + "database_require_filter": null, + "db": { + "description": "Some example data for you to play around with.", + "features": [ + "index-info", + "basic-aggregations", + "temporal-extract", + "actions/custom", + "now", + "standard-deviation-aggregations", + "date-arithmetics", + "actions", + "expression-aggregations", + "foreign-keys", + "right-join", + "left-join", + "native-parameters", + "schemas", + "nested-queries", + "expressions", + "uploads", + "regex", + "case-sensitivity-string-filter-options", + "binning", + "datetime-diff", + "upload-with-auto-pk", + "inner-join", + "advanced-math-expressions" + ], + "cache_field_values_schedule": "0 0 17 * * ? *", + "timezone": "GMT", + "auto_run_queries": true, + "metadata_sync_schedule": "0 15 * * * ? *", + "name": "Sample Database", + "settings": null, + "caveats": "You probably don't want to use this for your business-critical analyses, but hey, it's your world, we're just living in it.", + "creator_id": null, + "is_full_sync": true, + "updated_at": "2024-05-17T21:30:27.555139Z", + "cache_ttl": null, + "details": { + "db": "file:/plugins/sample-database.db;USER=GUEST;PASSWORD=guest" + }, + "is_sample": true, + "id": 1, + "is_on_demand": false, + "engine": "postgres", + "initial_sync_status": "complete", + "is_audit": false, + "refingerprint": null, + "created_at": "2024-05-17T21:30:26.74786Z", + "points_of_interest": "You can find all sorts of different joinable tables ranging from products to people to reviews here." + }, + "show_in_getting_started": false, + "name": "ACCOUNTS", + "caveats": "Piespace’s business operates with a two week trial period. If you see that “Canceled At” is null then that account is still happily paying for their pies.", + "updated_at": "2024-05-17T21:30:29.436225Z", + "pk_field": 48, + "active": true, + "id": 6, + "db_id": 1, + "visibility_type": null, + "field_order": "database", + "is_upload": false, + "initial_sync_status": "complete", + "display_name": "Accounts", + "created_at": "2024-05-17T21:30:26.9625Z", + "points_of_interest": "Is it? We’ll let you be the judge of that." +} diff --git a/metadata-ingestion/tests/integration/metabase/test_metabase.py b/metadata-ingestion/tests/integration/metabase/test_metabase.py index b39550f3d048a2..b2242a4b93a67c 100644 --- a/metadata-ingestion/tests/integration/metabase/test_metabase.py +++ b/metadata-ingestion/tests/integration/metabase/test_metabase.py @@ -245,3 +245,44 @@ def test_mode_ingest_failure(pytestconfig, tmp_path): assert exec_error.args[0] == "Source reported errors" assert len(exec_error.args[1].failures) == 1 assert list(exec_error.args[1].failures.keys())[0] == "metabase-dashboard" + + +@freeze_time(FROZEN_TIME) +def test_9767_query_with_template_tags_does_not_fail( + pytestconfig, tmp_path, test_pipeline, mock_datahub_graph +): + + baseUrl = "http://localhost:3000/api" + JSON_RESPONSE_MAP[f"{baseUrl}/card/3"] = "issue_9767/card_3.json" + JSON_RESPONSE_MAP[f"{baseUrl}/collection"] = "issue_9767/collection.json" + JSON_RESPONSE_MAP[ + f"{baseUrl}/collection/root/items?models=dashboard" + ] = "issue_9767/collection_dashboard_root.json" + JSON_RESPONSE_MAP[f"{baseUrl}/dashboard/3"] = "issue_9767/dashboard_3.json" + JSON_RESPONSE_MAP[f"{baseUrl}/database/1"] = "issue_9767/database_1.json" + JSON_RESPONSE_MAP[f"{baseUrl}/table/6"] = "issue_9767/table_6.json" + + with patch( + "datahub.ingestion.source.metabase.requests.session", + side_effect=mocked_requests_sucess, + ), patch( + "datahub.ingestion.source.metabase.requests.post", + side_effect=mocked_requests_session_post, + ), patch( + "datahub.ingestion.source.metabase.requests.delete", + side_effect=mocked_requests_session_delete, + ), patch( + "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph", + mock_datahub_graph, + ) as mock_checkpoint: + mock_checkpoint.return_value = mock_datahub_graph + + pipeline = Pipeline.create(test_pipeline) + pipeline.run() + pipeline.raise_from_status() + + report = pipeline.source.get_report() + for warning in report.warnings: + assert "Unable to retrieve lineage from query" not in str( + report.warnings[warning] + ) From 4718f31c88ad4c48653a58af4ddebd0a984e3561 Mon Sep 17 00:00:00 2001 From: Paul Rogalinski-Pinter Date: Sat, 18 May 2024 12:14:57 +0200 Subject: [PATCH 07/14] regenerate response mocks for #9767 using postgres database --- .../metabase/setup/issue_9767/card_1.json | 1 + .../metabase/setup/issue_9767/card_3.json | 159 ------------- .../metabase/setup/issue_9767/collection.json | 13 +- .../issue_9767/collection_dashboard_root.json | 26 +-- .../setup/issue_9767/dashboard_1.json | 1 + .../setup/issue_9767/dashboard_3.json | 215 ------------------ .../metabase/setup/issue_9767/database_1.json | 78 ------- .../metabase/setup/issue_9767/database_2.json | 1 + .../metabase/setup/issue_9767/table_6.json | 73 ------ .../metabase/setup/issue_9767/table_9.json | 1 + .../integration/metabase/test_metabase.py | 8 +- 11 files changed, 10 insertions(+), 566 deletions(-) create mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/card_1.json delete mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/card_3.json create mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/dashboard_1.json delete mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/dashboard_3.json delete mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/database_1.json create mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/database_2.json delete mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/table_6.json create mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/table_9.json diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/card_1.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/card_1.json new file mode 100644 index 00000000000000..6c00a1b17ecb25 --- /dev/null +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/card_1.json @@ -0,0 +1 @@ +{"description":null,"archived":false,"collection_position":null,"table_id":9,"result_metadata":[{"display_name":"dimension","field_ref":["field","dimension",{"base-type":"type/Text"}],"name":"dimension","base_type":"type/Text","effective_type":"type/Text","semantic_type":null,"fingerprint":{"global":{"distinct-count":2,"nil%":0.0},"type":{"type/Text":{"percent-json":0.0,"percent-url":0.0,"percent-email":0.0,"percent-state":0.0,"average-length":1.0}}}},{"display_name":"sum","field_ref":["field","sum",{"base-type":"type/BigInteger"}],"name":"sum","base_type":"type/BigInteger","effective_type":"type/BigInteger","semantic_type":null,"fingerprint":{"global":{"distinct-count":1,"nil%":0.0},"type":{"type/Number":{"min":300.0,"q1":300.0,"q3":300.0,"max":300.0,"sd":0.0,"avg":300.0}}}}],"creator":{"email":"test@metabase.local","first_name":"test","last_login":"2024-05-18T09:30:06.609263Z","is_qbnewb":false,"is_superuser":true,"id":1,"last_name":"user","date_joined":"2024-05-18T09:30:06.405812Z","common_name":"test user"},"initially_published_at":null,"can_write":true,"database_id":2,"enable_embedding":false,"collection_id":null,"query_type":"native","name":"test_question","last_query_start":"2024-05-18T09:37:22.713517Z","dashboard_count":1,"type":"question","average_query_time":27.1111111111111111,"creator_id":1,"moderation_reviews":[],"updated_at":"2024-05-18T09:37:22.729184Z","made_public_by_id":null,"embedding_params":null,"cache_ttl":null,"dataset_query":{"database":2,"type":"native","native":{"collection":"test_table","template-tags":{"dimension":{"type":"dimension","name":"dimension","id":"60f27c21-36ca-4a26-a718-1d9898348ea3","display-name":"Dimension","default":null,"dimension":["field",72,{"base-type":"type/Text"}],"widget-type":"string/=","options":null}},"query":"SELECT\n \"public\".\"test_table\".\"dimension\" AS \"dimension\",\n SUM(\"public\".\"test_table\".\"metric\") AS \"sum\"\nFROM\n \"public\".\"test_table\"\nwhere {{dimension}}\nGROUP BY\n \"public\".\"test_table\".\"dimension\"\nORDER BY\n \"public\".\"test_table\".\"dimension\" ASC\n"}},"id":1,"parameter_mappings":[],"display":"bar","entity_id":"qxLTXxQvaLpflMvpAwqMw","collection_preview":true,"last-edit-info":{"id":1,"email":"test@metabase.local","first_name":"test","last_name":"user","timestamp":"2024-05-18T09:36:45.214507Z"},"visualization_settings":{"graph.dimensions":["dimension"],"graph.metrics":["sum"]},"collection":{"metabase.models.collection.root/is-root?":true,"authority_level":null,"name":"Our analytics","is_personal":false,"id":"root","can_write":true},"metabase_version":"v0.49.11 (b894f2d)","parameters":[{"id":"60f27c21-36ca-4a26-a718-1d9898348ea3","type":"string/=","target":["dimension",["template-tag","dimension"]],"name":"Dimension","slug":"dimension"}],"dataset":false,"created_at":"2024-05-18T09:31:36.591212Z","parameter_usage_count":0,"public_uuid":null} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/card_3.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/card_3.json deleted file mode 100644 index 21308d73d8f84a..00000000000000 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/card_3.json +++ /dev/null @@ -1,159 +0,0 @@ -{ - "description": null, - "archived": false, - "collection_position": null, - "table_id": 6, - "result_metadata": [ - { - "display_name": "SOURCE", - "field_ref": [ - "field", - "SOURCE", - { - "base-type": "type/Text" - } - ], - "name": "SOURCE", - "base_type": "type/Text", - "effective_type": "type/Text", - "semantic_type": "type/Source", - "fingerprint": { - "global": { - "distinct-count": 5, - "nil%": 0.2 - }, - "type": { - "type/Text": { - "percent-json": 0.0, - "percent-url": 0.0, - "percent-email": 0.0, - "percent-state": 0.0, - "average-length": 5.4 - } - } - } - }, - { - "display_name": "sum", - "field_ref": [ - "field", - "sum", - { - "base-type": "type/BigInteger" - } - ], - "name": "sum", - "base_type": "type/BigInteger", - "effective_type": "type/BigInteger", - "semantic_type": null, - "fingerprint": { - "global": { - "distinct-count": 5, - "nil%": 0.0 - }, - "type": { - "type/Number": { - "min": 5634.0, - "q1": 5650.5, - "q3": 8961.75, - "max": 17178.0, - "sd": 5084.457080161067, - "avg": 8092.6 - } - } - } - } - ], - "creator": { - "email": "pulsar@snowtronix.local", - "first_name": "Johny", - "last_login": "2024-05-17T21:31:25.411414Z", - "is_qbnewb": false, - "is_superuser": true, - "id": 1, - "last_name": "Appleseed", - "date_joined": "2024-05-17T21:31:25.323423Z", - "common_name": "Johny Appleseed" - }, - "initially_published_at": null, - "can_write": true, - "database_id": 1, - "enable_embedding": false, - "collection_id": null, - "query_type": "native", - "name": "Accounts, Sum of Seats, Grouped by Source - Duplicate - Duplicate", - "last_query_start": "2024-05-17T21:51:20.209029Z", - "dashboard_count": 1, - "type": "question", - "average_query_time": 23.76923076923077, - "creator_id": 1, - "moderation_reviews": [], - "updated_at": "2024-05-17T21:51:20.217914Z", - "made_public_by_id": null, - "embedding_params": null, - "cache_ttl": null, - "dataset_query": { - "database": 1, - "type": "native", - "native": { - "template-tags": { - "COUNTRY": { - "type": "text", - "name": "COUNTRY", - "id": "74499e6a-146d-4107-9508-9801deeeae5c", - "display-name": "Country" - } - }, - "collection": "ACCOUNTS", - "query": "SELECT\n \"PUBLIC\".\"ACCOUNTS\".\"SOURCE\" AS \"SOURCE\",\n SUM(\"PUBLIC\".\"ACCOUNTS\".\"SEATS\") AS \"sum\"\nFROM\n \"PUBLIC\".\"ACCOUNTS\"\nGROUP BY\n \"PUBLIC\".\"ACCOUNTS\".\"SOURCE\"\nORDER BY\n \"PUBLIC\".\"ACCOUNTS\".\"SOURCE\" ASC\n[[where PUBLIC.ACCOUNTS.COUNTRY == {{COUNTRY}}]]" - } - }, - "id": 3, - "parameter_mappings": [], - "display": "bar", - "entity_id": "uns9F4BFNsCq8YFMbrCdO", - "collection_preview": true, - "last-edit-info": { - "id": 1, - "email": "pulsar@snowtronix.local", - "first_name": "Johny", - "last_name": "Appleseed", - "timestamp": "2024-05-17T21:51:07.409582Z" - }, - "visualization_settings": { - "graph.dimensions": [ - "SOURCE" - ], - "graph.metrics": [ - "sum" - ] - }, - "collection": { - "metabase.models.collection.root/is-root?": true, - "authority_level": null, - "name": "Our analytics", - "is_personal": false, - "id": "root", - "can_write": true - }, - "metabase_version": "v0.49.11 (b894f2d)", - "parameters": [ - { - "id": "74499e6a-146d-4107-9508-9801deeeae5c", - "type": "category", - "target": [ - "variable", - [ - "template-tag", - "COUNTRY" - ] - ], - "name": "Country", - "slug": "COUNTRY" - } - ], - "dataset": false, - "created_at": "2024-05-17T21:33:56.551981Z", - "parameter_usage_count": 0, - "public_uuid": null -} diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection.json index 13ae56d9bf9196..52fb4873fc2cf8 100644 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection.json +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection.json @@ -1,12 +1 @@ -[ - { - "authority_level": null, - "can_write": true, - "name": "Our analytics", - "effective_ancestors": [], - "effective_location": null, - "parent_id": null, - "id": "root", - "is_personal": false - } -] +[{"authority_level":null,"can_write":true,"name":"Our analytics","effective_ancestors":[],"effective_location":null,"parent_id":null,"id":"root","is_personal":false},{"authority_level":null,"description":null,"archived":false,"slug":"test__s_personal_collection","can_write":true,"name":"test 's Personal Collection","personal_owner_id":2,"type":null,"id":2,"entity_id":"QwfpYCNTDpVwl60_VcU_s","location":"/","namespace":null,"is_personal":true,"created_at":"2024-05-18T09:38:57.038406Z"},{"authority_level":null,"description":null,"archived":false,"slug":"test_user_s_personal_collection","can_write":true,"name":"test user's Personal Collection","personal_owner_id":1,"type":null,"id":1,"entity_id":"yJuxECR7_ZrIuURr26lYM","location":"/","namespace":null,"is_personal":true,"created_at":"2024-05-18T09:30:09.161522Z"}] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection_dashboard_root.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection_dashboard_root.json index c711f6d25d532b..9bf6c1a5489f7e 100644 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection_dashboard_root.json +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection_dashboard_root.json @@ -1,25 +1 @@ -{ - "total": 3, - "data": [ - { - "description": "test", - "collection_position": null, - "database_id": null, - "name": "test filter", - "id": 3, - "entity_id": "cNXZhyVHAOEONhw5hByKX", - "last-edit-info": { - "id": 1, - "last_name": "Appleseed", - "first_name": "Johny", - "email": "pulsar@snowtronix.local", - "timestamp": "2024-05-17T21:51:20.080515Z" - }, - "model": "dashboard" - }], - "models": [ - "dashboard" - ], - "limit": null, - "offset": null -} +{"total":1,"data":[{"description":null,"collection_position":null,"database_id":null,"name":"test_dashboard","id":1,"entity_id":"jvj_rwlh_i0KFlkMUVtb5","last-edit-info":{"id":1,"last_name":"user","first_name":"test","email":"test@metabase.local","timestamp":"2024-05-18T09:37:18.413013Z"},"model":"dashboard"}],"models":["dashboard"],"limit":null,"offset":null} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/dashboard_1.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/dashboard_1.json new file mode 100644 index 00000000000000..d80ced2071fb3a --- /dev/null +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/dashboard_1.json @@ -0,0 +1 @@ +{"description":null,"archived":false,"collection_position":null,"dashcards":[{"size_x":12,"dashboard_tab_id":null,"series":[],"action_id":null,"collection_authority_level":null,"card":{"description":null,"archived":false,"collection_position":null,"table_id":9,"result_metadata":[{"display_name":"dimension","field_ref":["field","dimension",{"base-type":"type/Text"}],"name":"dimension","base_type":"type/Text","effective_type":"type/Text","semantic_type":null,"fingerprint":{"global":{"distinct-count":2,"nil%":0.0},"type":{"type/Text":{"percent-json":0.0,"percent-url":0.0,"percent-email":0.0,"percent-state":0.0,"average-length":1.0}}}},{"display_name":"sum","field_ref":["field","sum",{"base-type":"type/BigInteger"}],"name":"sum","base_type":"type/BigInteger","effective_type":"type/BigInteger","semantic_type":null,"fingerprint":{"global":{"distinct-count":1,"nil%":0.0},"type":{"type/Number":{"min":300.0,"q1":300.0,"q3":300.0,"max":300.0,"sd":0.0,"avg":300.0}}}}],"initially_published_at":null,"can_write":true,"database_id":2,"enable_embedding":false,"collection_id":null,"query_type":"native","name":"test_question","type":"question","query_average_duration":16,"creator_id":1,"moderation_reviews":[],"updated_at":"2024-05-18T09:37:22.729184Z","made_public_by_id":null,"embedding_params":null,"cache_ttl":null,"dataset_query":{"database":2,"type":"native","native":{"collection":"test_table","template-tags":{"dimension":{"type":"dimension","name":"dimension","id":"60f27c21-36ca-4a26-a718-1d9898348ea3","display-name":"Dimension","default":null,"dimension":["field",72,{"base-type":"type/Text"}],"widget-type":"string/=","options":null}},"query":"SELECT\n \"public\".\"test_table\".\"dimension\" AS \"dimension\",\n SUM(\"public\".\"test_table\".\"metric\") AS \"sum\"\nFROM\n \"public\".\"test_table\"\nwhere {{dimension}}\nGROUP BY\n \"public\".\"test_table\".\"dimension\"\nORDER BY\n \"public\".\"test_table\".\"dimension\" ASC\n"}},"id":1,"parameter_mappings":[],"display":"bar","entity_id":"qxLTXxQvaLpflMvpAwqMw","collection_preview":true,"visualization_settings":{"graph.dimensions":["dimension"],"graph.metrics":["sum"]},"metabase_version":"v0.49.11 (b894f2d)","parameters":[{"id":"60f27c21-36ca-4a26-a718-1d9898348ea3","type":"string/=","target":["dimension",["template-tag","dimension"]],"name":"Dimension","slug":"dimension"}],"dataset":false,"created_at":"2024-05-18T09:31:36.591212Z","public_uuid":null},"updated_at":"2024-05-18T09:37:18.369753Z","col":0,"id":1,"parameter_mappings":[{"parameter_id":"70f9d7fc","card_id":1,"target":["dimension",["template-tag","dimension"]]}],"card_id":1,"entity_id":"1F8zX4QzW5W8kf7R4exVr","visualization_settings":{},"size_y":6,"dashboard_id":1,"created_at":"2024-05-18T09:31:52.570882Z","row":0}],"param_values":{"72":{"field_id":72,"human_readable_values":[],"values":["A","B"]}},"initially_published_at":null,"can_write":true,"tabs":[],"enable_embedding":false,"collection_id":null,"show_in_getting_started":false,"name":"test_dashboard","width":"fixed","caveats":null,"collection_authority_level":null,"creator_id":1,"updated_at":"2024-05-18T09:37:18.369753Z","made_public_by_id":null,"embedding_params":null,"cache_ttl":null,"id":1,"position":null,"entity_id":"jvj_rwlh_i0KFlkMUVtb5","param_fields":{"72":{"id":72,"table_id":9,"display_name":"Dimension","base_type":"type/Text","semantic_type":"type/Category","has_field_values":"list","name_field":null,"dimensions":[]}},"last-edit-info":{"id":1,"email":"test@metabase.local","first_name":"test","last_name":"user","timestamp":"2024-05-18T09:37:18.413013Z"},"collection":{"metabase.models.collection.root/is-root?":true,"authority_level":null,"name":"Our analytics","is_personal":false,"id":"root","can_write":true},"parameters":[{"name":"Text","slug":"text","id":"70f9d7fc","type":"string/=","sectionId":"string"}],"auto_apply_filters":true,"created_at":"2024-05-18T09:31:43.161863Z","public_uuid":null,"points_of_interest":null} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/dashboard_3.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/dashboard_3.json deleted file mode 100644 index b3c4206ce630cc..00000000000000 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/dashboard_3.json +++ /dev/null @@ -1,215 +0,0 @@ -{ - "description": "test", - "archived": false, - "collection_position": null, - "dashcards": [ - { - "size_x": 12, - "dashboard_tab_id": null, - "series": [], - "action_id": null, - "collection_authority_level": null, - "card": { - "description": null, - "archived": false, - "collection_position": null, - "table_id": 6, - "result_metadata": [ - { - "display_name": "SOURCE", - "field_ref": [ - "field", - "SOURCE", - { - "base-type": "type/Text" - } - ], - "name": "SOURCE", - "base_type": "type/Text", - "effective_type": "type/Text", - "semantic_type": "type/Source", - "fingerprint": { - "global": { - "distinct-count": 5, - "nil%": 0.2 - }, - "type": { - "type/Text": { - "percent-json": 0.0, - "percent-url": 0.0, - "percent-email": 0.0, - "percent-state": 0.0, - "average-length": 5.4 - } - } - } - }, - { - "display_name": "sum", - "field_ref": [ - "field", - "sum", - { - "base-type": "type/BigInteger" - } - ], - "name": "sum", - "base_type": "type/BigInteger", - "effective_type": "type/BigInteger", - "semantic_type": null, - "fingerprint": { - "global": { - "distinct-count": 5, - "nil%": 0.0 - }, - "type": { - "type/Number": { - "min": 5634.0, - "q1": 5650.5, - "q3": 8961.75, - "max": 17178.0, - "sd": 5084.457080161067, - "avg": 8092.6 - } - } - } - } - ], - "initially_published_at": null, - "can_write": true, - "database_id": 1, - "enable_embedding": false, - "collection_id": null, - "query_type": "native", - "name": "Accounts, Sum of Seats, Grouped by Source - Duplicate - Duplicate", - "type": "question", - "query_average_duration": null, - "creator_id": 1, - "moderation_reviews": [], - "updated_at": "2024-05-17T21:51:20.217914Z", - "made_public_by_id": null, - "embedding_params": null, - "cache_ttl": null, - "dataset_query": { - "database": 1, - "type": "native", - "native": { - "template-tags": { - "COUNTRY": { - "type": "text", - "name": "COUNTRY", - "id": "74499e6a-146d-4107-9508-9801deeeae5c", - "display-name": "Country" - } - }, - "collection": "ACCOUNTS", - "query": "SELECT\n \"PUBLIC\".\"ACCOUNTS\".\"SOURCE\" AS \"SOURCE\",\n SUM(\"PUBLIC\".\"ACCOUNTS\".\"SEATS\") AS \"sum\"\nFROM\n \"PUBLIC\".\"ACCOUNTS\"\nGROUP BY\n \"PUBLIC\".\"ACCOUNTS\".\"SOURCE\"\nORDER BY\n \"PUBLIC\".\"ACCOUNTS\".\"SOURCE\" ASC\n[[where PUBLIC.ACCOUNTS.COUNTRY == {{COUNTRY}}]]" - } - }, - "id": 3, - "parameter_mappings": [], - "display": "bar", - "entity_id": "uns9F4BFNsCq8YFMbrCdO", - "collection_preview": true, - "visualization_settings": { - "graph.dimensions": [ - "SOURCE" - ], - "graph.metrics": [ - "sum" - ] - }, - "metabase_version": "v0.49.11 (b894f2d)", - "parameters": [ - { - "id": "74499e6a-146d-4107-9508-9801deeeae5c", - "type": "category", - "target": [ - "variable", - [ - "template-tag", - "COUNTRY" - ] - ], - "name": "Country", - "slug": "COUNTRY" - } - ], - "dataset": false, - "created_at": "2024-05-17T21:33:56.551981Z", - "public_uuid": null - }, - "updated_at": "2024-05-17T21:51:20.06007Z", - "col": 0, - "id": 3, - "parameter_mappings": [ - { - "parameter_id": "5d1aeef8", - "card_id": 3, - "target": [ - "variable", - [ - "template-tag", - "COUNTRY" - ] - ] - } - ], - "card_id": 3, - "entity_id": "GT9BVzh-D95Kc7-slYhj1", - "visualization_settings": {}, - "size_y": 6, - "dashboard_id": 3, - "created_at": "2024-05-17T21:33:56.551981Z", - "row": 0 - } - ], - "param_values": null, - "initially_published_at": null, - "can_write": true, - "tabs": [], - "enable_embedding": false, - "collection_id": null, - "show_in_getting_started": false, - "name": "test filter", - "width": "fixed", - "caveats": null, - "collection_authority_level": null, - "creator_id": 1, - "updated_at": "2024-05-17T21:51:20.06007Z", - "made_public_by_id": null, - "embedding_params": null, - "cache_ttl": null, - "id": 3, - "position": null, - "entity_id": "cNXZhyVHAOEONhw5hByKX", - "param_fields": null, - "last-edit-info": { - "id": 1, - "email": "pulsar@snowtronix.local", - "first_name": "Johny", - "last_name": "Appleseed", - "timestamp": "2024-05-17T21:51:20.080515Z" - }, - "collection": { - "metabase.models.collection.root/is-root?": true, - "authority_level": null, - "name": "Our analytics", - "is_personal": false, - "id": "root", - "can_write": true - }, - "parameters": [ - { - "name": "Location", - "slug": "location", - "id": "5d1aeef8", - "type": "string/=", - "sectionId": "location" - } - ], - "auto_apply_filters": true, - "created_at": "2024-05-17T21:33:56.551981Z", - "public_uuid": null, - "points_of_interest": null -} diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/database_1.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/database_1.json deleted file mode 100644 index 851ec8aadcccc0..00000000000000 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/database_1.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "description": "Some example data for you to play around with.", - "features": [ - "index-info", - "basic-aggregations", - "temporal-extract", - "actions/custom", - "now", - "standard-deviation-aggregations", - "date-arithmetics", - "actions", - "expression-aggregations", - "foreign-keys", - "right-join", - "left-join", - "native-parameters", - "schemas", - "nested-queries", - "expressions", - "uploads", - "regex", - "case-sensitivity-string-filter-options", - "binning", - "datetime-diff", - "upload-with-auto-pk", - "inner-join", - "advanced-math-expressions" - ], - "cache_field_values_schedule": "0 0 17 * * ? *", - "timezone": "GMT", - "auto_run_queries": true, - "metadata_sync_schedule": "0 15 * * * ? *", - "name": "Sample Database", - "settings": null, - "caveats": "You probably don't want to use this for your business-critical analyses, but hey, it's your world, we're just living in it.", - "can-manage": true, - "creator_id": null, - "is_full_sync": true, - "updated_at": "2024-05-17T21:30:27.555139Z", - "cache_ttl": null, - "details": { - "db": "file:/plugins/sample-database.db;USER=GUEST;PASSWORD=guest" - }, - "is_sample": true, - "id": 1, - "is_on_demand": false, - "schedules": { - "metadata_sync": { - "schedule_minute": 15, - "schedule_day": null, - "schedule_frame": null, - "schedule_hour": null, - "schedule_type": "hourly" - }, - "cache_field_values": { - "schedule_minute": 0, - "schedule_day": null, - "schedule_frame": null, - "schedule_hour": 17, - "schedule_type": "daily" - } - }, - "engine": "postgres", - "initial_sync_status": "complete", - "is_audit": false, - "dbms_version": { - "flavor": "H2", - "version": "2.1.214 (2022-06-13)", - "semantic-version": [ - 2, - 1 - ] - }, - "refingerprint": null, - "created_at": "2024-05-17T21:30:26.74786Z", - "points_of_interest": "You can find all sorts of different joinable tables ranging from products to people to reviews here.", - "can_upload": false -} diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/database_2.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/database_2.json new file mode 100644 index 00000000000000..b723002021f835 --- /dev/null +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/database_2.json @@ -0,0 +1 @@ +{"description":null,"features":["index-info","full-join","basic-aggregations","temporal-extract","actions/custom","now","convert-timezone","nested-field-columns","standard-deviation-aggregations","test/jvm-timezone-setting","date-arithmetics","persist-models","actions","expression-aggregations","percentile-aggregations","connection-impersonation","foreign-keys","table-privileges","right-join","left-join","native-parameters","schemas","nested-queries","expressions","uploads","set-timezone","regex","case-sensitivity-string-filter-options","binning","datetime-diff","upload-with-auto-pk","inner-join","advanced-math-expressions"],"cache_field_values_schedule":"0 0 6 * * ? *","timezone":"GMT","auto_run_queries":true,"metadata_sync_schedule":"0 19 * * * ? *","name":"test_database","settings":null,"caveats":null,"can-manage":true,"creator_id":1,"is_full_sync":true,"updated_at":"2024-05-18T09:30:06.734897Z","cache_ttl":null,"details":{"ssl":false,"ssl-key-source":null,"password":"**MetabasePass**","ssl-key-creator-id":null,"ssl-key-password-source":null,"port":null,"advanced-options":false,"schema-filters-type":"all","dbname":"test_data","ssl-root-cert-creator-id":null,"host":"postgres","tunnel-enabled":false,"ssl-key-password-creator-id":null,"ssl-root-cert-source":null,"ssl-client-cert-creator-id":null,"user":"metabase","ssl-client-cert-source":null},"is_sample":false,"id":2,"is_on_demand":false,"schedules":{"metadata_sync":{"schedule_minute":19,"schedule_day":null,"schedule_frame":null,"schedule_hour":null,"schedule_type":"hourly"},"cache_field_values":{"schedule_minute":0,"schedule_day":null,"schedule_frame":null,"schedule_hour":6,"schedule_type":"daily"}},"engine":"postgres","initial_sync_status":"complete","is_audit":false,"dbms_version":{"flavor":"PostgreSQL","version":"16.3 (Debian 16.3-1.pgdg120+1)","semantic-version":[16,3]},"refingerprint":null,"created_at":"2024-05-18T09:30:06.405812Z","points_of_interest":null,"can_upload":false} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/table_6.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/table_6.json deleted file mode 100644 index aa2501fde6aebf..00000000000000 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/table_6.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "description": "Information on customer accounts registered with Piespace. Each account represents a new organization signing up for on-demand pies.", - "entity_type": "entity/UserTable", - "schema": "PUBLIC", - "database_require_filter": null, - "db": { - "description": "Some example data for you to play around with.", - "features": [ - "index-info", - "basic-aggregations", - "temporal-extract", - "actions/custom", - "now", - "standard-deviation-aggregations", - "date-arithmetics", - "actions", - "expression-aggregations", - "foreign-keys", - "right-join", - "left-join", - "native-parameters", - "schemas", - "nested-queries", - "expressions", - "uploads", - "regex", - "case-sensitivity-string-filter-options", - "binning", - "datetime-diff", - "upload-with-auto-pk", - "inner-join", - "advanced-math-expressions" - ], - "cache_field_values_schedule": "0 0 17 * * ? *", - "timezone": "GMT", - "auto_run_queries": true, - "metadata_sync_schedule": "0 15 * * * ? *", - "name": "Sample Database", - "settings": null, - "caveats": "You probably don't want to use this for your business-critical analyses, but hey, it's your world, we're just living in it.", - "creator_id": null, - "is_full_sync": true, - "updated_at": "2024-05-17T21:30:27.555139Z", - "cache_ttl": null, - "details": { - "db": "file:/plugins/sample-database.db;USER=GUEST;PASSWORD=guest" - }, - "is_sample": true, - "id": 1, - "is_on_demand": false, - "engine": "postgres", - "initial_sync_status": "complete", - "is_audit": false, - "refingerprint": null, - "created_at": "2024-05-17T21:30:26.74786Z", - "points_of_interest": "You can find all sorts of different joinable tables ranging from products to people to reviews here." - }, - "show_in_getting_started": false, - "name": "ACCOUNTS", - "caveats": "Piespace’s business operates with a two week trial period. If you see that “Canceled At” is null then that account is still happily paying for their pies.", - "updated_at": "2024-05-17T21:30:29.436225Z", - "pk_field": 48, - "active": true, - "id": 6, - "db_id": 1, - "visibility_type": null, - "field_order": "database", - "is_upload": false, - "initial_sync_status": "complete", - "display_name": "Accounts", - "created_at": "2024-05-17T21:30:26.9625Z", - "points_of_interest": "Is it? We’ll let you be the judge of that." -} diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/table_9.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/table_9.json new file mode 100644 index 00000000000000..458b1a11b505c0 --- /dev/null +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/table_9.json @@ -0,0 +1 @@ +{"description":null,"entity_type":"entity/GenericTable","schema":"public","database_require_filter":null,"db":{"description":null,"features":["index-info","full-join","basic-aggregations","temporal-extract","actions/custom","now","convert-timezone","nested-field-columns","standard-deviation-aggregations","test/jvm-timezone-setting","date-arithmetics","persist-models","actions","expression-aggregations","percentile-aggregations","connection-impersonation","foreign-keys","table-privileges","right-join","left-join","native-parameters","schemas","nested-queries","expressions","uploads","set-timezone","regex","case-sensitivity-string-filter-options","binning","datetime-diff","upload-with-auto-pk","inner-join","advanced-math-expressions"],"cache_field_values_schedule":"0 0 6 * * ? *","timezone":"GMT","auto_run_queries":true,"metadata_sync_schedule":"0 19 * * * ? *","name":"test_database","settings":null,"caveats":null,"creator_id":1,"is_full_sync":true,"updated_at":"2024-05-18T09:30:06.734897Z","cache_ttl":null,"details":{"ssl":false,"password":"**MetabasePass**","port":null,"advanced-options":false,"schema-filters-type":"all","dbname":"test_data","host":"postgres","tunnel-enabled":false,"user":"metabase"},"is_sample":false,"id":2,"is_on_demand":false,"engine":"postgres","initial_sync_status":"complete","is_audit":false,"dbms_version":{"flavor":"PostgreSQL","version":"16.3 (Debian 16.3-1.pgdg120+1)","semantic-version":[16,3]},"refingerprint":null,"created_at":"2024-05-18T09:30:06.405812Z","points_of_interest":null},"show_in_getting_started":false,"name":"test_table","caveats":null,"updated_at":"2024-05-18T09:30:06.783792Z","pk_field":74,"active":true,"id":9,"db_id":2,"visibility_type":null,"field_order":"database","is_upload":false,"initial_sync_status":"complete","display_name":"Test Table","created_at":"2024-05-18T09:30:06.660148Z","points_of_interest":null} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/test_metabase.py b/metadata-ingestion/tests/integration/metabase/test_metabase.py index b2242a4b93a67c..a34aa0c3c1fa1a 100644 --- a/metadata-ingestion/tests/integration/metabase/test_metabase.py +++ b/metadata-ingestion/tests/integration/metabase/test_metabase.py @@ -253,14 +253,14 @@ def test_9767_query_with_template_tags_does_not_fail( ): baseUrl = "http://localhost:3000/api" - JSON_RESPONSE_MAP[f"{baseUrl}/card/3"] = "issue_9767/card_3.json" + JSON_RESPONSE_MAP[f"{baseUrl}/card/3"] = "issue_9767/card_1.json" JSON_RESPONSE_MAP[f"{baseUrl}/collection"] = "issue_9767/collection.json" JSON_RESPONSE_MAP[ f"{baseUrl}/collection/root/items?models=dashboard" ] = "issue_9767/collection_dashboard_root.json" - JSON_RESPONSE_MAP[f"{baseUrl}/dashboard/3"] = "issue_9767/dashboard_3.json" - JSON_RESPONSE_MAP[f"{baseUrl}/database/1"] = "issue_9767/database_1.json" - JSON_RESPONSE_MAP[f"{baseUrl}/table/6"] = "issue_9767/table_6.json" + JSON_RESPONSE_MAP[f"{baseUrl}/dashboard/1"] = "issue_9767/dashboard_1.json" + JSON_RESPONSE_MAP[f"{baseUrl}/database/1"] = "issue_9767/database_2.json" + JSON_RESPONSE_MAP[f"{baseUrl}/table/9"] = "issue_9767/table_9.json" with patch( "datahub.ingestion.source.metabase.requests.session", From e720a7ebf7bedc468a0f9650fcb7fac7577de89f Mon Sep 17 00:00:00 2001 From: Paul Rogalinski-Pinter Date: Sat, 18 May 2024 22:59:46 +0200 Subject: [PATCH 08/14] fix and do not share mocked response mappings between tests --- .../metabase/setup/issue_9767/card.json | 1 + .../metabase/setup/issue_9767/collection.json | 13 +- .../metabase/setup/issue_9767/user.json | 1 + .../metabase/setup/issue_9767/user_1.json | 1 + .../integration/metabase/test_metabase.py | 176 ++++++++++++------ 5 files changed, 136 insertions(+), 56 deletions(-) create mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/card.json create mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/user.json create mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/user_1.json diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/card.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/card.json new file mode 100644 index 00000000000000..2cd697b0291f26 --- /dev/null +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/card.json @@ -0,0 +1 @@ +[{"description":null,"archived":false,"collection_position":null,"table_id":9,"result_metadata":[{"display_name":"dimension","field_ref":["field","dimension",{"base-type":"type/Text"}],"name":"dimension","base_type":"type/Text","effective_type":"type/Text","semantic_type":null,"fingerprint":{"global":{"distinct-count":2,"nil%":0.0},"type":{"type/Text":{"percent-json":0.0,"percent-url":0.0,"percent-email":0.0,"percent-state":0.0,"average-length":1.0}}}},{"display_name":"sum","field_ref":["field","sum",{"base-type":"type/BigInteger"}],"name":"sum","base_type":"type/BigInteger","effective_type":"type/BigInteger","semantic_type":null,"fingerprint":{"global":{"distinct-count":1,"nil%":0.0},"type":{"type/Number":{"min":300.0,"q1":300.0,"q3":300.0,"max":300.0,"sd":0.0,"avg":300.0}}}}],"creator":{"email":"test@metabase.local","first_name":"test","last_login":"2024-05-18T09:30:06.609263Z","is_qbnewb":false,"is_superuser":true,"id":1,"last_name":"user","date_joined":"2024-05-18T09:30:06.405812Z","common_name":"test user"},"initially_published_at":null,"database_id":2,"enable_embedding":false,"collection_id":null,"query_type":"native","name":"test_question","type":"question","creator_id":1,"updated_at":"2024-05-18T18:22:04.437454Z","made_public_by_id":null,"embedding_params":null,"cache_ttl":null,"dataset_query":{"database":2,"type":"native","native":{"collection":"test_table","template-tags":{"dimension":{"type":"dimension","name":"dimension","id":"60f27c21-36ca-4a26-a718-1d9898348ea3","display-name":"Dimension","default":null,"dimension":["field",72,{"base-type":"type/Text"}],"widget-type":"string/=","options":null}},"query":"SELECT\n \"public\".\"test_table\".\"dimension\" AS \"dimension\",\n SUM(\"public\".\"test_table\".\"metric\") AS \"sum\"\nFROM\n \"public\".\"test_table\"\nwhere {{dimension}}\nGROUP BY\n \"public\".\"test_table\".\"dimension\"\nORDER BY\n \"public\".\"test_table\".\"dimension\" ASC\n"}},"id":1,"parameter_mappings":[],"display":"bar","entity_id":"qxLTXxQvaLpflMvpAwqMw","collection_preview":true,"last-edit-info":{"id":1,"email":"test@metabase.local","first_name":"test","last_name":"user","timestamp":"2024-05-18T09:36:45.214507Z"},"visualization_settings":{"graph.dimensions":["dimension"],"graph.metrics":["sum"]},"collection":null,"metabase_version":"v0.49.11 (b894f2d)","parameters":[{"id":"60f27c21-36ca-4a26-a718-1d9898348ea3","type":"string/=","target":["dimension",["template-tag","dimension"]],"name":"Dimension","slug":"dimension"}],"dataset":false,"created_at":"2024-05-18T09:31:36.591212Z","public_uuid":null}] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection.json index 52fb4873fc2cf8..13ae56d9bf9196 100644 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection.json +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection.json @@ -1 +1,12 @@ -[{"authority_level":null,"can_write":true,"name":"Our analytics","effective_ancestors":[],"effective_location":null,"parent_id":null,"id":"root","is_personal":false},{"authority_level":null,"description":null,"archived":false,"slug":"test__s_personal_collection","can_write":true,"name":"test 's Personal Collection","personal_owner_id":2,"type":null,"id":2,"entity_id":"QwfpYCNTDpVwl60_VcU_s","location":"/","namespace":null,"is_personal":true,"created_at":"2024-05-18T09:38:57.038406Z"},{"authority_level":null,"description":null,"archived":false,"slug":"test_user_s_personal_collection","can_write":true,"name":"test user's Personal Collection","personal_owner_id":1,"type":null,"id":1,"entity_id":"yJuxECR7_ZrIuURr26lYM","location":"/","namespace":null,"is_personal":true,"created_at":"2024-05-18T09:30:09.161522Z"}] \ No newline at end of file +[ + { + "authority_level": null, + "can_write": true, + "name": "Our analytics", + "effective_ancestors": [], + "effective_location": null, + "parent_id": null, + "id": "root", + "is_personal": false + } +] diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/user.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/user.json new file mode 100644 index 00000000000000..c1adfc798982c8 --- /dev/null +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/user.json @@ -0,0 +1 @@ +{"data":[{"email":"test@metabase.local","first_name":"test","locale":null,"last_login":"2024-05-18T09:30:06.609263Z","is_active":true,"is_qbnewb":false,"updated_at":"2024-05-18T09:31:55.368683Z","group_ids":[1,2],"is_superuser":true,"login_attributes":null,"id":1,"last_name":"user","date_joined":"2024-05-18T09:30:06.405812Z","sso_source":null,"personal_collection_id":1,"common_name":"test user"}],"total":1,"limit":null,"offset":null} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/user_1.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/user_1.json new file mode 100644 index 00000000000000..bccf0a9cd6fb9a --- /dev/null +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/user_1.json @@ -0,0 +1 @@ +{"email":"test@metabase.local","first_name":"test","locale":null,"last_login":"2024-05-18T09:30:06.609263Z","is_active":true,"user_group_memberships":[{"id":1},{"id":2}],"is_qbnewb":false,"updated_at":"2024-05-18T09:31:55.368683Z","is_superuser":true,"login_attributes":null,"id":1,"last_name":"user","date_joined":"2024-05-18T09:30:06.405812Z","sso_source":null,"common_name":"test user"} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/test_metabase.py b/metadata-ingestion/tests/integration/metabase/test_metabase.py index a34aa0c3c1fa1a..932cdfc5fe5ef9 100644 --- a/metadata-ingestion/tests/integration/metabase/test_metabase.py +++ b/metadata-ingestion/tests/integration/metabase/test_metabase.py @@ -20,23 +20,6 @@ GMS_PORT = 8080 GMS_SERVER = f"http://localhost:{GMS_PORT}" -JSON_RESPONSE_MAP = { - "http://localhost:3000/api/session": "session.json", - "http://localhost:3000/api/user/current": "user.json", - "http://localhost:3000/api/collection/?exclude-other-user-collections=false": "collections.json", - "http://localhost:3000/api/collection/root/items?models=dashboard": "collection_dashboards.json", - "http://localhost:3000/api/collection/150/items?models=dashboard": "collection_dashboards.json", - "http://localhost:3000/api/dashboard/10": "dashboard_1.json", - "http://localhost:3000/api/dashboard/20": "dashboard_2.json", - "http://localhost:3000/api/user/1": "user.json", - "http://localhost:3000/api/card": "card.json", - "http://localhost:3000/api/database/1": "bigquery_database.json", - "http://localhost:3000/api/database/2": "postgres_database.json", - "http://localhost:3000/api/card/1": "card_1.json", - "http://localhost:3000/api/card/2": "card_2.json", - "http://localhost:3000/api/table/21": "table_21.json", - "http://localhost:3000/api/card/3": "card_3.json", -} RESPONSE_ERROR_LIST = ["http://localhost:3000/api/dashboard/public"] @@ -44,7 +27,9 @@ class MockResponse: - def __init__(self, url, data=None, jsond=None, error_list=None): + def __init__( + self, url, json_response_map=None, data=None, jsond=None, error_list=None + ): self.json_data = data self.url = url self.jsond = jsond @@ -52,11 +37,17 @@ def __init__(self, url, data=None, jsond=None, error_list=None): self.headers = {} self.auth = None self.status_code = 200 + self.response_map = json_response_map def json(self): - response_json_path = ( - f"{test_resources_dir}/setup/{JSON_RESPONSE_MAP.get(self.url)}" - ) + mocked_response_file = self.response_map.get(self.url) + response_json_path = f"{test_resources_dir}/setup/{mocked_response_file}" + + if not pathlib.Path(response_json_path).exists(): + raise Exception( + f"mock response file not found {self.url} -> {mocked_response_file}" + ) + with open(response_json_path) as file: data = json.loads(file.read()) self.json_data = data @@ -75,21 +66,68 @@ def raise_for_status(self): ) raise HTTPError(http_error_msg, response=self) + @staticmethod + def build_mocked_requests_sucess(json_response_map): + def mocked_requests_sucess_(*args, **kwargs): + return MockResponse(url=None, json_response_map=json_response_map) -def mocked_requests_sucess(*args, **kwargs): - return MockResponse(None) + return mocked_requests_sucess_ + @staticmethod + def build_mocked_requests_failure(json_response_map): + def mocked_requests_failure(*args, **kwargs): + return MockResponse( + url=None, + error_list=RESPONSE_ERROR_LIST, + json_response_map=json_response_map, + ) + + return mocked_requests_failure + + @staticmethod + def build_mocked_requests_session_post(json_response_map): + def mocked_requests_session_post(url, data, json): + return MockResponse( + url=url, + data=data, + jsond=json, + json_response_map=json_response_map, + ) -def mocked_requests_failure(*args, **kwargs): - return MockResponse(None, error_list=RESPONSE_ERROR_LIST) + return mocked_requests_session_post + @staticmethod + def build_mocked_requests_session_delete(json_response_map): + def mocked_requests_session_delete(url, headers): + return MockResponse( + url=url, + data=None, + jsond=headers, + json_response_map=json_response_map, + ) -def mocked_requests_session_post(url, data, json): - return MockResponse(url, data, json) + return mocked_requests_session_delete -def mocked_requests_session_delete(url, headers): - return MockResponse(url, data=None, jsond=headers) +@pytest.fixture +def default_json_response_map(): + return { + "http://localhost:3000/api/session": "session.json", + "http://localhost:3000/api/user/current": "user.json", + "http://localhost:3000/api/collection/?exclude-other-user-collections=false": "collections.json", + "http://localhost:3000/api/collection/root/items?models=dashboard": "collection_dashboards.json", + "http://localhost:3000/api/collection/150/items?models=dashboard": "collection_dashboards.json", + "http://localhost:3000/api/dashboard/10": "dashboard_1.json", + "http://localhost:3000/api/dashboard/20": "dashboard_2.json", + "http://localhost:3000/api/user/1": "user.json", + "http://localhost:3000/api/card": "card.json", + "http://localhost:3000/api/database/1": "bigquery_database.json", + "http://localhost:3000/api/database/2": "postgres_database.json", + "http://localhost:3000/api/card/1": "card_1.json", + "http://localhost:3000/api/card/2": "card_2.json", + "http://localhost:3000/api/table/21": "table_21.json", + "http://localhost:3000/api/card/3": "card_3.json", + } @pytest.fixture @@ -124,16 +162,24 @@ def test_pipeline(pytestconfig, tmp_path): @freeze_time(FROZEN_TIME) -def test_mode_ingest_success(pytestconfig, tmp_path, test_pipeline, mock_datahub_graph): +def test_mode_ingest_success( + pytestconfig, tmp_path, test_pipeline, mock_datahub_graph, default_json_response_map +): with patch( "datahub.ingestion.source.metabase.requests.session", - side_effect=mocked_requests_sucess, + side_effect=MockResponse.build_mocked_requests_sucess( + default_json_response_map + ), ), patch( "datahub.ingestion.source.metabase.requests.post", - side_effect=mocked_requests_session_post, + side_effect=MockResponse.build_mocked_requests_session_post( + default_json_response_map + ), ), patch( "datahub.ingestion.source.metabase.requests.delete", - side_effect=mocked_requests_session_delete, + side_effect=MockResponse.build_mocked_requests_session_delete( + default_json_response_map + ), ), patch( "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph", mock_datahub_graph, @@ -153,16 +199,21 @@ def test_mode_ingest_success(pytestconfig, tmp_path, test_pipeline, mock_datahub @freeze_time(FROZEN_TIME) -def test_stateful_ingestion(test_pipeline, mock_datahub_graph): +def test_stateful_ingestion( + test_pipeline, mock_datahub_graph, default_json_response_map +): + json_response_map = default_json_response_map with patch( "datahub.ingestion.source.metabase.requests.session", - side_effect=mocked_requests_sucess, + side_effect=MockResponse.build_mocked_requests_sucess(json_response_map), ), patch( "datahub.ingestion.source.metabase.requests.post", - side_effect=mocked_requests_session_post, + side_effect=MockResponse.build_mocked_requests_session_post(json_response_map), ), patch( "datahub.ingestion.source.metabase.requests.delete", - side_effect=mocked_requests_session_delete, + side_effect=MockResponse.build_mocked_requests_session_delete( + json_response_map + ), ), patch( "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph", mock_datahub_graph, @@ -176,10 +227,10 @@ def test_stateful_ingestion(test_pipeline, mock_datahub_graph): assert checkpoint1.state # Mock the removal of one of the dashboards - JSON_RESPONSE_MAP[ + json_response_map[ "http://localhost:3000/api/collection/root/items?models=dashboard" ] = "collection_dashboards_deleted_item.json" - JSON_RESPONSE_MAP[ + json_response_map[ "http://localhost:3000/api/collection/150/items?models=dashboard" ] = "collection_dashboards_deleted_item.json" @@ -208,16 +259,22 @@ def test_stateful_ingestion(test_pipeline, mock_datahub_graph): @freeze_time(FROZEN_TIME) -def test_mode_ingest_failure(pytestconfig, tmp_path): +def test_mode_ingest_failure(pytestconfig, tmp_path, default_json_response_map): with patch( "datahub.ingestion.source.metabase.requests.session", - side_effect=mocked_requests_failure, + side_effect=MockResponse.build_mocked_requests_failure( + default_json_response_map + ), ), patch( "datahub.ingestion.source.metabase.requests.post", - side_effect=mocked_requests_session_post, + side_effect=MockResponse.build_mocked_requests_session_post( + default_json_response_map + ), ), patch( "datahub.ingestion.source.metabase.requests.delete", - side_effect=mocked_requests_session_delete, + side_effect=MockResponse.build_mocked_requests_session_delete( + default_json_response_map + ), ): pipeline = Pipeline.create( { @@ -251,26 +308,35 @@ def test_mode_ingest_failure(pytestconfig, tmp_path): def test_9767_query_with_template_tags_does_not_fail( pytestconfig, tmp_path, test_pipeline, mock_datahub_graph ): - baseUrl = "http://localhost:3000/api" - JSON_RESPONSE_MAP[f"{baseUrl}/card/3"] = "issue_9767/card_1.json" - JSON_RESPONSE_MAP[f"{baseUrl}/collection"] = "issue_9767/collection.json" - JSON_RESPONSE_MAP[ - f"{baseUrl}/collection/root/items?models=dashboard" - ] = "issue_9767/collection_dashboard_root.json" - JSON_RESPONSE_MAP[f"{baseUrl}/dashboard/1"] = "issue_9767/dashboard_1.json" - JSON_RESPONSE_MAP[f"{baseUrl}/database/1"] = "issue_9767/database_2.json" - JSON_RESPONSE_MAP[f"{baseUrl}/table/9"] = "issue_9767/table_9.json" + + test_json_response_map = { + f"{baseUrl}/session": "session.json", + f"{baseUrl}/user/current": "issue_9767/user.json", + f"{baseUrl}/user/1": "issue_9767/user_1.json", + f"{baseUrl}/card": "issue_9767/card.json", + f"{baseUrl}/card/1": "issue_9767/card_1.json", + f"{baseUrl}/collection": "issue_9767/collection.json", + f"{baseUrl}/collection/?exclude-other-user-collections=false": "issue_9767/collection.json", + f"{baseUrl}/collection/root/items?models=dashboard": "issue_9767/collection_dashboard_root.json", + f"{baseUrl}/dashboard/1": "issue_9767/dashboard_1.json", + f"{baseUrl}/database/2": "issue_9767/database_2.json", + f"{baseUrl}/table/9": "issue_9767/table_9.json", + } with patch( "datahub.ingestion.source.metabase.requests.session", - side_effect=mocked_requests_sucess, + side_effect=MockResponse.build_mocked_requests_sucess(test_json_response_map), ), patch( "datahub.ingestion.source.metabase.requests.post", - side_effect=mocked_requests_session_post, + side_effect=MockResponse.build_mocked_requests_session_post( + test_json_response_map + ), ), patch( "datahub.ingestion.source.metabase.requests.delete", - side_effect=mocked_requests_session_delete, + side_effect=MockResponse.build_mocked_requests_session_delete( + test_json_response_map + ), ), patch( "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph", mock_datahub_graph, From 10763d0c5487d47874ddd163b730a69aca523a64 Mon Sep 17 00:00:00 2001 From: Paul Rogalinski-Pinter Date: Sat, 18 May 2024 23:14:20 +0200 Subject: [PATCH 09/14] validate ingestion results against golden mces --- .../metabase_mces_golden_issue_9767.json | 148 ++++++++++++++++++ .../integration/metabase/test_metabase.py | 7 + 2 files changed, 155 insertions(+) create mode 100644 metadata-ingestion/tests/integration/metabase/metabase_mces_golden_issue_9767.json diff --git a/metadata-ingestion/tests/integration/metabase/metabase_mces_golden_issue_9767.json b/metadata-ingestion/tests/integration/metabase/metabase_mces_golden_issue_9767.json new file mode 100644 index 00000000000000..8e26740f44380c --- /dev/null +++ b/metadata-ingestion/tests/integration/metabase/metabase_mces_golden_issue_9767.json @@ -0,0 +1,148 @@ +[ +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(metabase,1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "Metrics": "", + "Filters": "", + "Dimensions": "dimension, sum" + }, + "title": "test_question", + "description": "", + "lastModified": { + "created": { + "time": 1716025005214, + "actor": "urn:li:corpuser:test@metabase.local" + }, + "lastModified": { + "time": 1716025005214, + "actor": "urn:li:corpuser:test@metabase.local" + } + }, + "chartUrl": "http://localhost:3000/card/1", + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_data.public.test_table,PROD)" + } + ], + "type": "BAR" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartQuery": { + "rawQuery": "SELECT\n \"public\".\"test_table\".\"dimension\" AS \"dimension\",\n SUM(\"public\".\"test_table\".\"metric\") AS \"sum\"\nFROM\n \"public\".\"test_table\"\nwhere {{dimension}}\nGROUP BY\n \"public\".\"test_table\".\"dimension\"\nORDER BY\n \"public\".\"test_table\".\"dimension\" ASC\n", + "type": "SQL" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:test@metabase.local", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1636614000000, + "runId": "metabase-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(metabase,1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "test_dashboard", + "description": "", + "charts": [ + "urn:li:chart:(metabase,1)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1716025038413, + "actor": "urn:li:corpuser:test@metabase.local" + }, + "lastModified": { + "time": 1716025038413, + "actor": "urn:li:corpuser:test@metabase.local" + } + }, + "dashboardUrl": "http://localhost:3000/dashboard/1" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:test@metabase.local", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1636614000000, + "runId": "metabase-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(metabase,1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1636614000000, + "runId": "metabase-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(metabase,1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1636614000000, + "runId": "metabase-test", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/test_metabase.py b/metadata-ingestion/tests/integration/metabase/test_metabase.py index 932cdfc5fe5ef9..48f3bcb52437d4 100644 --- a/metadata-ingestion/tests/integration/metabase/test_metabase.py +++ b/metadata-ingestion/tests/integration/metabase/test_metabase.py @@ -352,3 +352,10 @@ def test_9767_query_with_template_tags_does_not_fail( assert "Unable to retrieve lineage from query" not in str( report.warnings[warning] ) + + mce_helpers.check_golden_file( + pytestconfig, + output_path=f"{tmp_path}/metabase_mces.json", + golden_path=test_resources_dir / "metabase_mces_golden_issue_9767.json", + ignore_paths=mce_helpers.IGNORE_PATH_TIMESTAMPS, + ) From f325cf684d69671de502d7abf27f4fdde027dbe2 Mon Sep 17 00:00:00 2001 From: Paul Rogalinski-Pinter Date: Sat, 18 May 2024 23:14:53 +0200 Subject: [PATCH 10/14] add test for issue #10380 --- .../metabase/setup/issue_10380/card.json | 2 + .../setup/issue_10380/collection.json | 12 ++ .../collection_dashboard_root.json | 26 +++++ .../setup/issue_10380/dashboard_1.json | 86 +++++++++++++++ .../setup/issue_10380/database_2.json | 103 ++++++++++++++++++ .../metabase/setup/issue_10380/table_9.json | 1 + .../metabase/setup/issue_10380/user.json | 1 + .../metabase/setup/issue_10380/user_1.json | 1 + .../integration/metabase/test_metabase.py | 49 +++++++++ 9 files changed, 281 insertions(+) create mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_10380/card.json create mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_10380/collection.json create mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_10380/collection_dashboard_root.json create mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_10380/dashboard_1.json create mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_10380/database_2.json create mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_10380/table_9.json create mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_10380/user.json create mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_10380/user_1.json diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/card.json b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/card.json new file mode 100644 index 00000000000000..0d4f101c7a37a4 --- /dev/null +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/card.json @@ -0,0 +1,2 @@ +[ +] diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/collection.json b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/collection.json new file mode 100644 index 00000000000000..13ae56d9bf9196 --- /dev/null +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/collection.json @@ -0,0 +1,12 @@ +[ + { + "authority_level": null, + "can_write": true, + "name": "Our analytics", + "effective_ancestors": [], + "effective_location": null, + "parent_id": null, + "id": "root", + "is_personal": false + } +] diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/collection_dashboard_root.json b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/collection_dashboard_root.json new file mode 100644 index 00000000000000..9eab3e12d4002a --- /dev/null +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/collection_dashboard_root.json @@ -0,0 +1,26 @@ +{ + "total": 1, + "data": [ + { + "description": null, + "collection_position": null, + "database_id": null, + "name": "test_dashboard", + "id": 1, + "entity_id": "jvj_rwlh_i0KFlkMUVtb5", + "last-edit-info": { + "id": 1, + "last_name": "user", + "first_name": "test", + "email": "test@metabase.local", + "timestamp": "2024-05-18T09:37:18.413013Z" + }, + "model": "dashboard" + } + ], + "models": [ + "dashboard" + ], + "limit": null, + "offset": null +} diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/dashboard_1.json b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/dashboard_1.json new file mode 100644 index 00000000000000..713ce256169565 --- /dev/null +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/dashboard_1.json @@ -0,0 +1,86 @@ +{ + "description": null, + "archived": false, + "collection_position": null, + "dashcards": [ + { + "size_x": 24, + "dashboard_tab_id": null, + "series": [], + "action_id": null, + "collection_authority_level": null, + "card": { + "query_average_duration": null + }, + "updated_at": "2024-05-18T18:22:19.900158Z", + "col": 0, + "id": 2, + "parameter_mappings": [], + "card_id": null, + "entity_id": "woqw4RKYx7rlOgqp-_UPc", + "visualization_settings": { + "dashcard.background": false, + "virtual_card": { + "name": null, + "dataset_query": {}, + "display": "heading", + "visualization_settings": {}, + "archived": false + }, + "text": "This is a heading" + }, + "size_y": 1, + "dashboard_id": 1, + "created_at": "2024-05-18T18:22:19.900158Z", + "row": 0 + } + ], + "param_values": null, + "initially_published_at": null, + "can_write": true, + "tabs": [], + "enable_embedding": false, + "collection_id": null, + "show_in_getting_started": false, + "name": "test_dashboard", + "width": "fixed", + "caveats": null, + "collection_authority_level": null, + "creator_id": 1, + "updated_at": "2024-05-18T18:22:19.900158Z", + "made_public_by_id": null, + "embedding_params": null, + "cache_ttl": null, + "id": 1, + "position": null, + "entity_id": "jvj_rwlh_i0KFlkMUVtb5", + "param_fields": null, + "last-edit-info": { + "id": 1, + "email": "test@metabase.local", + "first_name": "test", + "last_name": "user", + "timestamp": "2024-05-18T18:22:20.003876Z" + }, + "collection": { + "metabase.models.collection.root/is-root?": true, + "authority_level": null, + "name": "Our analytics", + "is_personal": false, + "id": "root", + "can_write": true + }, + "parameters": [ + { + "name": "Text", + "slug": "text", + "id": "70f9d7fc", + "type": "string/=", + "sectionId": "string" + } + ], + "auto_apply_filters": true, + "created_at": "2024-05-18T09:31:43.161863Z", + "public_uuid": null, + "points_of_interest": null +} diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/database_2.json b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/database_2.json new file mode 100644 index 00000000000000..8a9b3dae7dc1cb --- /dev/null +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/database_2.json @@ -0,0 +1,103 @@ +{ + "description": null, + "features": [ + "index-info", + "full-join", + "basic-aggregations", + "temporal-extract", + "actions/custom", + "now", + "convert-timezone", + "nested-field-columns", + "standard-deviation-aggregations", + "test/jvm-timezone-setting", + "date-arithmetics", + "persist-models", + "actions", + "expression-aggregations", + "percentile-aggregations", + "connection-impersonation", + "foreign-keys", + "table-privileges", + "right-join", + "left-join", + "native-parameters", + "schemas", + "nested-queries", + "expressions", + "uploads", + "set-timezone", + "regex", + "case-sensitivity-string-filter-options", + "binning", + "datetime-diff", + "upload-with-auto-pk", + "inner-join", + "advanced-math-expressions" + ], + "cache_field_values_schedule": "0 0 6 * * ? *", + "timezone": "GMT", + "auto_run_queries": true, + "metadata_sync_schedule": "0 19 * * * ? *", + "name": "test_database", + "settings": null, + "caveats": null, + "can-manage": true, + "creator_id": 1, + "is_full_sync": true, + "updated_at": "2024-05-18T09:30:06.734897Z", + "cache_ttl": null, + "details": { + "ssl": false, + "ssl-key-source": null, + "password": "**MetabasePass**", + "ssl-key-creator-id": null, + "ssl-key-password-source": null, + "port": null, + "advanced-options": false, + "schema-filters-type": "all", + "dbname": "test_data", + "ssl-root-cert-creator-id": null, + "host": "postgres", + "tunnel-enabled": false, + "ssl-key-password-creator-id": null, + "ssl-root-cert-source": null, + "ssl-client-cert-creator-id": null, + "user": "metabase", + "ssl-client-cert-source": null + }, + "is_sample": false, + "id": 2, + "is_on_demand": false, + "schedules": { + "metadata_sync": { + "schedule_minute": 19, + "schedule_day": null, + "schedule_frame": null, + "schedule_hour": null, + "schedule_type": "hourly" + }, + "cache_field_values": { + "schedule_minute": 0, + "schedule_day": null, + "schedule_frame": null, + "schedule_hour": 6, + "schedule_type": "daily" + } + }, + "engine": "postgres", + "initial_sync_status": "complete", + "is_audit": false, + "dbms_version": { + "flavor": "PostgreSQL", + "version": "16.3 (Debian 16.3-1.pgdg120+1)", + "semantic-version": [ + 16, + 3 + ] + }, + "refingerprint": null, + "created_at": "2024-05-18T09:30:06.405812Z", + "points_of_interest": null, + "can_upload": false +} diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/table_9.json b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/table_9.json new file mode 100644 index 00000000000000..458b1a11b505c0 --- /dev/null +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/table_9.json @@ -0,0 +1 @@ +{"description":null,"entity_type":"entity/GenericTable","schema":"public","database_require_filter":null,"db":{"description":null,"features":["index-info","full-join","basic-aggregations","temporal-extract","actions/custom","now","convert-timezone","nested-field-columns","standard-deviation-aggregations","test/jvm-timezone-setting","date-arithmetics","persist-models","actions","expression-aggregations","percentile-aggregations","connection-impersonation","foreign-keys","table-privileges","right-join","left-join","native-parameters","schemas","nested-queries","expressions","uploads","set-timezone","regex","case-sensitivity-string-filter-options","binning","datetime-diff","upload-with-auto-pk","inner-join","advanced-math-expressions"],"cache_field_values_schedule":"0 0 6 * * ? *","timezone":"GMT","auto_run_queries":true,"metadata_sync_schedule":"0 19 * * * ? *","name":"test_database","settings":null,"caveats":null,"creator_id":1,"is_full_sync":true,"updated_at":"2024-05-18T09:30:06.734897Z","cache_ttl":null,"details":{"ssl":false,"password":"**MetabasePass**","port":null,"advanced-options":false,"schema-filters-type":"all","dbname":"test_data","host":"postgres","tunnel-enabled":false,"user":"metabase"},"is_sample":false,"id":2,"is_on_demand":false,"engine":"postgres","initial_sync_status":"complete","is_audit":false,"dbms_version":{"flavor":"PostgreSQL","version":"16.3 (Debian 16.3-1.pgdg120+1)","semantic-version":[16,3]},"refingerprint":null,"created_at":"2024-05-18T09:30:06.405812Z","points_of_interest":null},"show_in_getting_started":false,"name":"test_table","caveats":null,"updated_at":"2024-05-18T09:30:06.783792Z","pk_field":74,"active":true,"id":9,"db_id":2,"visibility_type":null,"field_order":"database","is_upload":false,"initial_sync_status":"complete","display_name":"Test Table","created_at":"2024-05-18T09:30:06.660148Z","points_of_interest":null} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/user.json b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/user.json new file mode 100644 index 00000000000000..c1adfc798982c8 --- /dev/null +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/user.json @@ -0,0 +1 @@ +{"data":[{"email":"test@metabase.local","first_name":"test","locale":null,"last_login":"2024-05-18T09:30:06.609263Z","is_active":true,"is_qbnewb":false,"updated_at":"2024-05-18T09:31:55.368683Z","group_ids":[1,2],"is_superuser":true,"login_attributes":null,"id":1,"last_name":"user","date_joined":"2024-05-18T09:30:06.405812Z","sso_source":null,"personal_collection_id":1,"common_name":"test user"}],"total":1,"limit":null,"offset":null} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/user_1.json b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/user_1.json new file mode 100644 index 00000000000000..bccf0a9cd6fb9a --- /dev/null +++ b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/user_1.json @@ -0,0 +1 @@ +{"email":"test@metabase.local","first_name":"test","locale":null,"last_login":"2024-05-18T09:30:06.609263Z","is_active":true,"user_group_memberships":[{"id":1},{"id":2}],"is_qbnewb":false,"updated_at":"2024-05-18T09:31:55.368683Z","is_superuser":true,"login_attributes":null,"id":1,"last_name":"user","date_joined":"2024-05-18T09:30:06.405812Z","sso_source":null,"common_name":"test user"} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/test_metabase.py b/metadata-ingestion/tests/integration/metabase/test_metabase.py index 48f3bcb52437d4..46419c78e43f95 100644 --- a/metadata-ingestion/tests/integration/metabase/test_metabase.py +++ b/metadata-ingestion/tests/integration/metabase/test_metabase.py @@ -359,3 +359,52 @@ def test_9767_query_with_template_tags_does_not_fail( golden_path=test_resources_dir / "metabase_mces_golden_issue_9767.json", ignore_paths=mce_helpers.IGNORE_PATH_TIMESTAMPS, ) + + +@freeze_time(FROZEN_TIME) +def test_10380_ingestion_of_dashboard_with_virtual_cards_does_not_fail( + pytestconfig, tmp_path, test_pipeline, mock_datahub_graph +): + baseUrl = "http://localhost:3000/api" + test_json_response_map = { + f"{baseUrl}/session": "session.json", + f"{baseUrl}/user/current": "issue_10380/user.json", + f"{baseUrl}/user/1": "issue_10380/user_1.json", + f"{baseUrl}/card": "issue_10380/card.json", + f"{baseUrl}/card/1": "issue_10380/card_1.json", + f"{baseUrl}/collection": "issue_10380/collection.json", + f"{baseUrl}/collection/?exclude-other-user-collections=false": "issue_10380/collection.json", + f"{baseUrl}/collection/root/items?models=dashboard": "/issue_10380/collection_dashboard_root.json", + f"{baseUrl}/dashboard/1": "issue_10380/dashboard_1.json", + f"{baseUrl}/database/1": "issue_10380/database_2.json", + f"{baseUrl}/table/9": "issue_10380/table_9.json", + } + with patch( + "datahub.ingestion.source.metabase.requests.session", + side_effect=MockResponse.build_mocked_requests_sucess(test_json_response_map), + ), patch( + "datahub.ingestion.source.metabase.requests.post", + side_effect=MockResponse.build_mocked_requests_session_post( + test_json_response_map + ), + ), patch( + "datahub.ingestion.source.metabase.requests.delete", + side_effect=MockResponse.build_mocked_requests_session_delete( + test_json_response_map + ), + ), patch( + "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph", + mock_datahub_graph, + ) as mock_checkpoint: + mock_checkpoint.return_value = mock_datahub_graph + + pipeline = Pipeline.create(test_pipeline) + pipeline.run() + pipeline.raise_from_status(raise_warnings=True) + + mce_helpers.check_golden_file( + pytestconfig, + output_path=f"{tmp_path}/metabase_mces.json", + golden_path=test_resources_dir / "metabase_mces_golden_issue_10380.json", + ignore_paths=mce_helpers.IGNORE_PATH_TIMESTAMPS, + ) From 44359319939f83051d1e5f702e20bdb463f03969 Mon Sep 17 00:00:00 2001 From: Paul Rogalinski-Pinter Date: Tue, 21 May 2024 00:00:45 +0200 Subject: [PATCH 11/14] move virtual card test into main test model --- .../metabase/setup/dashboard_1.json | 31 ++++++ .../metabase/setup/dashboard_2.json | 31 ++++++ .../metabase/setup/issue_10380/card.json | 2 - .../setup/issue_10380/collection.json | 12 -- .../collection_dashboard_root.json | 26 ----- .../setup/issue_10380/dashboard_1.json | 86 --------------- .../setup/issue_10380/database_2.json | 103 ------------------ .../metabase/setup/issue_10380/table_9.json | 1 - .../metabase/setup/issue_10380/user.json | 1 - .../metabase/setup/issue_10380/user_1.json | 1 - .../integration/metabase/test_metabase.py | 49 --------- 11 files changed, 62 insertions(+), 281 deletions(-) delete mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_10380/card.json delete mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_10380/collection.json delete mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_10380/collection_dashboard_root.json delete mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_10380/dashboard_1.json delete mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_10380/database_2.json delete mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_10380/table_9.json delete mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_10380/user.json delete mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_10380/user_1.json diff --git a/metadata-ingestion/tests/integration/metabase/setup/dashboard_1.json b/metadata-ingestion/tests/integration/metabase/setup/dashboard_1.json index f58a1079c3620b..aa1197a0670bf3 100644 --- a/metadata-ingestion/tests/integration/metabase/setup/dashboard_1.json +++ b/metadata-ingestion/tests/integration/metabase/setup/dashboard_1.json @@ -3,6 +3,37 @@ "archived": false, "collection_position": null, "dashcards": [ + { + "size_x": 24, + "dashboard_tab_id": null, + "series": [], + "action_id": null, + "collection_authority_level": null, + "card": { + "query_average_duration": null + }, + "updated_at": "2024-05-18T18:22:19.900158Z", + "col": 0, + "id": 2, + "parameter_mappings": [], + "card_id": null, + "entity_id": "woqw4RKYx7rlOgqp-_UPc", + "visualization_settings": { + "dashcard.background": false, + "virtual_card": { + "name": null, + "dataset_query": {}, + "display": "heading", + "visualization_settings": {}, + "archived": false + }, + "text": "This is a virtual card and should be ignored by the ingestion." + }, + "size_y": 1, + "dashboard_id": 1, + "created_at": "2024-05-18T18:22:19.900158Z", + "row": 0 + }, { "size_x": 12, "dashboard_tab_id": null, diff --git a/metadata-ingestion/tests/integration/metabase/setup/dashboard_2.json b/metadata-ingestion/tests/integration/metabase/setup/dashboard_2.json index 2f9beaccc1e187..be2b0f23c39688 100644 --- a/metadata-ingestion/tests/integration/metabase/setup/dashboard_2.json +++ b/metadata-ingestion/tests/integration/metabase/setup/dashboard_2.json @@ -3,6 +3,37 @@ "archived": false, "collection_position": null, "dashcards": [ + { + "size_x": 24, + "dashboard_tab_id": null, + "series": [], + "action_id": null, + "collection_authority_level": null, + "card": { + "query_average_duration": null + }, + "updated_at": "2024-05-18T18:22:19.900158Z", + "col": 0, + "id": 2, + "parameter_mappings": [], + "card_id": null, + "entity_id": "woqw4RKYx7rlOgqp-_UPc", + "visualization_settings": { + "dashcard.background": false, + "virtual_card": { + "name": null, + "dataset_query": {}, + "display": "heading", + "visualization_settings": {}, + "archived": false + }, + "text": "This is a virtual card and should be ignored by the ingestion." + }, + "size_y": 1, + "dashboard_id": 1, + "created_at": "2024-05-18T18:22:19.900158Z", + "row": 0 + }, { "size_x": 12, "dashboard_tab_id": null, diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/card.json b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/card.json deleted file mode 100644 index 0d4f101c7a37a4..00000000000000 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/card.json +++ /dev/null @@ -1,2 +0,0 @@ -[ -] diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/collection.json b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/collection.json deleted file mode 100644 index 13ae56d9bf9196..00000000000000 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/collection.json +++ /dev/null @@ -1,12 +0,0 @@ -[ - { - "authority_level": null, - "can_write": true, - "name": "Our analytics", - "effective_ancestors": [], - "effective_location": null, - "parent_id": null, - "id": "root", - "is_personal": false - } -] diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/collection_dashboard_root.json b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/collection_dashboard_root.json deleted file mode 100644 index 9eab3e12d4002a..00000000000000 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/collection_dashboard_root.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "total": 1, - "data": [ - { - "description": null, - "collection_position": null, - "database_id": null, - "name": "test_dashboard", - "id": 1, - "entity_id": "jvj_rwlh_i0KFlkMUVtb5", - "last-edit-info": { - "id": 1, - "last_name": "user", - "first_name": "test", - "email": "test@metabase.local", - "timestamp": "2024-05-18T09:37:18.413013Z" - }, - "model": "dashboard" - } - ], - "models": [ - "dashboard" - ], - "limit": null, - "offset": null -} diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/dashboard_1.json b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/dashboard_1.json deleted file mode 100644 index 713ce256169565..00000000000000 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/dashboard_1.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "description": null, - "archived": false, - "collection_position": null, - "dashcards": [ - { - "size_x": 24, - "dashboard_tab_id": null, - "series": [], - "action_id": null, - "collection_authority_level": null, - "card": { - "query_average_duration": null - }, - "updated_at": "2024-05-18T18:22:19.900158Z", - "col": 0, - "id": 2, - "parameter_mappings": [], - "card_id": null, - "entity_id": "woqw4RKYx7rlOgqp-_UPc", - "visualization_settings": { - "dashcard.background": false, - "virtual_card": { - "name": null, - "dataset_query": {}, - "display": "heading", - "visualization_settings": {}, - "archived": false - }, - "text": "This is a heading" - }, - "size_y": 1, - "dashboard_id": 1, - "created_at": "2024-05-18T18:22:19.900158Z", - "row": 0 - } - ], - "param_values": null, - "initially_published_at": null, - "can_write": true, - "tabs": [], - "enable_embedding": false, - "collection_id": null, - "show_in_getting_started": false, - "name": "test_dashboard", - "width": "fixed", - "caveats": null, - "collection_authority_level": null, - "creator_id": 1, - "updated_at": "2024-05-18T18:22:19.900158Z", - "made_public_by_id": null, - "embedding_params": null, - "cache_ttl": null, - "id": 1, - "position": null, - "entity_id": "jvj_rwlh_i0KFlkMUVtb5", - "param_fields": null, - "last-edit-info": { - "id": 1, - "email": "test@metabase.local", - "first_name": "test", - "last_name": "user", - "timestamp": "2024-05-18T18:22:20.003876Z" - }, - "collection": { - "metabase.models.collection.root/is-root?": true, - "authority_level": null, - "name": "Our analytics", - "is_personal": false, - "id": "root", - "can_write": true - }, - "parameters": [ - { - "name": "Text", - "slug": "text", - "id": "70f9d7fc", - "type": "string/=", - "sectionId": "string" - } - ], - "auto_apply_filters": true, - "created_at": "2024-05-18T09:31:43.161863Z", - "public_uuid": null, - "points_of_interest": null -} diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/database_2.json b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/database_2.json deleted file mode 100644 index 8a9b3dae7dc1cb..00000000000000 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/database_2.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "description": null, - "features": [ - "index-info", - "full-join", - "basic-aggregations", - "temporal-extract", - "actions/custom", - "now", - "convert-timezone", - "nested-field-columns", - "standard-deviation-aggregations", - "test/jvm-timezone-setting", - "date-arithmetics", - "persist-models", - "actions", - "expression-aggregations", - "percentile-aggregations", - "connection-impersonation", - "foreign-keys", - "table-privileges", - "right-join", - "left-join", - "native-parameters", - "schemas", - "nested-queries", - "expressions", - "uploads", - "set-timezone", - "regex", - "case-sensitivity-string-filter-options", - "binning", - "datetime-diff", - "upload-with-auto-pk", - "inner-join", - "advanced-math-expressions" - ], - "cache_field_values_schedule": "0 0 6 * * ? *", - "timezone": "GMT", - "auto_run_queries": true, - "metadata_sync_schedule": "0 19 * * * ? *", - "name": "test_database", - "settings": null, - "caveats": null, - "can-manage": true, - "creator_id": 1, - "is_full_sync": true, - "updated_at": "2024-05-18T09:30:06.734897Z", - "cache_ttl": null, - "details": { - "ssl": false, - "ssl-key-source": null, - "password": "**MetabasePass**", - "ssl-key-creator-id": null, - "ssl-key-password-source": null, - "port": null, - "advanced-options": false, - "schema-filters-type": "all", - "dbname": "test_data", - "ssl-root-cert-creator-id": null, - "host": "postgres", - "tunnel-enabled": false, - "ssl-key-password-creator-id": null, - "ssl-root-cert-source": null, - "ssl-client-cert-creator-id": null, - "user": "metabase", - "ssl-client-cert-source": null - }, - "is_sample": false, - "id": 2, - "is_on_demand": false, - "schedules": { - "metadata_sync": { - "schedule_minute": 19, - "schedule_day": null, - "schedule_frame": null, - "schedule_hour": null, - "schedule_type": "hourly" - }, - "cache_field_values": { - "schedule_minute": 0, - "schedule_day": null, - "schedule_frame": null, - "schedule_hour": 6, - "schedule_type": "daily" - } - }, - "engine": "postgres", - "initial_sync_status": "complete", - "is_audit": false, - "dbms_version": { - "flavor": "PostgreSQL", - "version": "16.3 (Debian 16.3-1.pgdg120+1)", - "semantic-version": [ - 16, - 3 - ] - }, - "refingerprint": null, - "created_at": "2024-05-18T09:30:06.405812Z", - "points_of_interest": null, - "can_upload": false -} diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/table_9.json b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/table_9.json deleted file mode 100644 index 458b1a11b505c0..00000000000000 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/table_9.json +++ /dev/null @@ -1 +0,0 @@ -{"description":null,"entity_type":"entity/GenericTable","schema":"public","database_require_filter":null,"db":{"description":null,"features":["index-info","full-join","basic-aggregations","temporal-extract","actions/custom","now","convert-timezone","nested-field-columns","standard-deviation-aggregations","test/jvm-timezone-setting","date-arithmetics","persist-models","actions","expression-aggregations","percentile-aggregations","connection-impersonation","foreign-keys","table-privileges","right-join","left-join","native-parameters","schemas","nested-queries","expressions","uploads","set-timezone","regex","case-sensitivity-string-filter-options","binning","datetime-diff","upload-with-auto-pk","inner-join","advanced-math-expressions"],"cache_field_values_schedule":"0 0 6 * * ? *","timezone":"GMT","auto_run_queries":true,"metadata_sync_schedule":"0 19 * * * ? *","name":"test_database","settings":null,"caveats":null,"creator_id":1,"is_full_sync":true,"updated_at":"2024-05-18T09:30:06.734897Z","cache_ttl":null,"details":{"ssl":false,"password":"**MetabasePass**","port":null,"advanced-options":false,"schema-filters-type":"all","dbname":"test_data","host":"postgres","tunnel-enabled":false,"user":"metabase"},"is_sample":false,"id":2,"is_on_demand":false,"engine":"postgres","initial_sync_status":"complete","is_audit":false,"dbms_version":{"flavor":"PostgreSQL","version":"16.3 (Debian 16.3-1.pgdg120+1)","semantic-version":[16,3]},"refingerprint":null,"created_at":"2024-05-18T09:30:06.405812Z","points_of_interest":null},"show_in_getting_started":false,"name":"test_table","caveats":null,"updated_at":"2024-05-18T09:30:06.783792Z","pk_field":74,"active":true,"id":9,"db_id":2,"visibility_type":null,"field_order":"database","is_upload":false,"initial_sync_status":"complete","display_name":"Test Table","created_at":"2024-05-18T09:30:06.660148Z","points_of_interest":null} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/user.json b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/user.json deleted file mode 100644 index c1adfc798982c8..00000000000000 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/user.json +++ /dev/null @@ -1 +0,0 @@ -{"data":[{"email":"test@metabase.local","first_name":"test","locale":null,"last_login":"2024-05-18T09:30:06.609263Z","is_active":true,"is_qbnewb":false,"updated_at":"2024-05-18T09:31:55.368683Z","group_ids":[1,2],"is_superuser":true,"login_attributes":null,"id":1,"last_name":"user","date_joined":"2024-05-18T09:30:06.405812Z","sso_source":null,"personal_collection_id":1,"common_name":"test user"}],"total":1,"limit":null,"offset":null} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/user_1.json b/metadata-ingestion/tests/integration/metabase/setup/issue_10380/user_1.json deleted file mode 100644 index bccf0a9cd6fb9a..00000000000000 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_10380/user_1.json +++ /dev/null @@ -1 +0,0 @@ -{"email":"test@metabase.local","first_name":"test","locale":null,"last_login":"2024-05-18T09:30:06.609263Z","is_active":true,"user_group_memberships":[{"id":1},{"id":2}],"is_qbnewb":false,"updated_at":"2024-05-18T09:31:55.368683Z","is_superuser":true,"login_attributes":null,"id":1,"last_name":"user","date_joined":"2024-05-18T09:30:06.405812Z","sso_source":null,"common_name":"test user"} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/test_metabase.py b/metadata-ingestion/tests/integration/metabase/test_metabase.py index 46419c78e43f95..48f3bcb52437d4 100644 --- a/metadata-ingestion/tests/integration/metabase/test_metabase.py +++ b/metadata-ingestion/tests/integration/metabase/test_metabase.py @@ -359,52 +359,3 @@ def test_9767_query_with_template_tags_does_not_fail( golden_path=test_resources_dir / "metabase_mces_golden_issue_9767.json", ignore_paths=mce_helpers.IGNORE_PATH_TIMESTAMPS, ) - - -@freeze_time(FROZEN_TIME) -def test_10380_ingestion_of_dashboard_with_virtual_cards_does_not_fail( - pytestconfig, tmp_path, test_pipeline, mock_datahub_graph -): - baseUrl = "http://localhost:3000/api" - test_json_response_map = { - f"{baseUrl}/session": "session.json", - f"{baseUrl}/user/current": "issue_10380/user.json", - f"{baseUrl}/user/1": "issue_10380/user_1.json", - f"{baseUrl}/card": "issue_10380/card.json", - f"{baseUrl}/card/1": "issue_10380/card_1.json", - f"{baseUrl}/collection": "issue_10380/collection.json", - f"{baseUrl}/collection/?exclude-other-user-collections=false": "issue_10380/collection.json", - f"{baseUrl}/collection/root/items?models=dashboard": "/issue_10380/collection_dashboard_root.json", - f"{baseUrl}/dashboard/1": "issue_10380/dashboard_1.json", - f"{baseUrl}/database/1": "issue_10380/database_2.json", - f"{baseUrl}/table/9": "issue_10380/table_9.json", - } - with patch( - "datahub.ingestion.source.metabase.requests.session", - side_effect=MockResponse.build_mocked_requests_sucess(test_json_response_map), - ), patch( - "datahub.ingestion.source.metabase.requests.post", - side_effect=MockResponse.build_mocked_requests_session_post( - test_json_response_map - ), - ), patch( - "datahub.ingestion.source.metabase.requests.delete", - side_effect=MockResponse.build_mocked_requests_session_delete( - test_json_response_map - ), - ), patch( - "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph", - mock_datahub_graph, - ) as mock_checkpoint: - mock_checkpoint.return_value = mock_datahub_graph - - pipeline = Pipeline.create(test_pipeline) - pipeline.run() - pipeline.raise_from_status(raise_warnings=True) - - mce_helpers.check_golden_file( - pytestconfig, - output_path=f"{tmp_path}/metabase_mces.json", - golden_path=test_resources_dir / "metabase_mces_golden_issue_10380.json", - ignore_paths=mce_helpers.IGNORE_PATH_TIMESTAMPS, - ) From f442a2773e237c5037e08b710c3c7fe5b5bc3089 Mon Sep 17 00:00:00 2001 From: Paul Rogalinski-Pinter Date: Tue, 21 May 2024 00:28:21 +0200 Subject: [PATCH 12/14] only test query stripping --- .../src/datahub/ingestion/source/metabase.py | 4 +- .../metabase_mces_golden_issue_9767.json | 148 ------------------ .../metabase/setup/issue_9767/card.json | 1 - .../metabase/setup/issue_9767/card_1.json | 1 - .../metabase/setup/issue_9767/collection.json | 12 -- .../issue_9767/collection_dashboard_root.json | 1 - .../setup/issue_9767/dashboard_1.json | 1 - .../metabase/setup/issue_9767/database_2.json | 1 - .../metabase/setup/issue_9767/table_9.json | 1 - .../metabase/setup/issue_9767/user.json | 1 - .../metabase/setup/issue_9767/user_1.json | 1 - 11 files changed, 2 insertions(+), 170 deletions(-) delete mode 100644 metadata-ingestion/tests/integration/metabase/metabase_mces_golden_issue_9767.json delete mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/card.json delete mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/card_1.json delete mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection.json delete mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection_dashboard_root.json delete mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/dashboard_1.json delete mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/database_2.json delete mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/table_9.json delete mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/user.json delete mode 100644 metadata-ingestion/tests/integration/metabase/setup/issue_9767/user_1.json diff --git a/metadata-ingestion/src/datahub/ingestion/source/metabase.py b/metadata-ingestion/src/datahub/ingestion/source/metabase.py index cad1b06963f10b..12a76ff7b33ff7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/metabase.py +++ b/metadata-ingestion/src/datahub/ingestion/source/metabase.py @@ -632,10 +632,10 @@ def strip_template_expressions(raw_query: str) -> str: """ # drop [[ WHERE {{FILTER}} ]] - query_patched = re.sub(r"\[\[.+\]\]", r" ", raw_query) + query_patched = re.sub(r"\[\[.+?\]\]", r" ", raw_query) # replace {{FILTER}} with 1 - query_patched = re.sub(r"\{\{.+\}\}", r"1", query_patched) + query_patched = re.sub(r"\{\{.+?\}\}", r"1", query_patched) return query_patched @lru_cache(maxsize=None) diff --git a/metadata-ingestion/tests/integration/metabase/metabase_mces_golden_issue_9767.json b/metadata-ingestion/tests/integration/metabase/metabase_mces_golden_issue_9767.json deleted file mode 100644 index 8e26740f44380c..00000000000000 --- a/metadata-ingestion/tests/integration/metabase/metabase_mces_golden_issue_9767.json +++ /dev/null @@ -1,148 +0,0 @@ -[ -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(metabase,1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Metrics": "", - "Filters": "", - "Dimensions": "dimension, sum" - }, - "title": "test_question", - "description": "", - "lastModified": { - "created": { - "time": 1716025005214, - "actor": "urn:li:corpuser:test@metabase.local" - }, - "lastModified": { - "time": 1716025005214, - "actor": "urn:li:corpuser:test@metabase.local" - } - }, - "chartUrl": "http://localhost:3000/card/1", - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_data.public.test_table,PROD)" - } - ], - "type": "BAR" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartQuery": { - "rawQuery": "SELECT\n \"public\".\"test_table\".\"dimension\" AS \"dimension\",\n SUM(\"public\".\"test_table\".\"metric\") AS \"sum\"\nFROM\n \"public\".\"test_table\"\nwhere {{dimension}}\nGROUP BY\n \"public\".\"test_table\".\"dimension\"\nORDER BY\n \"public\".\"test_table\".\"dimension\" ASC\n", - "type": "SQL" - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:test@metabase.local", - "type": "DATAOWNER" - } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1636614000000, - "runId": "metabase-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { - "urn": "urn:li:dashboard:(metabase,1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": {}, - "title": "test_dashboard", - "description": "", - "charts": [ - "urn:li:chart:(metabase,1)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 1716025038413, - "actor": "urn:li:corpuser:test@metabase.local" - }, - "lastModified": { - "time": 1716025038413, - "actor": "urn:li:corpuser:test@metabase.local" - } - }, - "dashboardUrl": "http://localhost:3000/dashboard/1" - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:test@metabase.local", - "type": "DATAOWNER" - } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1636614000000, - "runId": "metabase-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(metabase,1)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1636614000000, - "runId": "metabase-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(metabase,1)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1636614000000, - "runId": "metabase-test", - "lastRunId": "no-run-id-provided" - } -} -] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/card.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/card.json deleted file mode 100644 index 2cd697b0291f26..00000000000000 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/card.json +++ /dev/null @@ -1 +0,0 @@ -[{"description":null,"archived":false,"collection_position":null,"table_id":9,"result_metadata":[{"display_name":"dimension","field_ref":["field","dimension",{"base-type":"type/Text"}],"name":"dimension","base_type":"type/Text","effective_type":"type/Text","semantic_type":null,"fingerprint":{"global":{"distinct-count":2,"nil%":0.0},"type":{"type/Text":{"percent-json":0.0,"percent-url":0.0,"percent-email":0.0,"percent-state":0.0,"average-length":1.0}}}},{"display_name":"sum","field_ref":["field","sum",{"base-type":"type/BigInteger"}],"name":"sum","base_type":"type/BigInteger","effective_type":"type/BigInteger","semantic_type":null,"fingerprint":{"global":{"distinct-count":1,"nil%":0.0},"type":{"type/Number":{"min":300.0,"q1":300.0,"q3":300.0,"max":300.0,"sd":0.0,"avg":300.0}}}}],"creator":{"email":"test@metabase.local","first_name":"test","last_login":"2024-05-18T09:30:06.609263Z","is_qbnewb":false,"is_superuser":true,"id":1,"last_name":"user","date_joined":"2024-05-18T09:30:06.405812Z","common_name":"test user"},"initially_published_at":null,"database_id":2,"enable_embedding":false,"collection_id":null,"query_type":"native","name":"test_question","type":"question","creator_id":1,"updated_at":"2024-05-18T18:22:04.437454Z","made_public_by_id":null,"embedding_params":null,"cache_ttl":null,"dataset_query":{"database":2,"type":"native","native":{"collection":"test_table","template-tags":{"dimension":{"type":"dimension","name":"dimension","id":"60f27c21-36ca-4a26-a718-1d9898348ea3","display-name":"Dimension","default":null,"dimension":["field",72,{"base-type":"type/Text"}],"widget-type":"string/=","options":null}},"query":"SELECT\n \"public\".\"test_table\".\"dimension\" AS \"dimension\",\n SUM(\"public\".\"test_table\".\"metric\") AS \"sum\"\nFROM\n \"public\".\"test_table\"\nwhere {{dimension}}\nGROUP BY\n \"public\".\"test_table\".\"dimension\"\nORDER BY\n \"public\".\"test_table\".\"dimension\" ASC\n"}},"id":1,"parameter_mappings":[],"display":"bar","entity_id":"qxLTXxQvaLpflMvpAwqMw","collection_preview":true,"last-edit-info":{"id":1,"email":"test@metabase.local","first_name":"test","last_name":"user","timestamp":"2024-05-18T09:36:45.214507Z"},"visualization_settings":{"graph.dimensions":["dimension"],"graph.metrics":["sum"]},"collection":null,"metabase_version":"v0.49.11 (b894f2d)","parameters":[{"id":"60f27c21-36ca-4a26-a718-1d9898348ea3","type":"string/=","target":["dimension",["template-tag","dimension"]],"name":"Dimension","slug":"dimension"}],"dataset":false,"created_at":"2024-05-18T09:31:36.591212Z","public_uuid":null}] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/card_1.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/card_1.json deleted file mode 100644 index 6c00a1b17ecb25..00000000000000 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/card_1.json +++ /dev/null @@ -1 +0,0 @@ -{"description":null,"archived":false,"collection_position":null,"table_id":9,"result_metadata":[{"display_name":"dimension","field_ref":["field","dimension",{"base-type":"type/Text"}],"name":"dimension","base_type":"type/Text","effective_type":"type/Text","semantic_type":null,"fingerprint":{"global":{"distinct-count":2,"nil%":0.0},"type":{"type/Text":{"percent-json":0.0,"percent-url":0.0,"percent-email":0.0,"percent-state":0.0,"average-length":1.0}}}},{"display_name":"sum","field_ref":["field","sum",{"base-type":"type/BigInteger"}],"name":"sum","base_type":"type/BigInteger","effective_type":"type/BigInteger","semantic_type":null,"fingerprint":{"global":{"distinct-count":1,"nil%":0.0},"type":{"type/Number":{"min":300.0,"q1":300.0,"q3":300.0,"max":300.0,"sd":0.0,"avg":300.0}}}}],"creator":{"email":"test@metabase.local","first_name":"test","last_login":"2024-05-18T09:30:06.609263Z","is_qbnewb":false,"is_superuser":true,"id":1,"last_name":"user","date_joined":"2024-05-18T09:30:06.405812Z","common_name":"test user"},"initially_published_at":null,"can_write":true,"database_id":2,"enable_embedding":false,"collection_id":null,"query_type":"native","name":"test_question","last_query_start":"2024-05-18T09:37:22.713517Z","dashboard_count":1,"type":"question","average_query_time":27.1111111111111111,"creator_id":1,"moderation_reviews":[],"updated_at":"2024-05-18T09:37:22.729184Z","made_public_by_id":null,"embedding_params":null,"cache_ttl":null,"dataset_query":{"database":2,"type":"native","native":{"collection":"test_table","template-tags":{"dimension":{"type":"dimension","name":"dimension","id":"60f27c21-36ca-4a26-a718-1d9898348ea3","display-name":"Dimension","default":null,"dimension":["field",72,{"base-type":"type/Text"}],"widget-type":"string/=","options":null}},"query":"SELECT\n \"public\".\"test_table\".\"dimension\" AS \"dimension\",\n SUM(\"public\".\"test_table\".\"metric\") AS \"sum\"\nFROM\n \"public\".\"test_table\"\nwhere {{dimension}}\nGROUP BY\n \"public\".\"test_table\".\"dimension\"\nORDER BY\n \"public\".\"test_table\".\"dimension\" ASC\n"}},"id":1,"parameter_mappings":[],"display":"bar","entity_id":"qxLTXxQvaLpflMvpAwqMw","collection_preview":true,"last-edit-info":{"id":1,"email":"test@metabase.local","first_name":"test","last_name":"user","timestamp":"2024-05-18T09:36:45.214507Z"},"visualization_settings":{"graph.dimensions":["dimension"],"graph.metrics":["sum"]},"collection":{"metabase.models.collection.root/is-root?":true,"authority_level":null,"name":"Our analytics","is_personal":false,"id":"root","can_write":true},"metabase_version":"v0.49.11 (b894f2d)","parameters":[{"id":"60f27c21-36ca-4a26-a718-1d9898348ea3","type":"string/=","target":["dimension",["template-tag","dimension"]],"name":"Dimension","slug":"dimension"}],"dataset":false,"created_at":"2024-05-18T09:31:36.591212Z","parameter_usage_count":0,"public_uuid":null} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection.json deleted file mode 100644 index 13ae56d9bf9196..00000000000000 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection.json +++ /dev/null @@ -1,12 +0,0 @@ -[ - { - "authority_level": null, - "can_write": true, - "name": "Our analytics", - "effective_ancestors": [], - "effective_location": null, - "parent_id": null, - "id": "root", - "is_personal": false - } -] diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection_dashboard_root.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection_dashboard_root.json deleted file mode 100644 index 9bf6c1a5489f7e..00000000000000 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/collection_dashboard_root.json +++ /dev/null @@ -1 +0,0 @@ -{"total":1,"data":[{"description":null,"collection_position":null,"database_id":null,"name":"test_dashboard","id":1,"entity_id":"jvj_rwlh_i0KFlkMUVtb5","last-edit-info":{"id":1,"last_name":"user","first_name":"test","email":"test@metabase.local","timestamp":"2024-05-18T09:37:18.413013Z"},"model":"dashboard"}],"models":["dashboard"],"limit":null,"offset":null} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/dashboard_1.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/dashboard_1.json deleted file mode 100644 index d80ced2071fb3a..00000000000000 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/dashboard_1.json +++ /dev/null @@ -1 +0,0 @@ -{"description":null,"archived":false,"collection_position":null,"dashcards":[{"size_x":12,"dashboard_tab_id":null,"series":[],"action_id":null,"collection_authority_level":null,"card":{"description":null,"archived":false,"collection_position":null,"table_id":9,"result_metadata":[{"display_name":"dimension","field_ref":["field","dimension",{"base-type":"type/Text"}],"name":"dimension","base_type":"type/Text","effective_type":"type/Text","semantic_type":null,"fingerprint":{"global":{"distinct-count":2,"nil%":0.0},"type":{"type/Text":{"percent-json":0.0,"percent-url":0.0,"percent-email":0.0,"percent-state":0.0,"average-length":1.0}}}},{"display_name":"sum","field_ref":["field","sum",{"base-type":"type/BigInteger"}],"name":"sum","base_type":"type/BigInteger","effective_type":"type/BigInteger","semantic_type":null,"fingerprint":{"global":{"distinct-count":1,"nil%":0.0},"type":{"type/Number":{"min":300.0,"q1":300.0,"q3":300.0,"max":300.0,"sd":0.0,"avg":300.0}}}}],"initially_published_at":null,"can_write":true,"database_id":2,"enable_embedding":false,"collection_id":null,"query_type":"native","name":"test_question","type":"question","query_average_duration":16,"creator_id":1,"moderation_reviews":[],"updated_at":"2024-05-18T09:37:22.729184Z","made_public_by_id":null,"embedding_params":null,"cache_ttl":null,"dataset_query":{"database":2,"type":"native","native":{"collection":"test_table","template-tags":{"dimension":{"type":"dimension","name":"dimension","id":"60f27c21-36ca-4a26-a718-1d9898348ea3","display-name":"Dimension","default":null,"dimension":["field",72,{"base-type":"type/Text"}],"widget-type":"string/=","options":null}},"query":"SELECT\n \"public\".\"test_table\".\"dimension\" AS \"dimension\",\n SUM(\"public\".\"test_table\".\"metric\") AS \"sum\"\nFROM\n \"public\".\"test_table\"\nwhere {{dimension}}\nGROUP BY\n \"public\".\"test_table\".\"dimension\"\nORDER BY\n \"public\".\"test_table\".\"dimension\" ASC\n"}},"id":1,"parameter_mappings":[],"display":"bar","entity_id":"qxLTXxQvaLpflMvpAwqMw","collection_preview":true,"visualization_settings":{"graph.dimensions":["dimension"],"graph.metrics":["sum"]},"metabase_version":"v0.49.11 (b894f2d)","parameters":[{"id":"60f27c21-36ca-4a26-a718-1d9898348ea3","type":"string/=","target":["dimension",["template-tag","dimension"]],"name":"Dimension","slug":"dimension"}],"dataset":false,"created_at":"2024-05-18T09:31:36.591212Z","public_uuid":null},"updated_at":"2024-05-18T09:37:18.369753Z","col":0,"id":1,"parameter_mappings":[{"parameter_id":"70f9d7fc","card_id":1,"target":["dimension",["template-tag","dimension"]]}],"card_id":1,"entity_id":"1F8zX4QzW5W8kf7R4exVr","visualization_settings":{},"size_y":6,"dashboard_id":1,"created_at":"2024-05-18T09:31:52.570882Z","row":0}],"param_values":{"72":{"field_id":72,"human_readable_values":[],"values":["A","B"]}},"initially_published_at":null,"can_write":true,"tabs":[],"enable_embedding":false,"collection_id":null,"show_in_getting_started":false,"name":"test_dashboard","width":"fixed","caveats":null,"collection_authority_level":null,"creator_id":1,"updated_at":"2024-05-18T09:37:18.369753Z","made_public_by_id":null,"embedding_params":null,"cache_ttl":null,"id":1,"position":null,"entity_id":"jvj_rwlh_i0KFlkMUVtb5","param_fields":{"72":{"id":72,"table_id":9,"display_name":"Dimension","base_type":"type/Text","semantic_type":"type/Category","has_field_values":"list","name_field":null,"dimensions":[]}},"last-edit-info":{"id":1,"email":"test@metabase.local","first_name":"test","last_name":"user","timestamp":"2024-05-18T09:37:18.413013Z"},"collection":{"metabase.models.collection.root/is-root?":true,"authority_level":null,"name":"Our analytics","is_personal":false,"id":"root","can_write":true},"parameters":[{"name":"Text","slug":"text","id":"70f9d7fc","type":"string/=","sectionId":"string"}],"auto_apply_filters":true,"created_at":"2024-05-18T09:31:43.161863Z","public_uuid":null,"points_of_interest":null} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/database_2.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/database_2.json deleted file mode 100644 index b723002021f835..00000000000000 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/database_2.json +++ /dev/null @@ -1 +0,0 @@ -{"description":null,"features":["index-info","full-join","basic-aggregations","temporal-extract","actions/custom","now","convert-timezone","nested-field-columns","standard-deviation-aggregations","test/jvm-timezone-setting","date-arithmetics","persist-models","actions","expression-aggregations","percentile-aggregations","connection-impersonation","foreign-keys","table-privileges","right-join","left-join","native-parameters","schemas","nested-queries","expressions","uploads","set-timezone","regex","case-sensitivity-string-filter-options","binning","datetime-diff","upload-with-auto-pk","inner-join","advanced-math-expressions"],"cache_field_values_schedule":"0 0 6 * * ? *","timezone":"GMT","auto_run_queries":true,"metadata_sync_schedule":"0 19 * * * ? *","name":"test_database","settings":null,"caveats":null,"can-manage":true,"creator_id":1,"is_full_sync":true,"updated_at":"2024-05-18T09:30:06.734897Z","cache_ttl":null,"details":{"ssl":false,"ssl-key-source":null,"password":"**MetabasePass**","ssl-key-creator-id":null,"ssl-key-password-source":null,"port":null,"advanced-options":false,"schema-filters-type":"all","dbname":"test_data","ssl-root-cert-creator-id":null,"host":"postgres","tunnel-enabled":false,"ssl-key-password-creator-id":null,"ssl-root-cert-source":null,"ssl-client-cert-creator-id":null,"user":"metabase","ssl-client-cert-source":null},"is_sample":false,"id":2,"is_on_demand":false,"schedules":{"metadata_sync":{"schedule_minute":19,"schedule_day":null,"schedule_frame":null,"schedule_hour":null,"schedule_type":"hourly"},"cache_field_values":{"schedule_minute":0,"schedule_day":null,"schedule_frame":null,"schedule_hour":6,"schedule_type":"daily"}},"engine":"postgres","initial_sync_status":"complete","is_audit":false,"dbms_version":{"flavor":"PostgreSQL","version":"16.3 (Debian 16.3-1.pgdg120+1)","semantic-version":[16,3]},"refingerprint":null,"created_at":"2024-05-18T09:30:06.405812Z","points_of_interest":null,"can_upload":false} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/table_9.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/table_9.json deleted file mode 100644 index 458b1a11b505c0..00000000000000 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/table_9.json +++ /dev/null @@ -1 +0,0 @@ -{"description":null,"entity_type":"entity/GenericTable","schema":"public","database_require_filter":null,"db":{"description":null,"features":["index-info","full-join","basic-aggregations","temporal-extract","actions/custom","now","convert-timezone","nested-field-columns","standard-deviation-aggregations","test/jvm-timezone-setting","date-arithmetics","persist-models","actions","expression-aggregations","percentile-aggregations","connection-impersonation","foreign-keys","table-privileges","right-join","left-join","native-parameters","schemas","nested-queries","expressions","uploads","set-timezone","regex","case-sensitivity-string-filter-options","binning","datetime-diff","upload-with-auto-pk","inner-join","advanced-math-expressions"],"cache_field_values_schedule":"0 0 6 * * ? *","timezone":"GMT","auto_run_queries":true,"metadata_sync_schedule":"0 19 * * * ? *","name":"test_database","settings":null,"caveats":null,"creator_id":1,"is_full_sync":true,"updated_at":"2024-05-18T09:30:06.734897Z","cache_ttl":null,"details":{"ssl":false,"password":"**MetabasePass**","port":null,"advanced-options":false,"schema-filters-type":"all","dbname":"test_data","host":"postgres","tunnel-enabled":false,"user":"metabase"},"is_sample":false,"id":2,"is_on_demand":false,"engine":"postgres","initial_sync_status":"complete","is_audit":false,"dbms_version":{"flavor":"PostgreSQL","version":"16.3 (Debian 16.3-1.pgdg120+1)","semantic-version":[16,3]},"refingerprint":null,"created_at":"2024-05-18T09:30:06.405812Z","points_of_interest":null},"show_in_getting_started":false,"name":"test_table","caveats":null,"updated_at":"2024-05-18T09:30:06.783792Z","pk_field":74,"active":true,"id":9,"db_id":2,"visibility_type":null,"field_order":"database","is_upload":false,"initial_sync_status":"complete","display_name":"Test Table","created_at":"2024-05-18T09:30:06.660148Z","points_of_interest":null} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/user.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/user.json deleted file mode 100644 index c1adfc798982c8..00000000000000 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/user.json +++ /dev/null @@ -1 +0,0 @@ -{"data":[{"email":"test@metabase.local","first_name":"test","locale":null,"last_login":"2024-05-18T09:30:06.609263Z","is_active":true,"is_qbnewb":false,"updated_at":"2024-05-18T09:31:55.368683Z","group_ids":[1,2],"is_superuser":true,"login_attributes":null,"id":1,"last_name":"user","date_joined":"2024-05-18T09:30:06.405812Z","sso_source":null,"personal_collection_id":1,"common_name":"test user"}],"total":1,"limit":null,"offset":null} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/user_1.json b/metadata-ingestion/tests/integration/metabase/setup/issue_9767/user_1.json deleted file mode 100644 index bccf0a9cd6fb9a..00000000000000 --- a/metadata-ingestion/tests/integration/metabase/setup/issue_9767/user_1.json +++ /dev/null @@ -1 +0,0 @@ -{"email":"test@metabase.local","first_name":"test","locale":null,"last_login":"2024-05-18T09:30:06.609263Z","is_active":true,"user_group_memberships":[{"id":1},{"id":2}],"is_qbnewb":false,"updated_at":"2024-05-18T09:31:55.368683Z","is_superuser":true,"login_attributes":null,"id":1,"last_name":"user","date_joined":"2024-05-18T09:30:06.405812Z","sso_source":null,"common_name":"test user"} \ No newline at end of file From 3b5e2cee63dad038dad69857fc518a45b5a8dd07 Mon Sep 17 00:00:00 2001 From: Paul Rogalinski-Pinter Date: Tue, 21 May 2024 00:29:31 +0200 Subject: [PATCH 13/14] do not greedy-strip multiple variables at once --- .../integration/metabase/test_metabase.py | 79 +++++++------------ 1 file changed, 27 insertions(+), 52 deletions(-) diff --git a/metadata-ingestion/tests/integration/metabase/test_metabase.py b/metadata-ingestion/tests/integration/metabase/test_metabase.py index 48f3bcb52437d4..4e4a9045d7bcb4 100644 --- a/metadata-ingestion/tests/integration/metabase/test_metabase.py +++ b/metadata-ingestion/tests/integration/metabase/test_metabase.py @@ -8,6 +8,7 @@ from datahub.configuration.common import PipelineExecutionError from datahub.ingestion.run.pipeline import Pipeline +from datahub.ingestion.source.metabase import MetabaseSource from tests.test_helpers import mce_helpers from tests.test_helpers.state_helpers import ( get_current_checkpoint_from_pipeline, @@ -305,57 +306,31 @@ def test_mode_ingest_failure(pytestconfig, tmp_path, default_json_response_map): @freeze_time(FROZEN_TIME) -def test_9767_query_with_template_tags_does_not_fail( +def test_9767_templated_query_is_stripped( pytestconfig, tmp_path, test_pipeline, mock_datahub_graph ): - baseUrl = "http://localhost:3000/api" - - test_json_response_map = { - f"{baseUrl}/session": "session.json", - f"{baseUrl}/user/current": "issue_9767/user.json", - f"{baseUrl}/user/1": "issue_9767/user_1.json", - f"{baseUrl}/card": "issue_9767/card.json", - f"{baseUrl}/card/1": "issue_9767/card_1.json", - f"{baseUrl}/collection": "issue_9767/collection.json", - f"{baseUrl}/collection/?exclude-other-user-collections=false": "issue_9767/collection.json", - f"{baseUrl}/collection/root/items?models=dashboard": "issue_9767/collection_dashboard_root.json", - f"{baseUrl}/dashboard/1": "issue_9767/dashboard_1.json", - f"{baseUrl}/database/2": "issue_9767/database_2.json", - f"{baseUrl}/table/9": "issue_9767/table_9.json", - } - - with patch( - "datahub.ingestion.source.metabase.requests.session", - side_effect=MockResponse.build_mocked_requests_sucess(test_json_response_map), - ), patch( - "datahub.ingestion.source.metabase.requests.post", - side_effect=MockResponse.build_mocked_requests_session_post( - test_json_response_map - ), - ), patch( - "datahub.ingestion.source.metabase.requests.delete", - side_effect=MockResponse.build_mocked_requests_session_delete( - test_json_response_map - ), - ), patch( - "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph", - mock_datahub_graph, - ) as mock_checkpoint: - mock_checkpoint.return_value = mock_datahub_graph - - pipeline = Pipeline.create(test_pipeline) - pipeline.run() - pipeline.raise_from_status() - - report = pipeline.source.get_report() - for warning in report.warnings: - assert "Unable to retrieve lineage from query" not in str( - report.warnings[warning] - ) - - mce_helpers.check_golden_file( - pytestconfig, - output_path=f"{tmp_path}/metabase_mces.json", - golden_path=test_resources_dir / "metabase_mces_golden_issue_9767.json", - ignore_paths=mce_helpers.IGNORE_PATH_TIMESTAMPS, - ) + query_with_variables = ( + "SELECT count(*) FROM products WHERE category = {{category}}", + "SELECT count(*) FROM products WHERE category = 1", + ) + query_with_optional_clause = ( + "SELECT count(*) FROM products [[WHERE category = {{category}}]]", + "SELECT count(*) FROM products ", + ) + query_with_dashboard_filters = ( + "SELECT count(*) FROM products WHERE {{Filter1}} AND {{Filter2}}", + "SELECT count(*) FROM products WHERE 1 AND 1", + ) + + assert ( + MetabaseSource.strip_template_expressions(query_with_variables[0]) + == query_with_variables[1] + ) + assert ( + MetabaseSource.strip_template_expressions(query_with_optional_clause[0]) + == query_with_optional_clause[1] + ) + assert ( + MetabaseSource.strip_template_expressions(query_with_dashboard_filters[0]) + == query_with_dashboard_filters[1] + ) From a5f21b121f5a1483f4af77c9821b04154fb088f0 Mon Sep 17 00:00:00 2001 From: Paul Rogalinski-Pinter Date: Tue, 21 May 2024 07:37:49 +0200 Subject: [PATCH 14/14] remove unused DI, rename test --- .../tests/integration/metabase/test_metabase.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/metadata-ingestion/tests/integration/metabase/test_metabase.py b/metadata-ingestion/tests/integration/metabase/test_metabase.py index 4e4a9045d7bcb4..5c433f14f380ff 100644 --- a/metadata-ingestion/tests/integration/metabase/test_metabase.py +++ b/metadata-ingestion/tests/integration/metabase/test_metabase.py @@ -305,10 +305,7 @@ def test_mode_ingest_failure(pytestconfig, tmp_path, default_json_response_map): assert list(exec_error.args[1].failures.keys())[0] == "metabase-dashboard" -@freeze_time(FROZEN_TIME) -def test_9767_templated_query_is_stripped( - pytestconfig, tmp_path, test_pipeline, mock_datahub_graph -): +def test_strip_template_expressions(): query_with_variables = ( "SELECT count(*) FROM products WHERE category = {{category}}", "SELECT count(*) FROM products WHERE category = 1",