From 0ced65fce2bff2564f3d0f1b0df89a009847832e Mon Sep 17 00:00:00 2001 From: Ayush Shah Date: Thu, 21 Nov 2024 14:40:32 +0530 Subject: [PATCH 1/3] fix: modify fqn to allow quotes with dots --- .../ingestion/source/database/sample_data.py | 14 +++++++++++--- ingestion/tests/unit/test_fqn.py | 1 + .../src/main/antlr4/org/openmetadata/schema/Fqn.g4 | 4 ++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/database/sample_data.py b/ingestion/src/metadata/ingestion/source/database/sample_data.py index b4d6246bad53..a10dc9b62541 100644 --- a/ingestion/src/metadata/ingestion/source/database/sample_data.py +++ b/ingestion/src/metadata/ingestion/source/database/sample_data.py @@ -40,6 +40,8 @@ from metadata.generated.schema.api.data.createDatabaseSchema import ( CreateDatabaseSchemaRequest, ) +from metadata.sampler.models import SampleData, SamplerResponse + from metadata.generated.schema.api.data.createMlModel import CreateMlModelRequest from metadata.generated.schema.api.data.createPipeline import CreatePipelineRequest from metadata.generated.schema.api.data.createSearchIndex import ( @@ -898,9 +900,15 @@ def ingest_tables(self) -> Iterable[Either[Entity]]: self.metadata.ingest_table_sample_data( table_entity, - TableData( - rows=table["sampleData"]["rows"], - columns=table["sampleData"]["columns"], + SamplerResponse( + table=table_entity, + sample_data=SampleData( + data=TableData( + rows=table["sampleData"]["rows"], + columns=table["sampleData"]["columns"], + ), + store=True + ) ), ) diff --git a/ingestion/tests/unit/test_fqn.py b/ingestion/tests/unit/test_fqn.py index 1dfb448c57d2..eac462b2144f 100644 --- a/ingestion/tests/unit/test_fqn.py +++ b/ingestion/tests/unit/test_fqn.py @@ -58,6 +58,7 @@ def validate(self, actual_parts, actual_fqn): FQNTest(["a.1", "b.2", "c", "d"], '"a.1"."b.2".c.d'), FQNTest(["a.1", "b.2", "c.3", "d"], '"a.1"."b.2"."c.3".d'), FQNTest(["a.1", "b.2", "c.3", "d.4"], '"a.1"."b.2"."c.3"."d.4"'), + FQNTest(["fqn", "test.test.test"], 'fqn."test.test.test"'), ] for x in xs: x.validate(fqn.split(x.fqn), fqn._build(*x.parts)) diff --git a/openmetadata-spec/src/main/antlr4/org/openmetadata/schema/Fqn.g4 b/openmetadata-spec/src/main/antlr4/org/openmetadata/schema/Fqn.g4 index e7ac76fb95a2..2aad7c487ef8 100644 --- a/openmetadata-spec/src/main/antlr4/org/openmetadata/schema/Fqn.g4 +++ b/openmetadata-spec/src/main/antlr4/org/openmetadata/schema/Fqn.g4 @@ -14,7 +14,7 @@ NAME ; NAME_WITH_RESERVED - : QUOTE NON_RESERVED* (RESERVED)* NON_RESERVED* QUOTE + : QUOTE NON_RESERVED* (RESERVED NON_RESERVED*)+ QUOTE ; QUOTE @@ -22,7 +22,7 @@ QUOTE ; NON_RESERVED - : ~[.] + : ~[".] ; RESERVED From 486516c2864b3a0198dee1898b0b3a839d0001e5 Mon Sep 17 00:00:00 2001 From: Ayush Shah Date: Thu, 21 Nov 2024 14:57:20 +0530 Subject: [PATCH 2/3] fix: lint --- .../src/metadata/ingestion/source/database/sample_data.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/database/sample_data.py b/ingestion/src/metadata/ingestion/source/database/sample_data.py index a10dc9b62541..9a0f5745bfbb 100644 --- a/ingestion/src/metadata/ingestion/source/database/sample_data.py +++ b/ingestion/src/metadata/ingestion/source/database/sample_data.py @@ -40,8 +40,6 @@ from metadata.generated.schema.api.data.createDatabaseSchema import ( CreateDatabaseSchemaRequest, ) -from metadata.sampler.models import SampleData, SamplerResponse - from metadata.generated.schema.api.data.createMlModel import CreateMlModelRequest from metadata.generated.schema.api.data.createPipeline import CreatePipelineRequest from metadata.generated.schema.api.data.createSearchIndex import ( @@ -138,6 +136,7 @@ InvalidSchemaTypeException, schema_parser_config_registry, ) +from metadata.sampler.models import SampleData, SamplerResponse from metadata.utils import entity_link, fqn from metadata.utils.constants import UTF_8 from metadata.utils.fqn import FQN_SEPARATOR @@ -907,8 +906,8 @@ def ingest_tables(self) -> Iterable[Either[Entity]]: rows=table["sampleData"]["rows"], columns=table["sampleData"]["columns"], ), - store=True - ) + store=True, + ), ), ) From cffdddc94a31229e683e2e90148b392dd2d92231 Mon Sep 17 00:00:00 2001 From: Ayush Shah Date: Thu, 21 Nov 2024 19:30:32 +0530 Subject: [PATCH 3/3] fix: modify sample_data --- .../looker/dashboardDataModels.json | 45 ------------------- .../ingestion/source/database/sample_data.py | 4 +- 2 files changed, 2 insertions(+), 47 deletions(-) diff --git a/ingestion/examples/sample_data/looker/dashboardDataModels.json b/ingestion/examples/sample_data/looker/dashboardDataModels.json index 2bdd0b68961d..1d30232b86bd 100644 --- a/ingestion/examples/sample_data/looker/dashboardDataModels.json +++ b/ingestion/examples/sample_data/looker/dashboardDataModels.json @@ -158,51 +158,6 @@ "ordinalPosition": 5 } ] - }, - { - "name": "\"orders_view\" || \"operations_view\"", - "displayName": "Orders View and Operations View", - "description": "Orders View and Operations View from Sample Data", - "dataModelType": "LookMlView", - "serviceType": "Looker", - "sql": "view: orders {\n sql_table_name: orders ;;\n\n dimension: \"1. Phase I\" {\n type: string\n sql: ${TABLE}.status ;;\n }\n\n dimension: \"4. Authorized\" {\n type: int\n sql: ${TABLE}.amount ;;\n }\n}", - "columns": [ - { - "name": "0. Pre-clinical", - "dataType": "NUMERIC", - "dataTypeDisplay": "numeric", - "description": "Vaccine Candidates in phase: 'Pre-clinical'", - "ordinalPosition": 1 - }, - { - "name": "2. Phase II or Combined I/II", - "dataType": "NUMERIC", - "dataTypeDisplay": "numeric", - "description": "Vaccine Candidates in phase: 'Phase II or Combined I/II'", - "ordinalPosition": 2 - }, - { - "name": "1. Phase I", - "dataType": "NUMERIC", - "dataTypeDisplay": "numeric", - "description": "Vaccine Candidates in phase: 'Phase I'", - "ordinalPosition": 3 - }, - { - "name": "3. Phase III", - "dataType": "NUMERIC", - "dataTypeDisplay": "numeric", - "description": "Vaccine Candidates in phase: 'Phase III'", - "ordinalPosition": 4 - }, - { - "name": "4. Authorized", - "dataType": "NUMERIC", - "dataTypeDisplay": "numeric", - "description": "Vaccine Candidates in phase: 'Authorize'", - "ordinalPosition": 5 - } - ] } ] diff --git a/ingestion/src/metadata/ingestion/source/database/sample_data.py b/ingestion/src/metadata/ingestion/source/database/sample_data.py index 9a0f5745bfbb..f809f045206b 100644 --- a/ingestion/src/metadata/ingestion/source/database/sample_data.py +++ b/ingestion/src/metadata/ingestion/source/database/sample_data.py @@ -899,7 +899,7 @@ def ingest_tables(self) -> Iterable[Either[Entity]]: self.metadata.ingest_table_sample_data( table_entity, - SamplerResponse( + sample_data=SamplerResponse( table=table_entity, sample_data=SampleData( data=TableData( @@ -908,7 +908,7 @@ def ingest_tables(self) -> Iterable[Either[Entity]]: ), store=True, ), - ), + ).sample_data.data, ) if table.get("customMetrics"):