Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(ingest/mssql): Add UNIQUEIDENTIFIER data type as String #8642

Merged
merged 5 commits into from
Sep 12, 2023
Merged

fix(ingest/mssql): Add UNIQUEIDENTIFIER data type as String #8642

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1306480
Added MSSQL data type UNIQUEIDENTIFER to String
cjm98332 Aug 15, 2023
44c5a45
deleted readme2.md
cjm98332 Aug 15, 2023
a9e5cbb
deleted comments
cjm98332 Aug 15, 2023
5a6b82a
update tests
asikowitz Aug 17, 2023
1962629
Merge branch 'master' into cjmuniqueident
asikowitz Sep 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,17 @@
BasicSQLAlchemyConfig,
make_sqlalchemy_uri,
)
from datahub.metadata.schema_classes import BooleanTypeClass, UnionTypeClass
from datahub.metadata.schema_classes import (
BooleanTypeClass,
StringTypeClass,
UnionTypeClass,
)

logger: logging.Logger = logging.getLogger(__name__)

register_custom_type(sqlalchemy.dialects.mssql.BIT, BooleanTypeClass)
register_custom_type(sqlalchemy.dialects.mssql.SQL_VARIANT, UnionTypeClass)
register_custom_type(sqlalchemy.dialects.mssql.UNIQUEIDENTIFIER, StringTypeClass)


class SQLServerConfig(BasicSQLAlchemyConfig):
Expand Down
12 changes: 12 additions & 0 deletions ...-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1752,6 +1752,18 @@
"recursive": false,
"isPartOfKey": true
},
{
"fieldPath": "SomeId",
"nullable": false,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "UNIQUEIDENTIFIER()",
"recursive": false,
"isPartOfKey": false
},
{
"fieldPath": "Name",
"nullable": true,
Expand Down
12 changes: 12 additions & 0 deletions ...estion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1752,6 +1752,18 @@
"recursive": false,
"isPartOfKey": true
},
{
"fieldPath": "SomeId",
"nullable": false,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "UNIQUEIDENTIFIER()",
"recursive": false,
"isPartOfKey": false
},
{
"fieldPath": "Name",
"nullable": true,
Expand Down
12 changes: 12 additions & 0 deletions metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1752,6 +1752,18 @@
"recursive": false,
"isPartOfKey": true
},
{
"fieldPath": "SomeId",
"nullable": false,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "UNIQUEIDENTIFIER()",
"recursive": false,
"isPartOfKey": false
},
{
"fieldPath": "Name",
"nullable": true,
Expand Down
12 changes: 12 additions & 0 deletions ...tion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1752,6 +1752,18 @@
"recursive": false,
"isPartOfKey": true
},
{
"fieldPath": "SomeId",
"nullable": false,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "UNIQUEIDENTIFIER()",
"recursive": false,
"isPartOfKey": false
},
{
"fieldPath": "Name",
"nullable": true,
Expand Down
27 changes: 14 additions & 13 deletions metadata-ingestion/tests/integration/sql_server/setup/setup.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ CREATE TABLE Foo.Persons (
GO
CREATE TABLE Foo.SalesReason
(
TempID int NOT NULL,
TempID int NOT NULL,
SomeId UNIQUEIDENTIFIER NOT NULL DEFAULT NEWID(),
Name nvarchar(50)
, CONSTRAINT PK_TempSales PRIMARY KEY NONCLUSTERED (TempID)
, CONSTRAINT FK_TempSales_SalesReason FOREIGN KEY (TempID)
Expand All @@ -49,20 +50,20 @@ AS
SELECT @ID AS ThatDB;
GO

GO
EXEC sys.sp_addextendedproperty
@name = N'MS_Description',
@value = N'Description for table Items of schema Foo.',
@level0type = N'SCHEMA', @level0name = 'Foo',
@level1type = N'TABLE', @level1name = 'Items';
GO
EXEC sys.sp_addextendedproperty
@name = N'MS_Description',
@value = N'Description for table Items of schema Foo.',
@level0type = N'SCHEMA', @level0name = 'Foo',
@level1type = N'TABLE', @level1name = 'Items';
GO

GO
EXEC sys.sp_addextendedproperty
@name = N'MS_Description',
@value = N'Description for column LastName of table Persons of schema Foo.',
@level0type = N'SCHEMA', @level0name = 'Foo',
@level1type = N'TABLE', @level1name = 'Persons',
GO
EXEC sys.sp_addextendedproperty
@name = N'MS_Description',
@value = N'Description for column LastName of table Persons of schema Foo.',
@level0type = N'SCHEMA', @level0name = 'Foo',
@level1type = N'TABLE', @level1name = 'Persons',
@level2type = N'COLUMN',@level2name = 'LastName';
GO
USE msdb ;
Expand Down
Loading