From 1306480f463778eb885f47154424ecfa6bc3e830 Mon Sep 17 00:00:00 2001 From: cjm98332 Date: Tue, 15 Aug 2023 11:29:50 -0700 Subject: [PATCH 1/4] Added MSSQL data type UNIQUEIDENTIFER to String --- .../src/datahub/ingestion/source/sql/mssql.py | 4 +- readme2.md | 173 ++++++++++++++++++ 2 files changed, 175 insertions(+), 2 deletions(-) create mode 100644 readme2.md diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql.py index a9afd40fd45b6..da40ef4355c7b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql.py @@ -30,13 +30,13 @@ BasicSQLAlchemyConfig, make_sqlalchemy_uri, ) -from datahub.metadata.schema_classes import BooleanTypeClass, UnionTypeClass +from datahub.metadata.schema_classes import BooleanTypeClass, UnionTypeClass, StringTypeClass #CJM08122023 logger: logging.Logger = logging.getLogger(__name__) register_custom_type(sqlalchemy.dialects.mssql.BIT, BooleanTypeClass) register_custom_type(sqlalchemy.dialects.mssql.SQL_VARIANT, UnionTypeClass) - +register_custom_type(sqlalchemy.dialects.mssql.UNIQUEIDENTIFIER, StringTypeClass) #CJM08122023 class SQLServerConfig(BasicSQLAlchemyConfig): # defaults diff --git a/readme2.md b/readme2.md new file mode 100644 index 0000000000000..d2208cf6ced49 --- /dev/null +++ b/readme2.md @@ -0,0 +1,173 @@ + +

+DataHub +

+ + +# DataHub: The Metadata Platform for the Modern Data Stack +## Built with ❤️ by [Acryl Data](https://acryldata.io) and [LinkedIn](https://engineering.linkedin.com) +[![Version](https://img.shields.io/github/v/release/datahub-project/datahub?include_prereleases)](https://github.com/datahub-project/datahub/releases/latest) +[![PyPI version](https://badge.fury.io/py/acryl-datahub.svg)](https://badge.fury.io/py/acryl-datahub) +[![build & test](https://github.com/datahub-project/datahub/workflows/build%20&%20test/badge.svg?branch=master&event=push)](https://github.com/datahub-project/datahub/actions?query=workflow%3A%22build+%26+test%22+branch%3Amaster+event%3Apush) +[![Docker Pulls](https://img.shields.io/docker/pulls/linkedin/datahub-gms.svg)](https://hub.docker.com/r/linkedin/datahub-gms) +[![Slack](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://slack.datahubproject.io) +[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/datahub-project/datahub/blob/master/docs/CONTRIBUTING.md) +[![GitHub commit activity](https://img.shields.io/github/commit-activity/m/datahub-project/datahub)](https://github.com/datahub-project/datahub/pulls?q=is%3Apr) +[![License](https://img.shields.io/github/license/datahub-project/datahub)](https://github.com/datahub-project/datahub/blob/master/LICENSE) +[![YouTube](https://img.shields.io/youtube/channel/subscribers/UC3qFQC5IiwR5fvWEqi_tJ5w?style=social)](https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w) +[![Medium](https://img.shields.io/badge/Medium-12100E?style=for-the-badge&logo=medium&logoColor=white)](https://medium.com/datahub-project) +[![Follow](https://img.shields.io/twitter/follow/datahubproject?label=Follow&style=social)](https://twitter.com/datahubproject) +### 🏠 Hosted DataHub Docs (Courtesy of Acryl Data): [datahubproject.io](https://datahubproject.io/docs) + +--- + +[Quickstart](https://datahubproject.io/docs/quickstart) | +[Features](https://datahubproject.io/docs/features) | +[Roadmap](https://feature-requests.datahubproject.io/roadmap) | +[Adoption](#adoption) | +[Demo](https://demo.datahubproject.io/) | +[Town Hall](https://datahubproject.io/docs/townhalls) + +--- +> 📣 DataHub Town Hall is the 4th Thursday at 9am US PT of every month - [add it to your calendar!](https://rsvp.datahubproject.io/) +> +> - Town-hall Zoom link: [zoom.datahubproject.io](https://zoom.datahubproject.io) +> - [Meeting details](docs/townhalls.md) & [past recordings](docs/townhall-history.md) + +> ✨ DataHub Community Highlights: +> +> - Read our Monthly Project Updates [here](https://blog.datahubproject.io/tagged/project-updates). +> - Bringing The Power Of The DataHub Real-Time Metadata Graph To Everyone At Acryl Data: [Data Engineering Podcast](https://www.dataengineeringpodcast.com/acryl-data-datahub-metadata-graph-episode-230/) +> - Check out our most-read blog post, [DataHub: Popular Metadata Architectures Explained](https://engineering.linkedin.com/blog/2020/datahub-popular-metadata-architectures-explained) @ LinkedIn Engineering Blog. +> - Join us on [Slack](docs/slack.md)! Ask questions and keep up with the latest announcements. + +## Introduction + +DataHub is an open-source metadata platform for the modern data stack. Read about the architectures of different metadata systems and why DataHub excels [here](https://engineering.linkedin.com/blog/2020/datahub-popular-metadata-architectures-explained). Also read our +[LinkedIn Engineering blog post](https://engineering.linkedin.com/blog/2019/data-hub), check out our [Strata presentation](https://speakerdeck.com/shirshanka/the-evolution-of-metadata-linkedins-journey-strata-nyc-2019) and watch our [Crunch Conference Talk](https://www.youtube.com/watch?v=OB-O0Y6OYDE). You should also visit [DataHub Architecture](docs/architecture/architecture.md) to get a better understanding of how DataHub is implemented. + +## Features & Roadmap + +Check out DataHub's [Features](docs/features.md) & [Roadmap](https://feature-requests.datahubproject.io/roadmap). + +## Demo and Screenshots + +There's a [hosted demo environment](https://demo.datahubproject.io/) courtesy of [Acryl Data](https://acryldata.io) where you can explore DataHub without installing it locally + +## Quickstart + +Please follow the [DataHub Quickstart Guide](https://datahubproject.io/docs/quickstart) to get a copy of DataHub up & running locally using [Docker](https://docker.com). As the guide assumes some basic knowledge of Docker, we'd recommend you to go through the "Hello World" example of [A Docker Tutorial for Beginners](https://docker-curriculum.com) if Docker is completely foreign to you. + +## Development + +If you're looking to build & modify datahub please take a look at our [Development Guide](https://datahubproject.io/docs/developers). + +[![DataHub Demo GIF](docs/imgs/entity.png)](https://demo.datahubproject.io/) + +## Source Code and Repositories + +- [datahub-project/datahub](https://github.com/datahub-project/datahub): This repository contains the complete source code for DataHub's metadata model, metadata services, integration connectors and the web application. +- [acryldata/datahub-actions](https://github.com/acryldata/datahub-actions): DataHub Actions is a framework for responding to changes to your DataHub Metadata Graph in real time. +- [acryldata/datahub-helm](https://github.com/acryldata/datahub-helm): Repository of helm charts for deploying DataHub on a Kubernetes cluster +- [acryldata/meta-world](https://github.com/acryldata/meta-world): A repository to store recipes, custom sources, transformations and other things to make your DataHub experience magical +- [dbt-impact-action](https://github.com/acryldata/dbt-impact-action) : This repository contains a github action for commenting on your PRs with a summary of the impact of changes within a dbt project +- [datahub-tools](https://github.com/makenotion/datahub-tools) : Additional python tools to interact with the DataHub GraphQL endpoints, built by Notion +- [business-glossary-sync-action](https://github.com/acryldata/business-glossary-sync-action) : This repository contains a github action that opens PRs to update your business glossary yaml file. + +## Releases + +See [Releases](https://github.com/datahub-project/datahub/releases) page for more details. We follow the [SemVer Specification](https://semver.org) when versioning the releases and adopt the [Keep a Changelog convention](https://keepachangelog.com/) for the changelog format. + +## Contributing + +We welcome contributions from the community. Please refer to our [Contributing Guidelines](docs/CONTRIBUTING.md) for more details. We also have a [contrib](contrib) directory for incubating experimental features. + +## Community + +Join our [Slack workspace](https://slack.datahubproject.io) for discussions and important announcements. You can also find out more about our upcoming [town hall meetings](docs/townhalls.md) and view past recordings. + +## Adoption + +Here are the companies that have officially adopted DataHub. Please feel free to add yours to the list if we missed it. + +- [ABLY](https://ably.team/) +- [Adevinta](https://www.adevinta.com/) +- [Banksalad](https://www.banksalad.com) +- [Cabify](https://cabify.tech/) +- [ClassDojo](https://www.classdojo.com/) +- [Coursera](https://www.coursera.org/) +- [DefinedCrowd](http://www.definedcrowd.com) +- [DFDS](https://www.dfds.com/) +- [Digital Turbine](https://www.digitalturbine.com/) +- [Expedia Group](http://expedia.com) +- [Experius](https://www.experius.nl) +- [Geotab](https://www.geotab.com) +- [Grofers](https://grofers.com) +- [Haibo Technology](https://www.botech.com.cn) +- [hipages](https://hipages.com.au/) +- [inovex](https://www.inovex.de/) +- [IOMED](https://iomed.health) +- [Klarna](https://www.klarna.com) +- [LinkedIn](http://linkedin.com) +- [Moloco](https://www.moloco.com/en) +- [N26](https://n26brasil.com/) +- [Optum](https://www.optum.com/) +- [Peloton](https://www.onepeloton.com) +- [PITS Global Data Recovery Services](https://www.pitsdatarecovery.net/) +- [Razer](https://www.razer.com) +- [Saxo Bank](https://www.home.saxo) +- [Showroomprive](https://www.showroomprive.com/) +- [SpotHero](https://spothero.com) +- [Stash](https://www.stash.com) +- [Shanghai HuaRui Bank](https://www.shrbank.com) +- [ThoughtWorks](https://www.thoughtworks.com) +- [TypeForm](http://typeform.com) +- [Udemy](https://www.udemy.com/) +- [Uphold](https://uphold.com) +- [Viasat](https://viasat.com) +- [Wikimedia](https://www.wikimedia.org) +- [Wolt](https://wolt.com) +- [Zynga](https://www.zynga.com) + + +## Select Articles & Talks + +- [DataHub Blog](https://blog.datahubproject.io/) +- [DataHub YouTube Channel](https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w) +- [Optum: Data Mesh via DataHub](https://optum.github.io/blog/2022/03/23/data-mesh-via-datahub/) +- [Saxo Bank: Enabling Data Discovery in Data Mesh](https://medium.com/datahub-project/enabling-data-discovery-in-a-data-mesh-the-saxo-journey-451b06969c8f) +- [Bringing The Power Of The DataHub Real-Time Metadata Graph To Everyone At Acryl Data](https://www.dataengineeringpodcast.com/acryl-data-datahub-metadata-graph-episode-230/) +- [DataHub: Popular Metadata Architectures Explained](https://engineering.linkedin.com/blog/2020/datahub-popular-metadata-architectures-explained) +- [Driving DataOps Culture with LinkedIn DataHub](https://www.youtube.com/watch?v=ccsIKK9nVxk) @ [DataOps Unleashed 2021](https://dataopsunleashed.com/#shirshanka-session) +- [The evolution of metadata: LinkedIn’s story](https://speakerdeck.com/shirshanka/the-evolution-of-metadata-linkedins-journey-strata-nyc-2019) @ [Strata Data Conference 2019](https://conferences.oreilly.com/strata/strata-ny-2019.html) +- [Journey of metadata at LinkedIn](https://www.youtube.com/watch?v=OB-O0Y6OYDE) @ [Crunch Data Conference 2019](https://crunchconf.com/2019) +- [DataHub Journey with Expedia Group](https://www.youtube.com/watch?v=ajcRdB22s5o) +- [Data Discoverability at SpotHero](https://www.slideshare.net/MaggieHays/data-discoverability-at-spothero) +- [Data Catalogue — Knowing your data](https://medium.com/albert-franzi/data-catalogue-knowing-your-data-15f7d0724900) +- [DataHub: A Generalized Metadata Search & Discovery Tool](https://engineering.linkedin.com/blog/2019/data-hub) +- [Open sourcing DataHub: LinkedIn’s metadata search and discovery platform](https://engineering.linkedin.com/blog/2020/open-sourcing-datahub--linkedins-metadata-search-and-discovery-p) +- [Emerging Architectures for Modern Data Infrastructure](https://future.com/emerging-architectures-for-modern-data-infrastructure-2020/) + +See the full list [here](docs/links.md). + +## License + +[Apache License 2.0](./LICENSE). From 44c5a45ed08ffa53a78e825eeb9963cce4bbb39e Mon Sep 17 00:00:00 2001 From: cjm98332 Date: Tue, 15 Aug 2023 14:23:34 -0700 Subject: [PATCH 2/4] deleted readme2.md --- readme2.md | 173 ----------------------------------------------------- 1 file changed, 173 deletions(-) delete mode 100644 readme2.md diff --git a/readme2.md b/readme2.md deleted file mode 100644 index d2208cf6ced49..0000000000000 --- a/readme2.md +++ /dev/null @@ -1,173 +0,0 @@ - -

-DataHub -

- - -# DataHub: The Metadata Platform for the Modern Data Stack -## Built with ❤️ by [Acryl Data](https://acryldata.io) and [LinkedIn](https://engineering.linkedin.com) -[![Version](https://img.shields.io/github/v/release/datahub-project/datahub?include_prereleases)](https://github.com/datahub-project/datahub/releases/latest) -[![PyPI version](https://badge.fury.io/py/acryl-datahub.svg)](https://badge.fury.io/py/acryl-datahub) -[![build & test](https://github.com/datahub-project/datahub/workflows/build%20&%20test/badge.svg?branch=master&event=push)](https://github.com/datahub-project/datahub/actions?query=workflow%3A%22build+%26+test%22+branch%3Amaster+event%3Apush) -[![Docker Pulls](https://img.shields.io/docker/pulls/linkedin/datahub-gms.svg)](https://hub.docker.com/r/linkedin/datahub-gms) -[![Slack](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://slack.datahubproject.io) -[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/datahub-project/datahub/blob/master/docs/CONTRIBUTING.md) -[![GitHub commit activity](https://img.shields.io/github/commit-activity/m/datahub-project/datahub)](https://github.com/datahub-project/datahub/pulls?q=is%3Apr) -[![License](https://img.shields.io/github/license/datahub-project/datahub)](https://github.com/datahub-project/datahub/blob/master/LICENSE) -[![YouTube](https://img.shields.io/youtube/channel/subscribers/UC3qFQC5IiwR5fvWEqi_tJ5w?style=social)](https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w) -[![Medium](https://img.shields.io/badge/Medium-12100E?style=for-the-badge&logo=medium&logoColor=white)](https://medium.com/datahub-project) -[![Follow](https://img.shields.io/twitter/follow/datahubproject?label=Follow&style=social)](https://twitter.com/datahubproject) -### 🏠 Hosted DataHub Docs (Courtesy of Acryl Data): [datahubproject.io](https://datahubproject.io/docs) - ---- - -[Quickstart](https://datahubproject.io/docs/quickstart) | -[Features](https://datahubproject.io/docs/features) | -[Roadmap](https://feature-requests.datahubproject.io/roadmap) | -[Adoption](#adoption) | -[Demo](https://demo.datahubproject.io/) | -[Town Hall](https://datahubproject.io/docs/townhalls) - ---- -> 📣 DataHub Town Hall is the 4th Thursday at 9am US PT of every month - [add it to your calendar!](https://rsvp.datahubproject.io/) -> -> - Town-hall Zoom link: [zoom.datahubproject.io](https://zoom.datahubproject.io) -> - [Meeting details](docs/townhalls.md) & [past recordings](docs/townhall-history.md) - -> ✨ DataHub Community Highlights: -> -> - Read our Monthly Project Updates [here](https://blog.datahubproject.io/tagged/project-updates). -> - Bringing The Power Of The DataHub Real-Time Metadata Graph To Everyone At Acryl Data: [Data Engineering Podcast](https://www.dataengineeringpodcast.com/acryl-data-datahub-metadata-graph-episode-230/) -> - Check out our most-read blog post, [DataHub: Popular Metadata Architectures Explained](https://engineering.linkedin.com/blog/2020/datahub-popular-metadata-architectures-explained) @ LinkedIn Engineering Blog. -> - Join us on [Slack](docs/slack.md)! Ask questions and keep up with the latest announcements. - -## Introduction - -DataHub is an open-source metadata platform for the modern data stack. Read about the architectures of different metadata systems and why DataHub excels [here](https://engineering.linkedin.com/blog/2020/datahub-popular-metadata-architectures-explained). Also read our -[LinkedIn Engineering blog post](https://engineering.linkedin.com/blog/2019/data-hub), check out our [Strata presentation](https://speakerdeck.com/shirshanka/the-evolution-of-metadata-linkedins-journey-strata-nyc-2019) and watch our [Crunch Conference Talk](https://www.youtube.com/watch?v=OB-O0Y6OYDE). You should also visit [DataHub Architecture](docs/architecture/architecture.md) to get a better understanding of how DataHub is implemented. - -## Features & Roadmap - -Check out DataHub's [Features](docs/features.md) & [Roadmap](https://feature-requests.datahubproject.io/roadmap). - -## Demo and Screenshots - -There's a [hosted demo environment](https://demo.datahubproject.io/) courtesy of [Acryl Data](https://acryldata.io) where you can explore DataHub without installing it locally - -## Quickstart - -Please follow the [DataHub Quickstart Guide](https://datahubproject.io/docs/quickstart) to get a copy of DataHub up & running locally using [Docker](https://docker.com). As the guide assumes some basic knowledge of Docker, we'd recommend you to go through the "Hello World" example of [A Docker Tutorial for Beginners](https://docker-curriculum.com) if Docker is completely foreign to you. - -## Development - -If you're looking to build & modify datahub please take a look at our [Development Guide](https://datahubproject.io/docs/developers). - -[![DataHub Demo GIF](docs/imgs/entity.png)](https://demo.datahubproject.io/) - -## Source Code and Repositories - -- [datahub-project/datahub](https://github.com/datahub-project/datahub): This repository contains the complete source code for DataHub's metadata model, metadata services, integration connectors and the web application. -- [acryldata/datahub-actions](https://github.com/acryldata/datahub-actions): DataHub Actions is a framework for responding to changes to your DataHub Metadata Graph in real time. -- [acryldata/datahub-helm](https://github.com/acryldata/datahub-helm): Repository of helm charts for deploying DataHub on a Kubernetes cluster -- [acryldata/meta-world](https://github.com/acryldata/meta-world): A repository to store recipes, custom sources, transformations and other things to make your DataHub experience magical -- [dbt-impact-action](https://github.com/acryldata/dbt-impact-action) : This repository contains a github action for commenting on your PRs with a summary of the impact of changes within a dbt project -- [datahub-tools](https://github.com/makenotion/datahub-tools) : Additional python tools to interact with the DataHub GraphQL endpoints, built by Notion -- [business-glossary-sync-action](https://github.com/acryldata/business-glossary-sync-action) : This repository contains a github action that opens PRs to update your business glossary yaml file. - -## Releases - -See [Releases](https://github.com/datahub-project/datahub/releases) page for more details. We follow the [SemVer Specification](https://semver.org) when versioning the releases and adopt the [Keep a Changelog convention](https://keepachangelog.com/) for the changelog format. - -## Contributing - -We welcome contributions from the community. Please refer to our [Contributing Guidelines](docs/CONTRIBUTING.md) for more details. We also have a [contrib](contrib) directory for incubating experimental features. - -## Community - -Join our [Slack workspace](https://slack.datahubproject.io) for discussions and important announcements. You can also find out more about our upcoming [town hall meetings](docs/townhalls.md) and view past recordings. - -## Adoption - -Here are the companies that have officially adopted DataHub. Please feel free to add yours to the list if we missed it. - -- [ABLY](https://ably.team/) -- [Adevinta](https://www.adevinta.com/) -- [Banksalad](https://www.banksalad.com) -- [Cabify](https://cabify.tech/) -- [ClassDojo](https://www.classdojo.com/) -- [Coursera](https://www.coursera.org/) -- [DefinedCrowd](http://www.definedcrowd.com) -- [DFDS](https://www.dfds.com/) -- [Digital Turbine](https://www.digitalturbine.com/) -- [Expedia Group](http://expedia.com) -- [Experius](https://www.experius.nl) -- [Geotab](https://www.geotab.com) -- [Grofers](https://grofers.com) -- [Haibo Technology](https://www.botech.com.cn) -- [hipages](https://hipages.com.au/) -- [inovex](https://www.inovex.de/) -- [IOMED](https://iomed.health) -- [Klarna](https://www.klarna.com) -- [LinkedIn](http://linkedin.com) -- [Moloco](https://www.moloco.com/en) -- [N26](https://n26brasil.com/) -- [Optum](https://www.optum.com/) -- [Peloton](https://www.onepeloton.com) -- [PITS Global Data Recovery Services](https://www.pitsdatarecovery.net/) -- [Razer](https://www.razer.com) -- [Saxo Bank](https://www.home.saxo) -- [Showroomprive](https://www.showroomprive.com/) -- [SpotHero](https://spothero.com) -- [Stash](https://www.stash.com) -- [Shanghai HuaRui Bank](https://www.shrbank.com) -- [ThoughtWorks](https://www.thoughtworks.com) -- [TypeForm](http://typeform.com) -- [Udemy](https://www.udemy.com/) -- [Uphold](https://uphold.com) -- [Viasat](https://viasat.com) -- [Wikimedia](https://www.wikimedia.org) -- [Wolt](https://wolt.com) -- [Zynga](https://www.zynga.com) - - -## Select Articles & Talks - -- [DataHub Blog](https://blog.datahubproject.io/) -- [DataHub YouTube Channel](https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w) -- [Optum: Data Mesh via DataHub](https://optum.github.io/blog/2022/03/23/data-mesh-via-datahub/) -- [Saxo Bank: Enabling Data Discovery in Data Mesh](https://medium.com/datahub-project/enabling-data-discovery-in-a-data-mesh-the-saxo-journey-451b06969c8f) -- [Bringing The Power Of The DataHub Real-Time Metadata Graph To Everyone At Acryl Data](https://www.dataengineeringpodcast.com/acryl-data-datahub-metadata-graph-episode-230/) -- [DataHub: Popular Metadata Architectures Explained](https://engineering.linkedin.com/blog/2020/datahub-popular-metadata-architectures-explained) -- [Driving DataOps Culture with LinkedIn DataHub](https://www.youtube.com/watch?v=ccsIKK9nVxk) @ [DataOps Unleashed 2021](https://dataopsunleashed.com/#shirshanka-session) -- [The evolution of metadata: LinkedIn’s story](https://speakerdeck.com/shirshanka/the-evolution-of-metadata-linkedins-journey-strata-nyc-2019) @ [Strata Data Conference 2019](https://conferences.oreilly.com/strata/strata-ny-2019.html) -- [Journey of metadata at LinkedIn](https://www.youtube.com/watch?v=OB-O0Y6OYDE) @ [Crunch Data Conference 2019](https://crunchconf.com/2019) -- [DataHub Journey with Expedia Group](https://www.youtube.com/watch?v=ajcRdB22s5o) -- [Data Discoverability at SpotHero](https://www.slideshare.net/MaggieHays/data-discoverability-at-spothero) -- [Data Catalogue — Knowing your data](https://medium.com/albert-franzi/data-catalogue-knowing-your-data-15f7d0724900) -- [DataHub: A Generalized Metadata Search & Discovery Tool](https://engineering.linkedin.com/blog/2019/data-hub) -- [Open sourcing DataHub: LinkedIn’s metadata search and discovery platform](https://engineering.linkedin.com/blog/2020/open-sourcing-datahub--linkedins-metadata-search-and-discovery-p) -- [Emerging Architectures for Modern Data Infrastructure](https://future.com/emerging-architectures-for-modern-data-infrastructure-2020/) - -See the full list [here](docs/links.md). - -## License - -[Apache License 2.0](./LICENSE). From a9e5cbb4ff003fc29168663bb2a4a4bf243813aa Mon Sep 17 00:00:00 2001 From: cjm98332 Date: Tue, 15 Aug 2023 14:36:21 -0700 Subject: [PATCH 3/4] deleted comments --- .../src/datahub/ingestion/source/sql/mssql.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql.py index da40ef4355c7b..f848835996cf4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql.py @@ -30,13 +30,18 @@ BasicSQLAlchemyConfig, make_sqlalchemy_uri, ) -from datahub.metadata.schema_classes import BooleanTypeClass, UnionTypeClass, StringTypeClass #CJM08122023 +from datahub.metadata.schema_classes import ( + BooleanTypeClass, + StringTypeClass, + UnionTypeClass, +) logger: logging.Logger = logging.getLogger(__name__) register_custom_type(sqlalchemy.dialects.mssql.BIT, BooleanTypeClass) register_custom_type(sqlalchemy.dialects.mssql.SQL_VARIANT, UnionTypeClass) -register_custom_type(sqlalchemy.dialects.mssql.UNIQUEIDENTIFIER, StringTypeClass) #CJM08122023 +register_custom_type(sqlalchemy.dialects.mssql.UNIQUEIDENTIFIER, StringTypeClass) + class SQLServerConfig(BasicSQLAlchemyConfig): # defaults From 5a6b82aaf97a3699e703295e485b983005b4087a Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Thu, 17 Aug 2023 13:17:00 -0400 Subject: [PATCH 4/4] update tests --- .../golden_mces_mssql_no_db_to_file.json | 12 ++++++++++++ .../golden_mces_mssql_no_db_with_filter.json | 12 ++++++++++++ .../golden_files/golden_mces_mssql_to_file.json | 12 ++++++++++++ .../golden_mces_mssql_with_lower_case_urn.json | 12 ++++++++++++ .../tests/integration/sql_server/setup/setup.sql | 5 +++-- 5 files changed, 51 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json index be4ae9e047aea..7e5586adb1048 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json @@ -1688,6 +1688,18 @@ "recursive": false, "isPartOfKey": true }, + { + "fieldPath": "SomeId", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "UNIQUEIDENTIFIER()", + "recursive": false, + "isPartOfKey": false + }, { "fieldPath": "Name", "nullable": true, diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json index bc81ce9633432..e6dd66a8b8317 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json @@ -1688,6 +1688,18 @@ "recursive": false, "isPartOfKey": true }, + { + "fieldPath": "SomeId", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "UNIQUEIDENTIFIER()", + "recursive": false, + "isPartOfKey": false + }, { "fieldPath": "Name", "nullable": true, diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json index 8be2fe134dca1..8d80b70f407b8 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json @@ -1688,6 +1688,18 @@ "recursive": false, "isPartOfKey": true }, + { + "fieldPath": "SomeId", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "UNIQUEIDENTIFIER()", + "recursive": false, + "isPartOfKey": false + }, { "fieldPath": "Name", "nullable": true, diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json index ba2ab7330fded..149f3ddf8389b 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json @@ -1688,6 +1688,18 @@ "recursive": false, "isPartOfKey": true }, + { + "fieldPath": "SomeId", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "UNIQUEIDENTIFIER()", + "recursive": false, + "isPartOfKey": false + }, { "fieldPath": "Name", "nullable": true, diff --git a/metadata-ingestion/tests/integration/sql_server/setup/setup.sql b/metadata-ingestion/tests/integration/sql_server/setup/setup.sql index 612de3eb1583c..4cc385cfcf3d1 100644 --- a/metadata-ingestion/tests/integration/sql_server/setup/setup.sql +++ b/metadata-ingestion/tests/integration/sql_server/setup/setup.sql @@ -34,7 +34,8 @@ CREATE TABLE Foo.Persons ( GO CREATE TABLE Foo.SalesReason ( - TempID int NOT NULL, + TempID int NOT NULL, + SomeId UNIQUEIDENTIFIER NOT NULL DEFAULT NEWID(), Name nvarchar(50) , CONSTRAINT PK_TempSales PRIMARY KEY NONCLUSTERED (TempID) , CONSTRAINT FK_TempSales_SalesReason FOREIGN KEY (TempID) @@ -60,4 +61,4 @@ EXEC sys.sp_addextendedproperty @level0type = N'SCHEMA', @level0name = 'Foo', @level1type = N'TABLE', @level1name = 'Persons', @level2type = N'COLUMN',@level2name = 'LastName'; -GO \ No newline at end of file +GO