From d129d192cdc189b386489fd60dd27269bb9a1f8a Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Fri, 27 Oct 2023 09:08:23 +0200 Subject: [PATCH] [Docs] - Organize upgrade steps (#13738) --- .../deployment/upgrade/post-upgrade-steps.md | 28 +++ .../upgrade/upgrade-prerequisites.md | 65 ++----- .../v0.13.3/how-to-guides/aws/index.md | 27 --- .../v1.2.x/deployment/upgrade/bare-metal.md | 4 +- .../v1.2.x/deployment/upgrade/docker.md | 50 +---- .../v1.2.x/deployment/upgrade/index.md | 14 +- .../v1.2.x/deployment/upgrade/kubernetes.md | 27 +-- .../deployment/upgrade/versions/012-to-013.md | 179 ------------------ .../deployment/upgrade/versions/013-to-100.md | 120 ------------ .../deployment/upgrade/versions/index.md | 18 +- openmetadata-docs/content/v1.2.x/menu.md | 6 +- 11 files changed, 60 insertions(+), 478 deletions(-) create mode 100644 openmetadata-docs/content/partials/v1.2/deployment/upgrade/post-upgrade-steps.md delete mode 100644 openmetadata-docs/content/v0.13.3/how-to-guides/aws/index.md delete mode 100644 openmetadata-docs/content/v1.2.x/deployment/upgrade/versions/012-to-013.md delete mode 100644 openmetadata-docs/content/v1.2.x/deployment/upgrade/versions/013-to-100.md diff --git a/openmetadata-docs/content/partials/v1.2/deployment/upgrade/post-upgrade-steps.md b/openmetadata-docs/content/partials/v1.2/deployment/upgrade/post-upgrade-steps.md new file mode 100644 index 000000000000..dd64f0196252 --- /dev/null +++ b/openmetadata-docs/content/partials/v1.2/deployment/upgrade/post-upgrade-steps.md @@ -0,0 +1,28 @@ +# Post-Upgrade Steps + +### Reindex + +{% partial file="/v1.2/deployment/reindex.md" /%} + +Since this is required after the upgrade, we want to reindex `All` the entities. + +### (Optional) Update your OpenMetadata Ingestion Client + +If you are running the ingestion workflows **externally** or using a custom Airflow installation, you need to make sure that the Python Client you use is aligned +with the OpenMetadata server version. + +For example, if you are upgrading the server to the version `x.y.z`, you will need to update your client with + +```bash +pip install openmetadata-ingestion[]==x.y.z +``` + +The `plugin` parameter is a list of the sources that we want to ingest. An example would look like this `openmetadata-ingestion[mysql,snowflake,s3]==1.2.0`. +You will find specific instructions for each connector [here](/connectors). + +Moreover, if working with your own Airflow deployment - not the `openmetadata-ingestion` image - you will need to upgrade +as well the `openmetadata-managed-apis` version: + +```bash +pip install openmetadata-managed-apis==x.y.z +``` diff --git a/openmetadata-docs/content/partials/v1.2/deployment/upgrade/upgrade-prerequisites.md b/openmetadata-docs/content/partials/v1.2/deployment/upgrade/upgrade-prerequisites.md index 1b6249e4c89a..216d99cd7120 100644 --- a/openmetadata-docs/content/partials/v1.2/deployment/upgrade/upgrade-prerequisites.md +++ b/openmetadata-docs/content/partials/v1.2/deployment/upgrade/upgrade-prerequisites.md @@ -1,8 +1,8 @@ -## Prerequisites +# Prerequisites Everytime that you plan on upgrading OpenMetadata to a newer version, make sure to go over all these steps: -### 1. Backup your Metadata +### Backup your Metadata Before upgrading your OpenMetadata version we strongly recommend backing up the metadata. @@ -64,46 +64,7 @@ You can refer to the following guide to get more details about the backup and re {% /inlineCallout %} {% /inlineCalloutContainer %} -### 2. Review the Deprecation Notice and Breaking Changes - -Releases might introduce deprecations and breaking changes that you should be aware of and understand before moving forward. - -Below in this page you will find the details for the latest release, and you can find older release notes [here](/deployment/upgrade/versions). - -The goal is to answer questions like: -- *Do I need to update my configurations?* -- *If I am running connectors externally, did their service connection change?* - -Carefully reviewing this will prevent easy errors. - -### (Optional) 3. Update your OpenMetadata Ingestion Client - -If you are running the ingestion workflows **externally** or using a custom Airflow installation, you need to make sure that the Python Client you use is aligned -with the OpenMetadata server version. - -For example, if you are upgrading the server to the version `x.y.z`, you will need to update your client with - -```bash -pip install openmetadata-ingestion[]==x.y.z -``` - -The `plugin` parameter is a list of the sources that we want to ingest. An example would look like this `openmetadata-ingestion[mysql,snowflake,s3]==1.2.0`. -You will find specific instructions for each connector [here](/connectors). - -Moreover, if working with your own Airflow deployment - not the `openmetadata-ingestion` image - you will need to upgrade -as well the `openmetadata-managed-apis` version: - -```bash -pip install openmetadata-managed-apis==x.y.z -``` - -## Deprecation Notice - -- OpenMetadata only supports Python version 3.8 to 3.10. We will add support for 3.11 in the release 1.3. - -## Breaking Changes for 1.2 Stable Release - -### BEFORE the migration - Update `sort_buffer_size` (MySQL) or `work_mem` (Postgres) +### Update `sort_buffer_size` (MySQL) or `work_mem` (Postgres) Before running the migrations, it is important to update these parameters to ensure there are no runtime errors. A safe value would be setting them to 20MB. @@ -141,27 +102,29 @@ during the migration after bumping this value, you can increase them further. After the migration is finished, you can revert this changes. +# Deprecation Notice + +- OpenMetadata only supports Python version 3.8 to 3.10. We will add support for 3.11 in the release 1.3. +- OpenMetadata version 0.13.x is deprecated. + +# Breaking Changes + ### Version Upgrades -- OpenMetadata now supports **Elasticsearch** up to version **8.10.2** and **Opensearch** up to version **2.7** - The OpenMetadata Server is now based on **JDK 17** - -#### Elasticsearch & Opensearch troubleshooting +- OpenMetadata now supports **Elasticsearch** up to version **8.10.2** and **Opensearch** up to version **2.7** There is no direct migration to bump the indexes to the new supported versions. You might see errors like: ``` -"java.lang.IllegalStateException: cannot upgrade a node from version [7.16.3] directly to version [8.5.1], upgrade to version [7.17.0] +"java.lang.IllegalStateException: cannot upgrade a node from version [7.16.3] directly to version [8.5.1] ERROR: Elasticsearch did not exit normally - check the logs at /usr/share/elasticsearch/logs/elasticsearch.log ERROR: Elasticsearch exited unexpectedly ``` In order to move forward, you can remove volumes / delete the indexes directly from your search instances. Note that -OpenMetadata stores everything in the database, so indexes can be recreated directly from the UI. - -{% partial file="/v1.2/deployment/reindex.md" /%} - -Since this is required after the upgrade, we want to reindex `All` the entities. +OpenMetadata stores everything in the database, so indexes can be recreated directly from the UI. We will +show you how in the [Post-Upgrade Steps](/deployment/upgrade#reindex). ### Query Entity diff --git a/openmetadata-docs/content/v0.13.3/how-to-guides/aws/index.md b/openmetadata-docs/content/v0.13.3/how-to-guides/aws/index.md deleted file mode 100644 index d0d83b1cda2f..000000000000 --- a/openmetadata-docs/content/v0.13.3/how-to-guides/aws/index.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -title: How to enable AWS RDS IAM Auth on postgresql -slug: /how-to-guides/aws/index.md ---- - -# Aws resources on Rds IAM Auth -https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.IAMDBAuth.html - -# Requirements -1. AWS Rds Cluster with IAM auth enabled -2. User on Db Cluster with iam enabled -3. IAM policy with permission on rds connect -4. Role with IAM policy attached -5. IAM role attached to ec2 instance on which openmetadata is deployed or ServiceAccount/Kube2Iam role attached to pod - -# How to enable ADS RDS IAM Auth on postgresql - -Set environment variables -```Commandline - AWS_ENABLE_IAM_DATABASE_AUTHENTICATION: true - AWS_REGION: your_region - DB_PARAMS: "allowPublicKeyRetrieval=true&sslmode=require&serverTimezone=UTC" -``` -Either through helm (if deployed in kubernetes) or as env vars - -# Note -The `DB_USER_PASSWORD` is still required and cannot be empty. Set it to a random/dummy string. diff --git a/openmetadata-docs/content/v1.2.x/deployment/upgrade/bare-metal.md b/openmetadata-docs/content/v1.2.x/deployment/upgrade/bare-metal.md index e77964a18c57..51fb9a9acfff 100644 --- a/openmetadata-docs/content/v1.2.x/deployment/upgrade/bare-metal.md +++ b/openmetadata-docs/content/v1.2.x/deployment/upgrade/bare-metal.md @@ -88,6 +88,4 @@ binaries. You may restart the server by running the following command. ./bin/openmetadata.sh start ``` -## Step 7: Re-index all your metadata - -{% partial file="/v1.2/deployment/reindex.md" /%} +{% partial file="/v1.2/deployment/upgrade/post-upgrade-steps.md" /%} diff --git a/openmetadata-docs/content/v1.2.x/deployment/upgrade/docker.md b/openmetadata-docs/content/v1.2.x/deployment/upgrade/docker.md index 675c332e68ac..6cd79e089c93 100644 --- a/openmetadata-docs/content/v1.2.x/deployment/upgrade/docker.md +++ b/openmetadata-docs/content/v1.2.x/deployment/upgrade/docker.md @@ -20,7 +20,7 @@ Below we have highlighted the steps needed to upgrade to the latest version with # Upgrade Process -## Step 1: Replace the docker compose file +## Replace the docker compose file - Stop the running compose deployment with below command ``` @@ -40,54 +40,10 @@ Please make sure to go through [breaking changes and release highlights](/deploy docker compose -f docker-compose.yml up -d ``` -## Step 2: Re-index all your metadata +{% partial file="/v1.2/deployment/upgrade/post-upgrade-steps.md" /%} -{% partial file="/v1.2/deployment/reindex.md" /%} ---- - -## Guide for Upgrading ingestion patch versions - -During the release lifespan we may publish new patch versions of `openmetadata-ingestion`. If you deployed -the ingestion container and require one of the fixes or improvements from a new patch release, there's usually no need -to re-deploy the full ingestion container. - -{% note %} - -Note that this process will only work if we are moving from PATCH versions. For example: `0.13.1.1` -> `0.13.1.2`. - -This method won't work when upgrading from `0.13.1.X` -> `0.13.2.X`, as that will also require to upgrade the -server version. - -{% /note %} - -The steps to follow are: - -- Connect to the ingestion container. If using our docker compose files or `metadata docker` CLI, this translates to - ``` - docker exec -it openmetadata_ingestion bash - ``` -- Validate your `metadata` version via ```metadata --version```. You will get back something like: - ``` - metadata 0.13.1.5 from /home/airflow/.local/lib/python3.9 (python 3.9) - ``` -- Upgrade the `openmetadata-ingestion` package via ```pip install "openmetadata-ingestion==0.13.1.X"```,for example, - ``` - pip install "openmetadata-ingestion==0.13.1.7" - ``` - You can find the list of all released versions of - the `openmetadata-ingestion` package [here](https://pypi.org/project/openmetadata-ingestion/#history). -- Exit the container by typing `exit`. -- Restart the ingestion container with `docker restart openmetadata_ingestion`. This will need a few minutes to - to stop the container and start it again. Now, Airflow will start with the upgraded `metadata` version. -- Connect to the ingestion container and validate the `metadata` version: - ``` - docker exec -it openmetadata_ingestion bash - ``` - - ```metadata version```: where we expect to get the same version that was previously installed. - - -## Troubleshooting +# Troubleshooting #### Permission Denied when running ```metadata openmetadata-imports-migration``` If you have a `Permission Denied` error thrown when running ```metadata openmetadata-imports-migration --change-config-file-path``` you might need to change the permission on the `/opt/airflow/dags` folder. SSH into the ingestion container and check the permission on the folder running the below commands diff --git a/openmetadata-docs/content/v1.2.x/deployment/upgrade/index.md b/openmetadata-docs/content/v1.2.x/deployment/upgrade/index.md index 624274b68a44..cd0a31ee7c0e 100644 --- a/openmetadata-docs/content/v1.2.x/deployment/upgrade/index.md +++ b/openmetadata-docs/content/v1.2.x/deployment/upgrade/index.md @@ -5,18 +5,20 @@ slug: /deployment/upgrade # Upgrade OpenMetadata +In this guide, you will find all the necessary information to safely upgrade your OpenMetadata instance to 1.2.x. + {% partial file="/v1.2/deployment/upgrade/upgrade-prerequisites.md" /%} -## Upgrade your installation +# Upgrade your installation -Once your metadata is safe, follow the required upgrade instructions: +Once your metadata is safe, follow the required upgrade instructions based on your environment: {% inlineCalloutContainer %} {% inlineCallout color="violet-70" icon="fit_screen" bold="Upgrade a Kubernetes Deployment" - href="/deployment/upgrade/kubernetes" %} + href="/deployment/upgrade/kubernetes#upgrade-process" %} Upgrade your Kubernetes installation {% /inlineCallout %} @@ -24,7 +26,7 @@ Once your metadata is safe, follow the required upgrade instructions: color="violet-70" icon="celebration" bold="Upgrade a Docker Deployment" - href="/deployment/upgrade/docker" %} + href="/deployment/upgrade/docker#upgrade-process" %} Upgrade your Docker installation {% /inlineCallout %} @@ -32,7 +34,9 @@ Once your metadata is safe, follow the required upgrade instructions: color="violet-70" icon="storage" bold="Upgrade a Bare Metal Deployment" - href="/deployment/upgrade/bare-metal" %} + href="/deployment/upgrade/bare-metal#upgrade-process" %} Upgrade your Bare Metal installation {% /inlineCallout %} {% /inlineCalloutContainer %} + +{% partial file="/v1.2/deployment/upgrade/post-upgrade-steps.md" /%} diff --git a/openmetadata-docs/content/v1.2.x/deployment/upgrade/kubernetes.md b/openmetadata-docs/content/v1.2.x/deployment/upgrade/kubernetes.md index 18d84ce1da05..571b3642fc47 100644 --- a/openmetadata-docs/content/v1.2.x/deployment/upgrade/kubernetes.md +++ b/openmetadata-docs/content/v1.2.x/deployment/upgrade/kubernetes.md @@ -93,11 +93,9 @@ You can learn more about how the migration process works [here](/deployment/upgr {% /note %} -## Step 5: Re-index all your metadata +{% partial file="/v1.2/deployment/upgrade/post-upgrade-steps.md" /%} -{% partial file="/v1.2/deployment/reindex.md" /%} - -## Troubleshooting +# Troubleshooting ### Helm Upgrade fails with additional property airflow not allowed @@ -168,27 +166,6 @@ openmetadata: Run the [helm lint](https://helm.sh/docs/helm/helm_lint/) command on your custom values after making the changes to validate with the JSON Schema. -### With 0.13.0 Release - -If your helm dependencies upgrade fails with the below command result - - -``` -Error: UPGRADE FAILED: cannot patch "mysql" with kind StatefulSet: StatefulSet.apps "mysql" is invalid: spec: -Forbidden: updates to statefulset spec for fields other than 'replicas', 'template', 'updateStrategy', 'persistentVolumeClaimRetentionPolicy' -and 'minReadySeconds' are forbidden -``` - -This is probably because with `0.13.0`, we have **default size of mysql persistence set to 50Gi**. - -Kubernetes does not allow changes to Persistent volume with helm upgrades. - -In order to work around this issue, you can either default the persistence size to 8Gi or run the below command which will patch Persistent Volumes and Persistent Volume Claims for mysql helm and then run the above `helm upgrade` command. - -``` -kubectl patch pvc data-mysql-0 -p '{"spec":{"resources":{"requests":{"storage":"50Gi"}}}}' -kubectl patch pv -p '{"spec":{"storage":"50Gi"}}' -``` - ### MySQL Pod fails on Upgrade {% note %} diff --git a/openmetadata-docs/content/v1.2.x/deployment/upgrade/versions/012-to-013.md b/openmetadata-docs/content/v1.2.x/deployment/upgrade/versions/012-to-013.md deleted file mode 100644 index a4fa899c26b3..000000000000 --- a/openmetadata-docs/content/v1.2.x/deployment/upgrade/versions/012-to-013.md +++ /dev/null @@ -1,179 +0,0 @@ ---- -title: Upgrade 0.12 to 0.13 -slug: /deployment/upgrade/versions/012-to-013 ---- - -# Upgrade from 0.12 to 0.13 - -Upgrading from 0.12 to 0.13 can be done directly on your instances. This page will list few general details you should take into consideration when running the upgrade. - -## 0.13.2 Highlights - -OpenMetadata 0.13.2 is a stable release. Please check the [release notes](https://github.com/open-metadata/OpenMetadata/releases/tag/0.13.1-release) - -If you are upgrading production this is the recommended version to upgrade. - -### Service Connection Updates - -- Oracle: - - Oracle connector now supports thick mode enabled by default. - - Added: `instantClientDirectory` with the path to the Oracle libraries. They are provided by default for release 19 - in the ingestion container. - -- Superset: - - The Superset connector now supports the ability to extract metadata directly from the database underlying Superset, making it useful for users who have SSO enabled. - - Added: `connection` field the connection can be of type `ApiConnection`, `postgresConnection` or `mysqlConnection` - - Removed: `username`, `password` & `provider` fields as now these fields will be part of `ApiConnection` - -## Breaking Changes for 0.13.2 Stable Release - -### EntityName -To better manage and harmonize `entityName` value and allow users to form better expectations around these values the team introduced an enforcement of the `entityName` format using regex pattern. - -All the OpenMetadata entities `entityName` fields will enforce by the default the following regex pattern: -- `^[\w'\- .&]+$`: match any word characters (equivalent to `[a-zA-Z0-9_]`) or the characters `'- .&`. For example when creating a pipeline service the following name will not be allowed `MyPipelineIngestion?!` while the following will be `My-Pipeline `. - -Some entities are enforcing specific patterns: -- Users: `([\w\-.]|[^@])+$` match any word characters (equivalent to `[a-zA-Z0-9_]`) or the characters `-.` and don't match the character `@` -- Webhook: `^[\w'\-.]+$` match any word characters (equivalent to `[a-zA-Z0-9_]`) or the characters `-.` -- Table: `^[\w'\- ./]+$` match any word characters (equivalent to `[a-zA-Z0-9_]`) or the characters `'-. /` -- Location: `^[\w'\-./]+$` match any word characters (equivalent to `[a-zA-Z0-9_]`) or the characters `-.'/` -- Type: `^[a-z][\w]+$` match any word characters (equivalent to `[a-zA-Z0-9_]`) starting with a lowercase letter (e.g. `tHisChar` will match, `ThisChar` will not) - -If an entity name does not follow the pattern an error will be returned by the OpenMetadata platform. - -**The change should be transparent for the end user and no action should be required.** - -## EntityLink -Similar to the implementation done for `entityName`, `entityLink` will now enforce a specific pattern. The structure of `entityLink` is in the form `<#E::{entities}::{entityType}::{field}::{fieldName}::{fieldValue}>` - -All the OpenMetadata entities `entityLink` fields will enforce by the default the following regex pattern: -- `^<#E::\w+::[\w'\- .&/\:+\"\\]+>$`: this means that `{entities}` value needs to match any word characters (equivalent to `[a-zA-Z0-9_]`) and the part after `{entities}` can match any word characters (equivalent to `[a-zA-Z0-9_]`) or the characters `'- .&/\:+\"\` - -If an entity name does not follow the pattern an error will be returned by the OpenMetadata platform. - -**The change should be transparent for the end user and no action should be required.** - -### Tags API - -Tags APIs were coded long before other entities' APIs were added. Due to this, tag API does not follow the API convention -that all the other entities are following. This issue makes backward incompatible changes to follow the same convention as glossaries. - -You can find the full list of API paths changes in the following [issue](https://github.com/open-metadata/OpenMetadata/issues/9259). - -### Metabase and Domo Dashboards `name` - -With the new restrictions on the `EntityName` and to ensure unicity of the assets, the **Metabase** and **Domo Dashboard** sources -now ingest the `name` of the charts and dashboards with their internal ID value. Their `name` value will be used -as the display name, but not as the OpenMetadata `name` anymore. - -The recommended approach here is to create a new service and ingest the metadata again. If you ingest from the same -service, the assets in there will end up being duplicated, as the `name` in OpenMetadata is used to identify each asset. - -Let us know if you have any questions around this topic. - -### Ingestion Framework sources directory structure - -We have converted source modules (e.g., `redshift.py`) into packages containing all the necessary information (e.g., `redshift/metadata.py`). -This has helped us reorganise functionalities and easily focus on each connector independently. - -If you're extending any of the sources, you'll need to update your imports. You can take a look at the new -structure [here](https://github.com/open-metadata/OpenMetadata/tree/main/ingestion/src/metadata/ingestion/source). - -### MySQL Helm Chart Version Reverted to 8.8.23 - -OpenMetadata Helm Chart Release with Application Version `0.13.2` updates the Bitnami MySQL Helm Chart version from `9.2.1` to `8.8.23`. This is a breaking change and users will face an issue as mentioned in the documentation [here](/deployment/upgrade/kubernetes#mysql-pod-fails-on-upgrade). Please note that OpenMetadata Dependencies Helm Chart is not recommended for production use cases. The steps mentioned in the section will help you fix the issue. - ---- - -## 0.13.1 Highlights - -OpenMetadata 0.13.1 is a stable release. Please check the [release notes](https://github.com/open-metadata/OpenMetadata/releases/tag/0.13.1-release). - -### Deprecating botPrincipals from OpenMetadata Configuration - -Starting with `0.13.0`, we have deprecated the initial configurations for Authorizer Bot Principals. This means that all -the configs will be taken care from UI once it's up and running under settings >> bots section. The default once available -and enabled will be `ingestion-bot` which will have JWT Token configured. You can edit the same for configuring service -principals instead of using the default JWT Auth mechanism. For more information on configuring Bots, you can -visit [Enable Security](/deployment/security). - -### Service Connection Updates - -- Databricks - - Databricks connector now supports usage and lineage extraction. - - Added: `supportsUsageExtraction`, `supportsLineageExtraction` - -- Dagster - - Dagster connector now support fetching metadata from dagster cloud. - - Added: `configSource` - - Removed: `hostPort`, `numberOfStatus` - -- Airbyte - - Airbyte connector now support Basic Authentication. - - Added: `username`, `password` - -### Lineage Backend - -- The import for the Airflow Lineage Backend has been updated from `airflow_provider_openmetadata.lineage.openmetadata.OpenMetadataLineageBackend` -to `airflow_provider_openmetadata.lineage.backend.OpenMetadataLineageBackend`. -- We removed support from Airflow v1. -- The failure callback now only updates the pipeline status if the Pipeline already exists in OpenMetadata. - -### Webhooks - -OpenMetadata will be deprecating the existing webhooks for Slack, MSTeams. - -Before upgrading to 0.13.1 it is recommended to save the existing Webhook configs(like webhook url) to use them later. - -We have added Alerts/Notifications , which can be configured to receive customised alerts on updates in OM using Triggers, Filtering Information to different destinations like Slack, MsTeams or even Emails. -Please use the same webhook config that you had saved from previous version to configure the Alerts Destination after upgrading. - -### dbt Workflow - -- Starting with `0.13.1` dbt ingestion has been separated from the metadata ingestion. It can now be configured as a separate workflow after completing the metadata ingestion workflow -- We will remove the dbt configuration from your existing metadata ingestion pipelines and they will keep working as expected. -- After upgrading you will have to create the dbt workflow for the dbt ingestion to start working again. -- dbt workflow can be configured via going to `services->selecting your service->ingestion tab->Add dbt Ingestion` - -## Breaking Changes for 0.13.1 Stable Release - -OpenMetadata Release 0.13.1 introduces below breaking changes - - -### Webhooks - -Starting from 0.13.1 , OpenMetadata will be deprecating the existing webhooks for Slack, MSTeams. - -Before upgrading to 0.13.1 it is recommended to save the existing Webhook configs(like webhook url) to use them later. - -We have added Alerts/Notifications , which can be configured to receive customised alerts on updates in OM using Triggers, Filtering Information to different destinations like Slack, MsTeams or even Emails. -Please use the same webhook config that you had saved from previous version to configure the Alerts Destination after upgrading. - - -OpenMetadata Release 0.13.x introduces below breaking changes: - -### Docker Volumes - -OpenMetadata Release 0.13.x introduces Default Docker Volumes for Database (MySQL, PostgreSQL) and ElasticSearch with Docker deployment. - -- If you are looking for the fresh deployment of 0.13.x - [here](https://docs.open-metadata.org/deployment/docker) -- If you are looking for upgrading of the new version i.e 0.13.x - [here](https://docs.open-metadata.org/deployment/upgrade/docker) - -### MySQL Helm Chart Version Updated to 9.2.1 - -OpenMetadata Helm Chart Release with Application Version `0.13.1` updates the Bitnami MySQL Helm Chart version to `9.2.1` from `8.8.23`. This is not a breaking change but existing user's trying to upgrade will experience a slight delay in OpenMetadata Dependencies Helm Chart Upgrades as it pulls new docker image for MySQL. Please note that OpenMetadata Dependencies Helm Chart is not recommended for production use cases. Please follow the [kubernetes deployment](/deployment/kubernetes) for new installation or [upgrade kubernetes](/deployment/upgrade/kubernetes) for upgrading OpenMetadata in Kubernetes. - -### dbt Workflow - -dbt ingestion has been separated from the metadata ingestion. It can now be configured as a separate workflow after completing the metadata ingestion workflow. - -We will remove the dbt configuration from your existing metadata ingestion pipelines and they will keep working as expected. - -After upgrading you will have to create the dbt workflow for the dbt ingestion to start working again. - -### Airflow Lineage Backend - -- The import for the Airflow Lineage Backend has been updated from `airflow_provider_openmetadata.lineage.openmetadata.OpenMetadataLineageBackend` - to `airflow_provider_openmetadata.lineage.backend.OpenMetadataLineageBackend`. -- We removed support from Airflow v1. -- The failure callback now only updates the pipeline status if the Pipeline already exists in OpenMetadata. diff --git a/openmetadata-docs/content/v1.2.x/deployment/upgrade/versions/013-to-100.md b/openmetadata-docs/content/v1.2.x/deployment/upgrade/versions/013-to-100.md deleted file mode 100644 index 18a9b0e5d88d..000000000000 --- a/openmetadata-docs/content/v1.2.x/deployment/upgrade/versions/013-to-100.md +++ /dev/null @@ -1,120 +0,0 @@ ---- -title: Upgrade 0.13 to 1.0 -slug: /deployment/upgrade/versions/013-to-100 ---- - -# Upgrade from 0.13 to 1.0 - -Upgrading from 0.13 to 1.0 can be done directly on your instances. This page will list few general details you should take into consideration when running the upgrade. - -## 1.0.0 Highlights - -### Service Connection Updates - -- Postgres: - - The database field for `Postgres` connector has been made a required field as initial database is required to connect to Postgres. - - `Ingest All Database` field has been added since earlier when database field was empty we were ingesting all the databases but since now we have made the database field required you need to enable `Ingest All Database` flag in case you want to ingest all the available database in OpenMetadata. - -- BigQuery: - - Removed: `partitionQueryDuration`, `partitionQuery` & `partitionField` - - since these fields are not used during the ingestion and has already being moved to profiler hence we have removed these fields. - -- Salesforce: - - Removed: `scheme`, `hostPort`, `supportsProfiler` & `supportsQueryComment` - - These fields are related to a SQLAlchemy based connector and since `Salesforce` is based on APIs and not SQLAlchemy, these fields has been removed - -- Superset: - - Removed: `connectionOptions` - - This field was not being used anywhere, hence removed. - -## Breaking Changes for 1.0 Stable Release - -### JWT Authentication Public Keys URL Change - -With Release 1.0.0, JWT Authentication endpoint needs to be updated from `{OPENMETADATA_HOST_NAME}/api/v1/config/jwks` to `{OPENMETADATA_HOST_NAME}/api/v1/system/config/jwks`. This is required as part of [API Endpoints](/deployment/upgrade#api-endpoint-changes). The Environment variable name is `AUTHENTICATION_PUBLIC_KEYS`. It expects list of URLs. One of the URL will be for OpenMetadata JWT Authentication Endpoint. - -### Airflow Configuration & Pipeline Service Client - -The new section on the `openmetadata.yaml` configuration for the Pipeline Service Client has been updated. - -We now have a `pipelineServiceClientConfiguration`, instead of the old [airflowConfiguration](https://github.com/open-metadata/OpenMetadata/blob/0.13.3/conf/openmetadata.yaml#L214). - -```yaml -pipelineServiceClientConfiguration: - className: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"} - apiEndpoint: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://localhost:8080} - metadataApiEndpoint: ${SERVER_HOST_API_URL:-http://localhost:8585/api} - hostIp: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""} - verifySSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"} # Possible values are "no-ssl", "ignore", "validate" - sslConfig: - validate: - certificatePath: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""} # Local path for the Pipeline Service Client - - # Default required parameters for Airflow as Pipeline Service Client - parameters: - username: ${AIRFLOW_USERNAME:-admin} - password: ${AIRFLOW_PASSWORD:-admin} - timeout: ${AIRFLOW_TIMEOUT:-10} -``` - -Most existing environment variables remain the same, except for these three: -- `AIRFLOW_HOST_IP` → `PIPELINE_SERVICE_CLIENT_HOST_IP` -- `AIRFLOW_VERIFY_SSL` → `PIPELINE_SERVICE_CLIENT_VERIFY_SSL` -- `AIRFLOW_SSL_CERT_PATH` → `PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH` - -When upgrading, make sure to update the environment variables and, if working on Bare Metal, make sure to use the updated `openmetadata.yaml`. - -### Deprecation Notice - -- When configuring Bots, **JWT** tokens will be the preferred method of authentication. Any existing SSO-based service accounts -will continue to work on 1.0, but will be fully deprecated on future releases. -- As we added the new Impala connector, We will remove the `impala` scheme from Hive in the next release. - -### API Endpoint Changes -The following endpoints have been renamed in 1.0 - -|Previous Endpoint|New Endpoint| -|---|---| -|`api/v1`|**Removed**| -|`api/v1/services`|**Removed**| -|`api/v1/version`|`api/v1/system/version`| -|`api/v1/util/entities/count`|`api/v1/system/entities/count`| -|`api/v1/util/services/count`|`api/v1/system/services/count`| -|`api/v1/settings`|`api/v1/system/settings`| -|`api/v1/config`|`api/v1/system/config`| -|`api/v1/testSuite`|`api/v1/dataQuality/testSuites`| -|`api/v1/testCase`|`api/v1/dataQuality/testCases`| -|`api/v1/testDefinition`|`api/v1/dataQuality/testDefinitions`| -|`api/v1/automations/workflow`|`api/v1/automations/workflows`| -|`api/v1/events/subscription`|`api/v1/events/subscriptions`| -|`api/v1/analytic/reportData`|`api/v1/analytics/dataInsights/data`| -|`api/v1/analytics/webAnalyticEvent/`|`api/v1/analytics/web/events/`| -|`api/v1/indexResource/reindex`|`api/v1/search/reindex`| -|`api/v1/indexResource/reindex/status/{runMode}`|`api/v1/search/reindex/status/{runMode}`| - -### Sample Data Deprecation - -The `SampleData` service has been deprecated. It is now a `CustomConnector`. If you have some entities in `SampleData`, please DELETE the service if you don’t want to keep them, or we can help you migrate them to a Custom Connector. - -Note that this service type was mostly used on quickstarts and tests to add some example assets into OpenMetadata. This should be transparent for most of the users. - -### Location Entity - -We are deprecating the `Location` Entity in favor of the Containers and new Storage Service: -- Dropping the `location_entity` table, -- Removing the `Location` APIs. - -If you did not have any custom implementation, this was partially used in the Glue Database connector. However, the information was not being actively shown. - -If you had custom implementations on top of the `Location` APIs, reach out to us, and we can help migrate to the new Storage Services. - -### AWS Connectors - -The `endpointURL` property is now formatted as a proper URI, e.g., `http://something.com`. If you have added this configuration -in your connectors, please update the `endpointURL` format with the right scheme. - -Note that this property is OPTIONAL, and for the most cases it will either be left blank or already configured with the right format for it to work properly, e.g., `s3://...`. - -### Python SDK Submodules name change -- **`metadata.test_suite.*`**: this submodule has been renamed `metadata.data_quality.*`. You can view the full change [here](https://github.com/open-metadata/OpenMetadata/pull/10890/files) -- **`metadata.orm_profiler.*`**: this submodule has been renamed `metadata.profiler.*`. You can view the full change [here](https://github.com/open-metadata/OpenMetadata/pull/10350/files) \ No newline at end of file diff --git a/openmetadata-docs/content/v1.2.x/deployment/upgrade/versions/index.md b/openmetadata-docs/content/v1.2.x/deployment/upgrade/versions/index.md index 542bd2ade60d..015702a898ef 100644 --- a/openmetadata-docs/content/v1.2.x/deployment/upgrade/versions/index.md +++ b/openmetadata-docs/content/v1.2.x/deployment/upgrade/versions/index.md @@ -11,22 +11,8 @@ You can find further information about specific version changes in the following {% inlineCallout color="violet-70" icon="10k" - bold="Upgrade to 0.13" - href="/deployment/upgrade/versions/012-to-013" %} - Upgrade from 0.12 to 0.13 - {% /inlineCallout %} - {% inlineCallout - color="violet-70" - icon="10k" - bold="Upgrade to 1.0" - href="/deployment/upgrade/versions/013-to-100" %} - Upgrade from 0.13 to 1.0 - {% /inlineCallout %} - {% inlineCallout - color="violet-70" - icon="10k" - bold="Upgrade to 1.1" + bold="Upgrade to 1.1.x" href="/deployment/upgrade/versions/100-to-110" %} - Upgrade from 1.0 to 1.1 + Upgrade from 1.0.x to 1.1.x {% /inlineCallout %} {% /inlineCalloutContainer %} \ No newline at end of file diff --git a/openmetadata-docs/content/v1.2.x/menu.md b/openmetadata-docs/content/v1.2.x/menu.md index 7afdf2ff169d..e04f61326c9c 100644 --- a/openmetadata-docs/content/v1.2.x/menu.md +++ b/openmetadata-docs/content/v1.2.x/menu.md @@ -196,11 +196,7 @@ site_menu: url: /deployment/upgrade/kubernetes - category: Deployment / Upgrade OpenMetadata / Version Changes url: /deployment/upgrade/versions - - category: Deployment / Upgrade OpenMetadata / Version Changes / 0.12 to 0.13 - url: /deployment/upgrade/versions/012-to-013 - - category: Deployment / Upgrade OpenMetadata / Version Changes / 0.13 to 1.0 - url: /deployment/upgrade/versions/013-to-100 - - category: Deployment / Upgrade OpenMetadata / Version Changes / 1.0 to 1.1 + - category: Deployment / Upgrade OpenMetadata / Version Changes / 1.0.x to 1.1.x url: /deployment/upgrade/versions/100-to-110 - category: Deployment / Backup & Restore Metadata