diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000000..127bf0a5f5 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,76 @@ +# Based on https://imfing.github.io/hextra/docs/guide/deploy-site/ +name: Deploy docs + +on: + # Runs on pushes targeting the default branch + push: + branches: ["main"] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. +# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: "pages" + cancel-in-progress: false + +# Default to bash +defaults: + run: + shell: bash + +jobs: + # Build job + build: + runs-on: ubuntu-latest + env: + HUGO_VERSION: 0.138.0 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 # fetch all history for .GitInfo and .Lastmod + submodules: recursive + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.22' + - name: Setup Pages + id: pages + uses: actions/configure-pages@v4 + - name: Setup Hugo + run: | + wget -O ${{ runner.temp }}/hugo.deb https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.deb \ + && sudo dpkg -i ${{ runner.temp }}/hugo.deb + - name: Build with Hugo + env: + # For maximum backward compatibility with Hugo modules + HUGO_ENVIRONMENT: production + HUGO_ENV: production + run: | + cd docs && hugo \ + --gc --minify \ + --baseURL "${{ steps.pages.outputs.base_url }}/" + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: docs/public + + # Deployment job + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8a030e881a..a62b36e9f0 100644 --- a/.gitignore +++ b/.gitignore @@ -152,5 +152,4 @@ dev/cleanup.py .python-version .databricks-login.json -*.out -foo +*.out \ No newline at end of file diff --git a/Makefile b/Makefile index 4b62910e46..caff5f58ea 100644 --- a/Makefile +++ b/Makefile @@ -31,4 +31,12 @@ known: solacc: hatch run python tests/integration/source_code/solacc.py -.PHONY: all clean dev lint fmt test integration coverage known solacc +docs: + cd docs && hugo server --buildDrafts --disableFastRender + +docs-test: + @echo 'Make sure to run `make docs` in another terminal' + cd docs && yarn linkinator http://localhost:1313/ucx/ + + +.PHONY: all clean dev lint fmt test integration coverage known solacc docs docs-test diff --git a/README.md b/README.md index c43c3f4c62..0f06d0b42d 100644 --- a/README.md +++ b/README.md @@ -1,2362 +1,42 @@ -Databricks Labs UCX -=== -![UCX by Databricks Labs](docs/logo-no-background.png) +# UCX by Databricks Labs -The companion for upgrading to Unity Catalog (UC). +

+ + logo + +

-After [installation](#install-ucx), ensure to [trigger](#ensure-assessment-run-command) the [assessment workflow](#assessment-workflow), -so that you'll be able to [scope the migration](docs/assessment.md) and execute the [group migration workflow](#group-migration-workflow). +

+ 🚀 UCX - Unity Catalog Migration Assistant +

-The [README notebook](#readme-notebook), which can be found in the installation folder contains further instructions and explanations of the different ucx workflows & dashboards. -Once the migration is scoped, you can start with the [table migration process](#Table-Migration). +UCX is a command line tool that helps you migrate your Databricks workspace to Unity Catalog. UCX provides a set of commands to help you migrate your tables, dashboards, and notebooks to be UC compatible. -More workflows, like notebook code migration are coming in future releases. - - -UCX also provides a number of command line utilities accessible via `databricks labs ucx`. - -For questions, troubleshooting or bug fixes, please see our [troubleshooting guide](docs/troubleshooting.md) or submit [an issue](https://github.com/databrickslabs/ucx/issues). -See [contributing instructions](CONTRIBUTING.md) to help improve this project. - [![build](https://github.com/databrickslabs/ucx/actions/workflows/push.yml/badge.svg)](https://github.com/databrickslabs/ucx/actions/workflows/push.yml) [![codecov](https://codecov.io/github/databrickslabs/ucx/graph/badge.svg?token=p0WKAfW5HQ)](https://codecov.io/github/databrickslabs/ucx) ![linesofcode](https://aschey.tech/tokei/github/databrickslabs/ucx?category=code) - -* [Databricks Labs UCX](#databricks-labs-ucx) -* [Installation](#installation) - * [Installation requirements](#installation-requirements) - * [Authenticate Databricks CLI](#authenticate-databricks-cli) - * [Install UCX](#install-ucx) - * [Installation resources](#installation-resources) - * [Installation folder](#installation-folder) - * [Readme notebook](#readme-notebook) - * [Debug notebook](#debug-notebook) - * [Debug logs](#debug-logs) - * [Installation configuration](#installation-configuration) - * [Advanced installation options](#advanced-installation-options) - * [Force install over existing UCX](#force-install-over-existing-ucx) - * [Installing UCX on all workspaces within a Databricks account](#installing-ucx-on-all-workspaces-within-a-databricks-account) - * [Installing UCX with company hosted PYPI mirror](#installing-ucx-with-company-hosted-pypi-mirror) - * [Upgrading UCX for newer versions](#upgrading-ucx-for-newer-versions) - * [Uninstall UCX](#uninstall-ucx) -* [Migration process](#migration-process) - * [Table migration process](#table-migration-process) - * [Table mapping](#table-mapping) - * [Step 1 : Create the mapping file](#step-1--create-the-mapping-file) - * [Step 2: Update the mapping file](#step-2-update-the-mapping-file) - * [Data access](#data-access) - * [Step 1 : Map cloud principals to cloud storage locations](#step-1--map-cloud-principals-to-cloud-storage-locations) - * [Step 2 : Create or modify cloud principals and credentials](#step-2--create-or-modify-cloud-principals-and-credentials) - * [Step 3: Create the "uber" Principal](#step-3-create-the-uber-principal) - * [New Unity Catalog resources](#new-unity-catalog-resources) - * [Step 0: Attach a metastore](#step-0-attach-a-metastore) - * [Step 1: Create external Locations](#step-1-create-external-locations) - * [Step 2: Create Catalogs and Schemas](#step-2-create-catalogs-and-schemas) - * [Migrate Hive metastore data objects](#migrate-hive-metastore-data-objects) - * [Odds and Ends](#odds-and-ends) - * [Skip migrating schemas, tables or views](#skip-migrating-schemas-tables-or-views) - * [Move data objects](#move-data-objects) - * [Alias data objects](#alias-data-objects) - * [Revert migrated data objects](#revert-migrated-data-objects) -* [Workflows](#workflows) - * [Assessment workflow](#assessment-workflow) - * [Group migration workflow](#group-migration-workflow) - * [Table migration workflows](#table-migration-workflows) - * [Migrate tables](#migrate-tables) - * [Migrate external Hive SerDe tables](#migrate-external-hive-serde-tables) - * [Migrate external tables CTAS](#migrate-external-tables-ctas) - * [Post-migration data reconciliation workflow](#post-migration-data-reconciliation-workflow) - * [[LEGACY] Scan tables in mounts Workflow](#legacy-scan-tables-in-mounts-workflow) - * [Always run this workflow AFTER the assessment has finished](#balways-run-this-workflow-after-the-assessment-has-finishedb) - * [[LEGACY] Migrate tables in mounts Workflow](#legacy-migrate-tables-in-mounts-workflow) - * [[EXPERIMENTAL] Migration Progress Workflow](#experimental-migration-progress-workflow) -* [Dashboards](#dashboards) -* [Linter message codes](#linter-message-codes) - * [`cannot-autofix-table-reference`](#cannot-autofix-table-reference) - * [`catalog-api-in-shared-clusters`](#catalog-api-in-shared-clusters) - * [`changed-result-format-in-uc`](#changed-result-format-in-uc) - * [`direct-filesystem-access-in-sql-query`](#direct-filesystem-access-in-sql-query) - * [`direct-filesystem-access`](#direct-filesystem-access) - * [`dependency-not-found`](#dependency-not-found) - * [`jvm-access-in-shared-clusters`](#jvm-access-in-shared-clusters) - * [`legacy-context-in-shared-clusters`](#legacy-context-in-shared-clusters) - * [`not-supported`](#not-supported) - * [`notebook-run-cannot-compute-value`](#notebook-run-cannot-compute-value) - * [`python-udf-in-shared-clusters`](#python-udf-in-shared-clusters) - * [`rdd-in-shared-clusters`](#rdd-in-shared-clusters) - * [`spark-logging-in-shared-clusters`](#spark-logging-in-shared-clusters) - * [`sql-parse-error`](#sql-parse-error) - * [`sys-path-cannot-compute-value`](#sys-path-cannot-compute-value) - * [`table-migrated-to-uc`](#table-migrated-to-uc) - * [`to-json-in-shared-clusters`](#to-json-in-shared-clusters) - * [`unsupported-magic-line`](#unsupported-magic-line) -* [Utility commands](#utility-commands) - * [`logs` command](#logs-command) - * [`ensure-assessment-run` command](#ensure-assessment-run-command) - * [`update-migration-progress` command](#update-migration-progress-command) - * [`repair-run` command](#repair-run-command) - * [`workflows` command](#workflows-command) - * [`open-remote-config` command](#open-remote-config-command) - * [`installations` command](#installations-command) - * [`report-account-compatibility` command](#report-account-compatibility-command) - * [`export-assessment` command](#export-assessment-command) -* [Metastore related commands](#metastore-related-commands) - * [`show-all-metastores` command](#show-all-metastores-command) - * [`assign-metastore` command](#assign-metastore-command) - * [`create-ucx-catalog` command](#create-ucx-catalog-command) -* [Table migration commands](#table-migration-commands) - * [`principal-prefix-access` command](#principal-prefix-access-command) - * [Access for AWS S3 Buckets](#access-for-aws-s3-buckets) - * [Access for Azure Storage Accounts](#access-for-azure-storage-accounts) - * [`create-missing-principals` command (AWS Only)](#create-missing-principals-command-aws-only) - * [`delete-missing-principals` command (AWS Only)](#delete-missing-principals-command-aws-only) - * [`create-uber-principal` command](#create-uber-principal-command) - * [`migrate-credentials` command](#migrate-credentials-command) - * [`validate-external-locations` command](#validate-external-locations-command) - * [`migrate-locations` command](#migrate-locations-command) - * [`create-table-mapping` command](#create-table-mapping-command) - * [`skip` command](#skip-command) - * [`unskip` command](#unskip-command) - * [`create-catalogs-schemas` command](#create-catalogs-schemas-command) - * [`assign-owner-group` command](#assign-owner-group-command) - * [`migrate-tables` command](#migrate-tables-command) - * [`revert-migrated-tables` command](#revert-migrated-tables-command) - * [`move` command](#move-command) - * [`alias` command](#alias-command) -* [Code migration commands](#code-migration-commands) - * [`lint-local-code` command](#lint-local-code-command) - * [`migrate-local-code` command](#migrate-local-code-command) - * [`migrate-dbsql-dashboards` command](#migrate-dbsql-dashboards-command) - * [`revert-dbsql-dashboards` command](#revert-dbsql-dashboards-command) -* [Cross-workspace installations](#cross-workspace-installations) - * [`sync-workspace-info` command](#sync-workspace-info-command) - * [`manual-workspace-info` command](#manual-workspace-info-command) - * [`create-account-groups` command](#create-account-groups-command) - * [`validate-groups-membership` command](#validate-groups-membership-command) - * [`validate-table-locations` command](#validate-table-locations-command) - * [`cluster-remap` command](#cluster-remap-command) - * [`revert-cluster-remap` command](#revert-cluster-remap-command) - * [`upload` command](#upload-command) - * [`download` command](#download-command) - * [`join-collection` command](#join-collection-command) - * [collection eligible command](#collection-eligible-command) -* [Common Challenges and the Solutions](#common-challenges-and-the-solutions) - * [Network Connectivity Issues](#network-connectivity-issues) - * [Insufficient Privileges](#insufficient-privileges) - * [Version Issues](#version-issues) - * [Authentication Issues](#authentication-issues) - * [Multiple Profiles in Databricks CLI](#multiple-profiles-in-databricks-cli) - * [Workspace has an external Hive Metastore (HMS)](#workspace-has-an-external-hive-metastore-hms) - * [Verify the Installation](#verify-the-installation) -* [Star History](#star-history) -* [Project Support](#project-support) - - -# Installation - -UCX installation is covered by this section. - -## Installation requirements - -UCX has the following installation requirements: -- Databricks CLI v0.213 or later. See [instructions](#authenticate-databricks-cli). -- Python 3.10 or later. See [Windows](https://www.python.org/downloads/windows/) instructions. -- Databricks Premium or Enterprise workspace. -- Network access to your Databricks Workspace used for the [installation process](#install-ucx). -- Network access to the Internet for [pypi.org](https://pypi.org) and [github.com](https://github.com) from machine running the installation. -- Databricks Workspace Administrator privileges for the user, that runs the installation. Running UCX as a Service Principal is not supported. -- Account level Identity Setup. See instructions for [AWS](https://docs.databricks.com/en/administration-guide/users-groups/best-practices.html), [Azure](https://learn.microsoft.com/en-us/azure/databricks/administration-guide/users-groups/best-practices), and [GCP](https://docs.gcp.databricks.com/administration-guide/users-groups/best-practices.html). -- Unity Catalog Metastore Created (per region). See instructions for [AWS](https://docs.databricks.com/en/data-governance/unity-catalog/create-metastore.html), [Azure](https://learn.microsoft.com/en-us/azure/databricks/data-governance/unity-catalog/create-metastore), and [GCP](https://docs.gcp.databricks.com/data-governance/unity-catalog/create-metastore.html). -- If your Databricks Workspace relies on an external Hive Metastore (such as AWS Glue), make sure to read [this guide](docs/external_hms_glue.md). -- A PRO or Serverless SQL Warehouse to render the [report](docs/assessment.md) for the [assessment workflow](#assessment-workflow). - -Once you [install UCX](#install-ucx), you can proceed to the [assessment workflow](#assessment-workflow) to ensure -the compatibility of your workspace with Unity Catalog. - -[[back to top](#databricks-labs-ucx)] - -## Authenticate Databricks CLI - -We only support installations and upgrades through [Databricks CLI](https://docs.databricks.com/en/dev-tools/cli/index.html), as UCX requires an installation script run -to make sure all the necessary and correct configurations are in place. Install Databricks CLI on macOS: -![macos_install_databricks](docs/macos_1_databrickslabsmac_installdatabricks.gif) - -Install Databricks CLI on Windows: -![windows_install_databricks.png](docs/windows_install_databricks.png) - -Once you install Databricks CLI, authenticate your current machine to a Databricks Workspace: - -```commandline -databricks auth login --host WORKSPACE_HOST -``` - -To enable debug logs, simply add `--debug` flag to any command. - -[[back to top](#databricks-labs-ucx)] - -## Install UCX - -Install UCX via Databricks CLI: - -```commandline -databricks labs install ucx -``` - -You'll be prompted to select a [configuration profile](https://docs.databricks.com/en/dev-tools/auth.html#databricks-client-unified-authentication) created by `databricks auth login` command. - -Once you install, proceed to the [assessment workflow](#assessment-workflow) to ensure the compatibility of your workspace with UCX. - -The `WorkspaceInstaller` class is used to create a new configuration for Unity Catalog migration in a Databricks workspace. -It guides the user through a series of prompts to gather necessary information, such as selecting an inventory database, choosing -a PRO or SERVERLESS SQL warehouse, specifying a log level and number of threads, and setting up an external Hive Metastore if necessary. -Upon the first installation, you're prompted for a workspace local [group migration strategy](docs/group_name_conflict.md). -Based on user input, the class creates a new cluster policy with the specified configuration. The user can review and confirm the configuration, -which is saved to the workspace and can be opened in a web browser. - -The [`WorkspaceInstallation`](src/databricks/labs/ucx/install.py) manages the installation and uninstallation of UCX in a workspace. It handles -the configuration and exception management during the installation process. The installation process creates dashboards, databases, and jobs. -It also includes the creation of a database with given configuration and the deployment of workflows with specific settings. The installation -process can handle exceptions and infer errors from job runs and task runs. The workspace installation uploads wheels, creates cluster policies, -and wheel runners to the workspace. It can also handle the creation of job tasks for a given task, such as job dashboard tasks, job notebook tasks, -and job wheel tasks. The class handles the installation of UCX, including configuring the workspace, installing necessary libraries, and verifying -the installation, making it easier for users to migrate their workspaces to UCX. -At the end of the installation, the user will be prompted if the current installation needs to join an existing collection (create new collection if none present). -For large organization with many workspaces, grouping workspaces into collection helps in managing UCX migration at collection level (instead of workspaces level) -User should be an account admin to be able to join a collection. - -After this, UCX will be installed locally and a number of assets will be deployed in the selected workspace. -These assets are available under the installation folder, i.e. `/Applications/ucx` is the default installation folder. Please check [here](#advanced-force-install-over-existing-ucx) for more details. - -You can also install a specific version by specifying it like `@v0.13.2` - `databricks labs install ucx@v0.13.2`. - -![macos_install_ucx](docs/macos_2_databrickslabsmac_installucx.gif) - -[[back to top](#databricks-labs-ucx)] - -## Installation resources - -The following resources are installed by UCX: - -| Installed UCX resources | Description | -|---------------------------------------------------|--------------------------------------------------------------------------------------------------| -| [Inventory database](./docs/table_persistence.md) | A Hive metastore database/schema in which UCX persist inventory required for the upgrade process | -| [Workflows](#workflows) | Workflows to execute UCX | -| [Dashboards](#dashboards) | Dashboards to visualize UCX outcomes | -| [Installation folder](#installation-folder) | A workspace folder containing UCX files in `/Applications/ucx/`. | - -## Installation folder - -UCX is in installed in the workspace folder `/Applications/ucx/`. This folder contains UCX's code resources, like the -[source code](./src) from this GitHub repository and the [dashboard](#dashboards). Generally, these resources are not -*directly* used by UCX users. Resources that can be of importance to users are detailed in the subsections below. - -### Readme notebook - -![readme](docs/readme-notebook.png) - -Every installation creates a `README` notebook with a detailed description of all deployed workflows and their tasks, -providing quick links to the relevant workflows and dashboards. - -[[back to top](#databricks-labs-ucx)] - -### Debug notebook - -![debug](docs/debug-notebook.png) - -Every installation creates a `DEBUG` notebook, that initializes UCX as a library for you to execute interactively. - -[[back to top](#databricks-labs-ucx)] - -### Debug logs - -![debug](docs/debug-logs.png) - -The [workflow](#workflows) runs store debug logs in the `logs` folder of the installation folder. The logs are flushed -every minute in a separate file. Debug logs for [the command-line interface](#authenticate-databricks-cli) are shown -by adding the `--debug` flag: - -```commandline -databricks --debug labs ucx -``` - -[[back to top](#databricks-labs-ucx)] - -### Installation configuration - -In the installation folder, the UCX configuration is kept. - -## Advanced installation options - -Advanced installation options are detailed below. - -### Force install over existing UCX -Using an environment variable `UCX_FORCE_INSTALL` you can force the installation of UCX over an existing installation. -The values for the environment variable are 'global' and 'user'. - -Global Install: When UCX is installed at '/Applications/ucx' -User Install: When UCX is installed at '/Users//.ucx' - -If there is an existing global installation of UCX, you can force a user installation of UCX over the existing installation by setting the environment variable `UCX_FORCE_INSTALL` to 'global'. - -At this moment there is no global override over a user installation of UCX. As this requires migration and can break existing installations. - - -| global | user | expected install location | install_folder | mode | -|--------|------|---------------------------|---------------------|-------------------------------------------------| -| no | no | default | `/Applications/ucx` | install | -| yes | no | default | `/Applications/ucx` | upgrade | -| no | yes | default | `/Users/X/.ucx` | upgrade (existing installations must not break) | -| yes | yes | default | `/Users/X/.ucx` | upgrade | -| yes | no | **USER** | `/Users/X/.ucx` | install (show prompt) | -| no | yes | **GLOBAL** | ... | migrate | - - -* `UCX_FORCE_INSTALL=user databricks labs install ucx` - will force the installation to be for user only -* `UCX_FORCE_INSTALL=global databricks labs install ucx` - will force the installation to be for root only - - -[[back to top](#databricks-labs-ucx)] - -### Installing UCX on all workspaces within a Databricks account -Setting the environment variable `UCX_FORCE_INSTALL` to 'account' will install UCX on all workspaces within a Databricks account. - -* `UCX_FORCE_INSTALL=account databricks labs install ucx` - -After the first installation, UCX will prompt the user to confirm whether to install UCX on the remaining workspaces with the same answers. If confirmed, the remaining installations will be completed silently. - -This installation mode will automatically select the following options: -* Automatically create and enable HMS lineage init script -* Automatically create a new SQL warehouse for UCX assessment - -[[back to top](#databricks-labs-ucx)] - -### Installing UCX with company hosted PYPI mirror - -Some enterprise block the public PYPI index and host a company controlled PYPI mirror. To install UCX while using a -company hosted PYPI mirror for finding its dependencies, add all UCX dependencies to the company hosted PYPI mirror (see -"dependencies" in [`pyproject.toml`](./pyproject.toml)) and set the environment variable `PIP_INDEX_URL` to the company -hosted PYPI mirror URL while installing UCX: - -```commandline -PIP_INDEX_URL="https://url-to-company-hosted-pypi.internal" databricks labs install ucx -``` - -During installation reply *yes* to the question "Does the given workspace block Internet access"? - -[[back to top](#databricks-labs-ucx)] - -## Upgrading UCX for newer versions - -Verify that UCX is installed - -```commandline -databricks labs installed - -Name Description Version -ucx Unity Catalog Migration Toolkit (UCX) -``` - -Upgrade UCX via Databricks CLI: - -```commandline -databricks labs upgrade ucx -``` - -The prompts will be similar to [Installation](#install-ucx) - -![macos_upgrade_ucx](docs/macos_3_databrickslabsmac_upgradeucx.gif) - -[[back to top](#databricks-labs-ucx)] - -## Uninstall UCX - -Uninstall UCX via Databricks CLI: - -```commandline -databricks labs uninstall ucx -``` - -Databricks CLI will confirm a few options: -- Whether you want to remove all ucx artefacts from the workspace as well. Defaults to no. -- Whether you want to delete the inventory database in `hive_metastore`. Defaults to no. - -![macos_uninstall_ucx](docs/macos_4_databrickslabsmac_uninstallucx.gif) - -[[back to top](#databricks-labs-ucx)] - -# Migration process - -On a high level, the steps in migration process are: -1. [assessment](#assessment-workflow) -2. [group migration](#group-migration-workflow) -3. [table migration](#table-migration-process) -4. [data reconciliation](#post-migration-data-reconciliation-workflow) -5. [code migration](#code-migration-commands) - -The migration process can be schematic visualized as: - -```mermaid -flowchart TD - subgraph workspace-admin - assessment --> group-migration - group-migration --> table-migration - table-migration --> code-migration - assessment --> create-table-mapping - create-table-mapping --> table-migration - create-table-mapping --> code-migration - validate-external-locations --> table-migration - assessment --> validate-table-locations - validate-table-locations --> table-migration - table-migration --> revert-migrated-tables - revert-migrated-tables --> table-migration - end - subgraph account-admin - create-account-groups --> group-migration - sync-workspace-info --> create-table-mapping - group-migration --> validate-groups-membership - end - subgraph iam-admin - setup-account-scim --> create-account-groups - assessment --> create-uber-principal - create-uber-principal --> table-migration - assessment --> principal-prefix-access - principal-prefix-access --> migrate-credentials - migrate-credentials --> validate-external-locations - setup-account-scim - end -``` - -[[back to top](#databricks-labs-ucx)] - -## Table migration process - -> You are required to complete the [assessment workflow](#assessment-workflow) before starting the table migration workflow. - -This section explains how to migrate Hive metastore data objects to Unity Catalog. The table migration process consists -of more steps than only a [workflow](#table-migration-workflow), these steps are: - -1. [Table mapping](#table-mapping) : Create a file that maps Hive metastore data objects to Unity Catalog locations. -2. [Data access](#data-access) : Setup identities to access table data. -3. [New Unity Catalog resources](#new-unity-catalog-resources) : Create **new** Unity Catalog resources that do not - require touching existing Hive metastore resources. -4. [Migrate Hive metastore data objects](#migrate-hive-metastore-data-objects) : Migrate Hive metastore data objects to UC. - -### Table mapping - -This section details how to create the table mapping file. This file points existing Hive metastore tables and views to -Unity Catalog locations. When [migrating the tables and views](#migrate-hive-metastore-data-objects), the file is read -to decide where to migrate the tables and views to. - -#### Step 1 : Create the mapping file - -Create the mapping file in the [UCX installation folder](#installation-resources) by running the -[`create-table-mapping` command](#create-table-mapping-command). By default, the file contains all the Hive metastore -tables and views mapped to a single UC catalog, while maintaining the original schema and table names. - -#### Step 2: Update the mapping file - -Edit the mapping file from the previous step to: -- Exclude tables and/or views by removing the lines -- Change UC location by editing the destination catalog and/or schema - -The mapping file is in CSV format and can be edited using any text editor or Excel. If using Excel, save the file in CSV -format. - -Example changes: - -**Before editing** - -| **workspace_name** | **catalog_name** | **src_schema** | **dst_schema** | **src_table** | **dst_table** | -|----------------------|--------------|----------------|----------------|---------------|---------------| -| data_engineering_ws | 123333333 | sales_analysis | sales_analysis | ytd_sales | ytd_sales | - -**After editing** - -| **workspace_name** | **catalog_name** | **src_schema** | **dst_schema** | **src_table** | **dst_table** | -|----------------------|------------------|----------------|----------------|---------------|---------------| -| data_engineering_ws | data_engineering | sales_analysis | sales | ytd_sales | ytd_analysis | - -### Data access - -> Throughout this guide, we refer to IAM roles/instance profiles in AWS & service principals/managed identities in -> as "cloud principals". - -This section creates the cloud principals to access data in Unity Catalog and during the table data during migration. To -understand the motivation for this step, read how Databricks accesses cloud locations: -- [AWS - Create External Locations](https://docs.databricks.com/en/connect/unity-catalog/external-locations.html) -- [Azure - Create External Locations](https://learn.microsoft.com/en-us/azure/databricks/connect/unity-catalog/external-locations) - -#### Step 1 : Map cloud principals to cloud storage locations - -Map the cloud principals to cloud storage by running the -[`principal-prefix-access` command](#principal-prefix-access-command). - -#### Step 2 : Create or modify cloud principals and credentials - -Manually create the cloud principals to access data from Unity Catalog: - -- AWS: - - [`create-missing-principals` command](#create-missing-principals-command-aws-only) creates new AWS roles for Unity - Catalog to access data. - - Or, [Manually create storage credentials](https://docs.databricks.com/en/connect/unity-catalog/storage-credentials.html) -- Azure: - - [Manually create storage Credentials](https://learn.microsoft.com/en-us/azure/databricks/sql/language-manual/sql-ref-storage-credentials) - -Then, run the [`migrate-credentials` command](#migrate-credentials-command) to migrate the cloud principal -credentials to Unity Catalog. - -#### Step 3: Create the "uber" Principal - -Create the "uber" principal by running the [`create-uber-principal` command](#create-uber-principal-command). -The table migration requires a cloud principal with access to table data stored in cloud storage. These tables are known -as external tables. UCX name this cloud principal the "uber" principal as it has access to **all** in-scope cloud -storage. This principal is only required for the table migration process and not for ongoing UC usage. Once the upgrade -is completed, this principal should be deleted. - -### New Unity Catalog resources - -This section details the **new** Unity Catalog resources required for migration the data objects. These resources can be -created without touching the existing Hive metastore objecs. - -#### Step 0: Attach a metastore - -If skipping the [group migration](#group-migration-workflow), then a metastore should be attached to the workspace by -following [these instructions](https://docs.databricks.com/en/data-governance/unity-catalog/create-metastore.html) or running the [`assign-metastore` command](#assign-metastore-command). - -#### Step 1: Create external Locations - -Create UC external locations by running the [`migration-locations` command](#migrate-locations-command). The command -creates a location for each location found during the [assessment](#assessment-workflow). It uses the credentials -created in the previous steps. - -Alternatively, manually create the external locations -- [AWS - Create External Locations](https://docs.databricks.com/en/connect/unity-catalog/external-locations.html) -- [Azure - Create External Locations](https://learn.microsoft.com/en-us/azure/databricks/connect/unity-catalog/external-locations) - -##### Step 2: Create Catalogs and Schemas - -Create Uniyt Catalog [catalogs](https://learn.microsoft.com/en-us/azure/databricks/catalogs/) and -[schemas](https://learn.microsoft.com/en-us/azure/databricks/schemas/) to organize the destination tables and views in -by running the -[`create-catalogs-schemas` command](#create-catalogs-schemas-command). The command creates the UC catalogs and -schemas based on the [table mapping file](#table-mapping). Additionally, it migrates Hive metastore database -permissions if present. - -This step requires considering how to [physically separate data in storage](https://docs.databricks.com/en/data-governance/unity-catalog/best-practices.html#data-is-physically-separated-in-storage) -within UC. As [Databricks recommends storing managed data at the catalog level](https://docs.databricks.com/en/data-governance/unity-catalog/best-practices.html#configure-a-unity-catalog-metastore), -we advise to prepare the external locations for the to-be created catalogs before running the `create-catalogs-schemas` -command. Either, reuse [previously created external locations](#step-23-create-external-locations) or create additional -external locations outside of UCX if data separation restrictions requires that. Note that external locations can be -reused when using subpaths, for example, a folder in a cloud storage -(`abfss://container@storage.dfs.core.windows.net/folder`) can reuse the external location of the cloud storage -(`abfss://container@storage.dfs.core.windows.net/`). (The previous example also holds for other clouds.) - -### Migrate Hive metastore data objects - -In this step, tables and views are migrated using the following workflows: - -```mermaid -flowchart LR - subgraph CLI - migrate-tables[migrate-tables] - end - - subgraph mt_workflow[workflow: migrate-tables] - dbfs_root_delta_mt_task[migrate_dbfs_root_delta_tables] - migrate_dbfs_root_non_delta_tables[migrate_dbfs_root_non_delta_tables] - external_tables_sync_mt_task[migrate_external_tables_sync] - view_mt_task[migrate_views] - refresh_migration_status[refresh_migration_status] - external_tables_sync_mt_task --> view_mt_task - dbfs_root_delta_mt_task --> view_mt_task - migrate_dbfs_root_non_delta_tables --> view_mt_task - view_mt_task --> refresh_migration_status - end - - subgraph mt_serde_inplace_wf[workflow: migrate-external-hiveserde-tables-in-place-experimental] - serde_inplace_mt_task[migrate_hive_serde_in_place] --> view_mt_task_inplace[migrate_views] - view_mt_task_inplace --> refresh_migration_status_hiveserde[refresh_migration_status] - end - - subgraph mt_ctas_wf[workflow: migrate-external-tables-ctas] - ctas_mt_task[migrate_other_external_ctas] - migrate_hive_serde_ctas[migrate_hive_serde_ctas] - view_mt_ctas_task[migrate_views] - refresh_migration_status_ctas[refresh_migration_status] - ctas_mt_task --> view_mt_ctas_task - migrate_hive_serde_ctas --> view_mt_ctas_task - view_mt_ctas_task --> refresh_migration_status_ctas - end - - migrate-tables -- 1st --> mt_workflow - migrate-tables -- 2nd (optional) table migrated here will be excluded in ctas workflow --> mt_serde_inplace_wf - migrate-tables -- 3rd --> mt_ctas_wf -``` - -The [table migration workflows](#table-migration-workflow) can be triggered using the [`migrate-tables` command](#migrate-tables-command) or by -starting the workflows from the workspace UI manually. The table migration workflows are designed to minimize data -copying and to maintain metadata while allowing users to choose preferred strategies where possible. Chose the -strategies for migrating tables using the table below: - -| Object Type | Description | Upgrade Method | -|-----------------------|--------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `DBFS_ROOT_DELTA` | Delta tables persisted on the Databricks file system (dbfs). | Create a copy of the table with the [`DEEP CLONE` command](https://docs.databricks.com/en/sql/language-manual/delta-clone.html). | -| `DBFS_ROOT_NON_DELTA` | Non-delta tables persisted on the Databricks file system (dbfs). | Create a copy of the table with a [`CREATE TABLE AS SELECT * FROM` command](https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-ddl-create-table-using.html). The UC table will be a Delta table. | | -| `MANAGED` | Managed Hive metastore tables. | Depending on the managed table migration strategy chosen during installation:
1. `CLONE`: Create a copy of the table with a [`CREATE TABLE LOCATION '' AS SELECT * FROM` command](https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-ddl-create-table-using.html).
2. `SYNC_AS_EXTERNAL`, synchronize the table metadata to UC with the [`SYNC` command](https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-aux-sync.html). **Warning**: If the managed Hive metastore table is dropped, the drop deletes the underlying data affecting the synchronised UC table as well.
3. `CONVERT_TO_EXTERNAL`: First, in-place convert the managed Hive metastore to a non-managed table. Then, synchronize the table metadata to UC with the [`SYNC` command](https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-aux-sync.html). **Warning**: This strategy has the advantage over `SYNC_AS_EXTERNAL` that dropping the Hive metastore table does not delete the underlying data, however, impact should be carefully validated in existing workloads as the strategy converts the managed Hive metastore to an external table **in-place**. | -| `EXTERNAL_SYNC` | External tables that the `SYNC` command supports: Delta, Parquet, CSV, JSON, ORC, text or Avro tables. | Synchronize the table metadata to UC with the [`SYNC` command](https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-aux-sync.html). | -| `EXTERNAL_NO_SYNC` | External tables that the `SYNC` command does not support. | Create a copy of the table with a [`CREATE TABLE AS SELECT * FROM` command](https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-ddl-create-table-using.html). The UC table will be a Delta table. | -| `EXTERNAL_HIVESERDE` | External Hive SerDe tables that the `SYNC` command does not support. | Depending on the migration workflow chosen:
1. `migrate-external-tables-ctas` (officially supported) : Create a copy of the table with a [`CREATE TABLE AS SELECT * FROM` command](https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-ddl-create-table-using.html) The UC table will be a Delta table.
2. `migrate-external-hiveserde-tables-in-place-experimental` (experimental) : Recreate the [Hive SerDe](https://cwiki.apache.org/confluence/display/Hive/SerDe) tables using the serialization and deserialization protocols. **Warning:** Although this strategy is supported, there is a risk that the old files created by Hive SerDe may not be processed correctly by Spark datasource in corner cases. | -| `VIEW` | Views | Recreate [views](https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-ddl-create-view.html) using their definitions after repointing their dependencies to UC objects. The migration process supports views depending on other views. | - -The workflows may be executed multiple times. Each step is designed as a standalone operation that migrates all -in-scope tables. After migration, each object is marked with an `upgraded_to` property containing the UC identifier to -which the object is migrated. This property signals that the object is out-of-scope for other migration operations and -that the view dependency exists within UC. The created UC objects are marked with an `upgraded_from` property containing -the Hive metastore identifier from which the object was migrated. - -Finally, the table migration workflows also migrate Hive metastore permissions to Unity Catalog. - -Considerations: -- You may want to run the workflows multiple times to migrate tables in phases. -- If your Delta tables in DBFS root have a large number of files, consider: - - Setting higher `Min` and `Max workers for auto-scale` when being asked during the UCX installation. More cores in the cluster means more concurrency for calling cloud storage API to copy files when deep cloning the Delta tables. - - Setting higher `Parallelism for migrating DBFS root Delta tables with deep clone` (default 200) when being asked during the UCX installation. This controls the number of Spark tasks/partitions to be created for deep clone. -- Consider creating an instance pool, and setting its id when prompted during the UCX installation. This instance pool will be specified in the cluster policy used by all UCX workflows job clusters. -- You may also manually edit the job cluster configration per job or per task after the workflows are deployed. - -Additional references: -- [Databricks file system (dbfs)](https://docs.databricks.com/en/dbfs/index.html) -- [External tables](https://docs.databricks.com/en/tables/external.html) - -### Odds and Ends - -The following sections detail how to repair/amend the UC metastore after the upgrade process. - -#### Skip migrating schemas, tables or views - -```bash -databricks labs ucx skip --schema X [--table Y] [--view Zj] -``` - -The [`skip` command](#skip-command) marks a schema, table or view as to-be skipped during the migration processes. - -#### Move data objects - -```bash -databricks labs ucx move --from-catalog A --from-schema B --from-table C --to-catalog D --to-schema E -``` - -The [`move` command](#move-command) moves the object from the source to the target location. The `upgraded_from` -property are updated to reflect the new location on the source object. Use this command if the data object is migrated -to the wrong destination. - -#### Alias data objects - -```bash -databricks labs ucx alias --from-catalog A --from-schema B --from-table C --to-catalog D --to-schema E -``` - -This [`alias` command](#alias-command) creates an alias for the object in the target location by creating a view reading -from the table that needs aliasing. It will create a mirror view to view that is marked as alias. -Use this command if Hive metastore tables point to the same location as UC does not support UC does not support tables -with overlapping data locations. - -#### Revert migrated data objects - -```bash -databricks labs ucx revert-migrated-tables --schema X --table Y [--delete-managed] -``` - -The [`revert-migrated-tables` command](#revert-migrated-tables-command) drops the Unity Catalog table or view and reset -the `upgraded_to` property on the source object. Use this command to allow for migrating a table or view again. - -[[back to top](#databricks-labs-ucx)] - -# Workflows - -Part of this application is deployed as [Databricks Workflows](https://docs.databricks.com/en/workflows/index.html). -You can view the status of deployed workflows via the [`workflows` command](#workflows-command). -Failed workflows can be fixed with the [`repair-run` command](#repair-run-command). - -[[back to top](#databricks-labs-ucx)] - -## Assessment workflow - -![ucx_assessment_workflow](docs/ucx_assessment_workflow.png) - -The assessment workflow can be triggered using the Databricks UI or via the -[`ensure-assessment-run` command](#ensure-assessment-run-command). - -The assessment workflow retrieves - or *crawls* - details of [workspace assets](https://docs.databricks.com/en/workspace/workspace-assets.html) -and [securable objects in the Hive metastore](https://docs.databricks.com/en/data-governance/table-acls/object-privileges.html#securable-objects-in-the-hive-metastore) -relevant for upgrading to UC to assess the compatibility with UC. The `crawl_` tasks retrieve assess and objects. The -`assess_` tasks assess the compatibility with UC. The output of each task is stored in the -[inventory database](#installation-resources) so that it can be used for further analysis and decision-making through -the [assessment report](docs/assessment.md). - -1. `crawl_tables`: This task retrieves table definitions from the Hive metastore and persists the definitions in - the `tables` table. The definitions include information such as: - - Database/schema name - - Table name - - Table type - - Table location -2. `crawl_udfs`: This task retrieves UDF definitions from the Hive metastore and persists the definitions in - the `udfs` table. -3. `setup_tacl`: (Optimization) This task starts the `tacl` job cluster in parallel to other tasks. -4. `crawl_grants`: This task retrieves [privileges you can grant on Hive objects](https://docs.databricks.com/en/data-governance/table-acls/object-privileges.html#privileges-you-can-grant-on-hive-metastore-objects) - and persists the privilege definitions in the `grants` table - The retrieved permission definitions include information such as: - - Securable object: schema, table, view, (anonymous) function or any file. - - Principal: user, service principal or group - - Action type: grant, revoke or deny -5. `estimate_table_size_for_migration`: This task analyzes the Delta table retrieved by `crawl_tables` to retrieve - an estimate of their size and persists the table sizes in the `table_size` table. The table size support the decision - for using the `SYNC` or `CLONE` [table migration strategy](#step-3-upgrade-the-metastore). -6. `crawl_mounts`: This task retrieves mount point definitions and persists the definitions in the `mounts` table. -7. `guess_external_locations`: This task guesses shared mount path prefixes of [external tables](https://docs.databricks.com/en/sql/language-manual/sql-ref-external-tables.html) - retrieved by `crawl_tables` that use mount points and persists the locations in the `external_locations` table. The - goal is to identify the to-be created UC [external locations](#step-23-create-external-locations). -8. `assess_jobs`: This task retrieves the job definitions and persists the definitions in the `jobs` table. Job - definitions may require updating to become UC compatible. -9. `assess_clusters`: This task retrieves the clusters definitions and persists the definitions in the `clusters` table. - Cluster definitions may require updating to become UC compatible. -10. `assess_pipelines`: This task retrieves the [Delta Live Tables](https://www.databricks.com/product/delta-live-tables) - (DLT) pipelines definitions and persists the definitions in the `pipelines` table. DLT definitions may require - updating to become UC compatible. -11. `assess_incompatible_submit_runs` : This task retrieves - [job runs](https://docs.databricks.com/en/jobs/monitor.html#view-runs-for-a-job), also known as - [job submit runs](https://docs.databricks.com/api/workspace/jobs/submit), and persists the definitions in the - `submit_runs` table. Incompatibility with UC is assessed: - - [Databricks runtime](https://www.databricks.com/glossary/what-is-databricks-runtime) should be version 11.3 or above - - [Access mode](https://docs.databricks.com/en/compute/configure.html#access-modes) should be set. -12. `crawl_cluster_policies` : This tasks retrieves - [cluster policies](https://docs.databricks.com/en/admin/clusters/policies.html) and persists the policies in the - `policies` table. Incompatibility with UC is assessed: - - [Databricks runtime](https://www.databricks.com/glossary/what-is-databricks-runtime) should be version 11.3 or above -13. `assess_azure_service_principals`: This tasks retrieves Azure service principal - [authentications information](https://learn.microsoft.com/en-us/azure/databricks/connect/storage/tutorial-azure-storage#connect-adls) - from Spark configurations to access storage accounts and persists these configuration information - in `azure_service_principals` table. The Spark configurations from the following places are retrieved: - - Clusters configurations - - Cluster policies - - Job cluster configurations - - Pipeline configurations - - Warehouse configuration -14. `assess_global_init_scripts`: This task retrieves - [global init scripts](https://docs.databricks.com/en/init-scripts/global.html) and persists references to the - scripts in the `global_init_scripts` table. Again, Azure service principal authentication information might be given - in those scripts. -15. `workspace_listing` : This tasks lists [workspace files](https://docs.databricks.com/en/workspace/workspace-assets.html#files) - recursively to compile a collection of directories, notebooks, files, repos and libraries. The task uses multi-threading - to parallelize the listing process for speeding up execution on big workspaces. -16. `crawl_permissions` : This tasks retrieves workspace-local groups permissions and persists these permissions in the `permissions` table. -17. `crawl_groups`: This tasks retrieves workspace-local groups and persists these permissions in the `groups` table. -18. `assess_dashboards`: This task retrieves the dashboards to analyze their queries for - [migration problems](#linter-message-codes) -19. `assess_workflows`: This task retrieves the jobs to analyze their notebooks and files for - [migration problems](#linter-message-codes). - -After UCX assessment workflow finished, see the assessment dashboard for findings and recommendations. -See [this guide](docs/assessment.md) for more details. - -![report](docs/assessment-report.png) - -Proceed to the [group migration workflow](#group-migration-workflow) below or go back to the -[migration process diagram](#migration-process). - -The UCX assessment workflow is designed to only run once, re-running will **not** update the existing results. If the -inventory and findings for a workspace need to be updated then first reinstall UCX by [uninstalling](#uninstall-ucx) -and [installing](#install-ucx) it again. - -[[back to top](#databricks-labs-ucx)] - -## Group migration workflow - -> You are required to complete the [assessment workflow](#assessment-workflow) before starting the group migration workflow. - -**The group migration workflow does NOT CREATE account groups.** In contrast to account groups, [the (legacy) -workspace-local groups cannot be assigned to additional workspaces or granted access to data in a Unity Catalog -metastore](https://docs.databricks.com/en/admin/users-groups/groups.html#difference-between-account-groups-and-workspace-local-groups). -A Databricks admin [assigns account groups to workspaces](https://docs.databricks.com/en/admin/users-groups/index.html#assign-users-to-workspaces) -using [identity federation](https://docs.databricks.com/en/admin/users-groups/index.html#enable-identity-federation) -to manage groups from a single place: your Databricks account. We expect UCX users to create account groups -centrally while most other Databricks resources that UCX touches are scoped to a single workspace. -If you do not have account groups matching the workspace in which UCX is installed, please -run [`create-account-groups` command](#create-account-groups-command) before running the group migration workflow. - -The group migration workflow is designed to migrate workspace-local groups to account-level groups. It verifies if -the necessary groups are available to the workspace with the correct permissions, and removes unnecessary groups and -permissions. The group migration workflow depends on the output of the assessment workflow, thus, should only be -executed after a successful run of the assessment workflow. The group migration workflow may be executed multiple times. - -1. `verify_metastore_attached`: Verifies if a metastore is attached. Account level groups are only available when - a metastore is attached. [See `assign-metastore` command.](#assign-metastore-command) -2. `rename_workspace_local_groups`: This task renames workspace-local groups by adding a `ucx-renamed-` prefix. This - step is taken to avoid conflicts with account groups that may have the same name as workspace-local groups. -3. `reflect_account_groups_on_workspace`: This task adds matching account groups to this workspace. The matching account - groups must exist for this step to be successful. This step is necessary to ensure that the account groups - are available in the workspace for assigning permissions. -4. `apply_permissions_to_account_groups`: This task assigns the full set of permissions of the original group to the - account-level one. This step is necessary to ensure that the account-level groups have the necessary permissions to - manage the entities in the workspace. It covers workspace-local permissions for all entities including: - - Legacy Table ACLs - - Entitlements - - AWS instance profiles - - Clusters - - Cluster policies - - Instance Pools - - Databricks SQL warehouses - - Delta Live Tables - - Jobs - - MLflow experiments - - MLflow registry - - SQL Dashboards & Queries - - SQL Alerts - - Token and Password usage permissions - - Secret Scopes - - Notebooks - - Directories - - Repos - - Files -5. `validate_groups_permissions`: This task validates that all the crawled permissions are applied correctly to the - destination groups. - -After successfully running the group migration workflow: -1. Use [`validate-groups-membership` command](#validate-groups-membership-command) for extra confidence the newly created - account level groups are considered to be valid. -2. Run the [`remove-workspace-local-backup-grups`](#validate-groups-membership-command) to remove workspace-level backup - groups, along with their permissions. This should only be executed after confirming that the workspace-local - migration worked successfully for all the groups involved. This step is necessary to clean up the workspace and - remove any unnecessary groups and permissions. -3. Proceed to the [table migration process](#Table-Migration). - -For additional information see: -- The [detailed design](docs/local-group-migration.md) of thie group migration workflow. -- The [migration process diagram](#migration-process) showing the group migration workflow in context of the whole - migration process. - -[[back to top](#databricks-labs-ucx)] - -## Table migration workflows - -This section lists the workflows that migrate tables and views. See [this section](#migrate-hive-metastore-data-objects) -for deciding which workflow to run and additional context for migrating tables. - -### Migrate tables - -The general table migration workflow `migrate-tables` migrates all tables and views using default strategies. - -1. `migrate_external_tables_sync` : This step migrates the external tables that are supported by `SYNC` command. -2. `migrate_dbfs_root_delta_tables` : This step migrates delta tables stored in DBFS root using the `DEEP CLONE` - command. -3. `migrate_dbfs_root_non_delta_tables` : This step migrates non-delta tables stored in DBFS root using the `CREATE - TABLE AS SELECT * FROM` command. -4. `migrate_views` : This step migrates views using the `CREATE VIEW` command. -5. `update_migration_status` : Refresh the migration status of all data objects. - -### Migrate external Hive SerDe tables - -The experimental table migration workflow `migrate-external-hiveserde-tables-in-place-experimental` migrates tables that -support the `SYNC AS EXTERNAL` command. - -1. `migrate_hive_serde_in_place` : This step migrates the Hive SerDe tables that are supported by `SYNC AS EXTERNAL` - command. -2. `migrate_views` : This step migrates views using the `CREATE VIEW` command. -3. `update_migration_status` : Refresh the migration status of all data objects. - -### Migrate external tables CTAS - -The table migration workflow `migrate-external-tables-ctas` migrates tables with the `CREATE TABLE AS SELECT * FROM` -command. - -1. `migrate_other_external_ctas` This step migrates the Hive Serde tables using the `CREATE TABLE AS SELECT * FROM` - command. -2. `migrate_hive_serde_ctas` : This step migrates the Hive Serde tables using the `CREATE TABLE AS SELECT * FROM` - command. -4. `migrate_views` : This step migrates views using the `CREATE VIEW` command. -4. `update_migration_status` : Refresh the migration status of all data objects. - -## Post-migration data reconciliation workflow - -The `migrate-data-reconciliation` workflow validates the integrity of the migrated tables and persists its results in -the `recon_results` table. The workflow compares the following between the migrated Hive metastore and its UC -counterpart table: -- `Schema` : See this result in the `schema_matches` column. -- `Column by column` : See this result in the `column_comparison` column. -- `Row counts` : If the row count is within the reconciliation threshold (defaults to 5%), the `data_matches` column is - set to `true`, otherwise it is set to `false`. -- `Rows` : If the `compare_rows` flag is set to `true`, rows are compared using a hash comparison. Number of missing - rows are stored in the `source_missing_count` and `target_missing_count` column, respectively. - -The output is processed and displayed in the migration dashboard using the in `reconciliation_results` view. - -![reconciliation results](docs/recon_results.png) - -### [LEGACY] Scan tables in mounts Workflow -#### Always run this workflow AFTER the assessment has finished -- This experimental workflow attempts to find all Tables inside mount points that are present on your workspace. -- If you do not run this workflow, then `migrate-tables-in-mounts-experimental` won't do anything. -- It writes all results to `hive_metastore..tables`, you can query those tables found by filtering on database values that starts with `mounted_` -- This command is incremental, meaning that each time you run it, it will overwrite the previous tables in mounts found. -- Current format are supported: - - DELTA - PARQUET - CSV - JSON - - Also detects partitioned DELTA and PARQUET -- You can configure these workflows with the following options available on conf.yml: - - include_mounts : A list of mount points to scans, by default the workflow scans for all mount points - - exclude_paths_in_mount : A list of paths to exclude in all mount points - - include_paths_in_mount : A list of paths to include in all mount points - -### [LEGACY] Migrate tables in mounts Workflow -- An experimental workflow that migrates tables in mount points using a `CREATE TABLE` command, optinally sets a default tables owner if provided in `default_table_owner` conf parameter. -- You must do the following in order to make this work: - - run the Assessment [workflow](#assessment-workflow) - - run the scan tables in mounts [workflow](#EXPERIMENTAL-scan-tables-in-mounts-workflow) - - run the [`create-table-mapping` command](#create-table-mapping-command) - - or manually create a `mapping.csv` file in Workspace -> Applications -> ucx - - -[[back to top](#databricks-labs-ucx)] - -## [EXPERIMENTAL] Migration Progress Workflow - -The manually triggered `migration-progress-experimental` workflow populates the tables visualized in the -[migration progress dashboard](#dashboards) by updating a **subset** of the [inventory tables](#assessment-workflow) -to [track Unity Catalog compatability](docs/migration-progress.md) of Hive and workspace objects that need to be migrated. - -The following pre-requisites need to be fulfilled before running the workflow: -- [UC metastore attached to workspace](../README.md#assign-metastore-command) -- [UCX catalog exists](../README.md#create-ucx-catalog-command) -- [Assessment job ran successfully](../README.md#ensure-assessment-run-command) - -[[back to top](#databricks-labs-ucx)] - -# Dashboards - -The dashboards visualize UCX's outcomes. Each dashboard has text widgets detailing the contents in more detail. Below an -overview with a short description is given. - -| Name | Description | -|-----------------------------------------------------------------------------------------------------------------|----------------------------------------------------| -| [Assessment \[Main\]](./src/databricks/labs/ucx/queries/assessment/main/00_0__assessment_overview.md) | Assessment overview | -| [Assessment \[Estimates\]](./src/databricks/labs/ucx/queries/assessment/estimates/00_0_metastore_assignment.md) | Migration effort estimates based on the assessment | -| [Assessment \[Interactive\]](./src/databricks/labs/ucx/queries/assessment/interactive/00_0_interactive.md) | Assessment outcomes on interactive clusters | -| [Assessment \[Azure\]](./src/databricks/labs/ucx/queries/assessment/azure/00_0_azure_service_principals.md) | Assessment outcomes specific to Azure | -| [Migration \[Main\]](./src/databricks/labs/ucx/queries/migration/main/00_0_migration_overview.md) | Migration overview | -| [Migration \[Groups\]](./src/databricks/labs/ucx/queries/migration/groups/00_0_migration_overview.md) | Group migration outcomes | -| [Progress \[Main\]](./src/databricks/labs/ucx/queries/progress/main/00_0_migration_progress.md) | [Migration progress](./docs/migration-progress.md) | - -[[back to top](#databricks-labs-ucx)] - -# Linter message codes - -![code compatibility problems](docs/code_compatibility_problems.png) - -Here's the detailed explanation of the linter message codes: - -## `cannot-autofix-table-reference` - -This indicates that the linter has found a table reference that cannot be automatically fixed. The user must manually -update the table reference to point to the correct table in Unity Catalog. This mostly occurs, when table name is -computed dynamically, and it's too complex for our static code analysis to detect it. We detect this problem anywhere -where table name could be used: `spark.sql`, `spark.catalog.*`, `spark.table`, `df.write.*` and many more. Code examples -that trigger this problem: - -```python -spark.table(f"foo_{some_table_name}") -# .. -df = spark.range(10) -df.write.saveAsTable(f"foo_{some_table_name}") -# .. or even -df.write.insertInto(f"foo_{some_table_name}") -``` - -Here the `some_table_name` variable is not defined anywhere in the visible scope. Though, the analyser would -successfully detect table name if it is defined: - -```python -some_table_name = 'bar' -spark.table(f"foo_{some_table_name}") -``` - -We even detect string constants when coming either from `dbutils.widgets.get` (via job named parameters) or through -loop variables. If `old.things` table is migrated to `brand.new.stuff` in Unity Catalog, the following code will -trigger two messages: [`table-migrated-to-uc`](#table-migrated-to-uc) for the first query, as the contents are clearly -analysable, and `cannot-autofix-table-reference` for the second query. - -```python -# ucx[table-migrated-to-uc:+4:4:+4:20] Table old.things is migrated to brand.new.stuff in Unity Catalog -# ucx[cannot-autofix-table-reference:+3:4:+3:20] Can't migrate table_name argument in 'spark.sql(query)' because its value cannot be computed -table_name = f"table_{index}" -for query in ["SELECT * FROM old.things", f"SELECT * FROM {table_name}"]: - spark.sql(query).collect() -``` - -[[back to top](#databricks-labs-ucx)] - -## `catalog-api-in-shared-clusters` - -`spark.catalog.*` functions require Databricks Runtime 14.3 LTS or above on Unity Catalog clusters in Shared access -mode, so of your code has `spark.catalog.tableExists("table")` or `spark.catalog.listDatabases()`, you need to ensure -that your cluster is running the correct runtime version and data security mode. - -[[back to top](#databricks-labs-ucx)] - -## `changed-result-format-in-uc` - -Calls to these functions would return a list of `..` instead of `.
`. So if -you have code like this: - -```python -for table in spark.catalog.listTables(): - do_stuff_with_table(table) -``` - -you need to make sure that `do_stuff_with_table` can handle the new format. - -[[back to top](#databricks-labs-ucx)] - -## `direct-filesystem-access-in-sql-query` - -Direct filesystem access is deprecated in Unity Catalog. -DBFS is no longer supported, so if you have code like this: - -```python -df = spark.sql("SELECT * FROM parquet.`/mnt/foo/path/to/parquet.file`") -``` - -you need to change it to use UC tables. - -[[back to top](#databricks-labs-ucx)] - -## `direct-filesystem-access` - -Direct filesystem access is deprecated in Unity Catalog. -DBFS is no longer supported, so if you have code like this: - -```python -display(spark.read.csv('/mnt/things/data.csv')) -``` - -or this: - -```python -display(spark.read.csv('s3://bucket/folder/data.csv')) -``` - -You need to change it to use UC tables or UC volumes. - -[[back to top](#databricks-labs-ucx)] - -## `dependency-not-found` - -This message indicates that the linter has found a dependency, like Python source file or a notebook, that is not -available in the workspace. The user must ensure that the dependency is available in the workspace. This usually -means an error in the user code. - -[[back to top](#databricks-labs-ucx)] - -## `jvm-access-in-shared-clusters` - -You cannot access Spark Driver JVM on Unity Catalog clusters in Shared Access mode. If you have code like this: - -```python -spark._jspark._jvm.com.my.custom.Name() -``` - -or like this: - -```python -log4jLogger = sc._jvm.org.apache.log4j -LOGGER = log4jLogger.LogManager.getLogger(__name__) -``` - -you need to change it to use Python equivalents. - -[[back to top](#databricks-labs-ucx)] - -## `legacy-context-in-shared-clusters` - -SparkContext (`sc`) is not supported on Unity Catalog clusters in Shared access mode. Rewrite it using SparkSession -(`spark`). Example code that triggers this message: - -```python -df = spark.createDataFrame(sc.emptyRDD(), schema) -``` - -or this: - -```python -sc.parallelize([1, 2, 3]) -``` - -[[back to top](#databricks-labs-ucx)] - -## `not-supported` - -Installing eggs is no longer supported on Databricks 14.0 or higher. - -[[back to top](#databricks-labs-ucx)] - -## `notebook-run-cannot-compute-value` - -Path for `dbutils.notebook.run` cannot be computed and requires adjusting the notebook path. -It is not clear for automated code analysis where the notebook is located, so you need to simplify the code like: - -```python -b = some_function() -dbutils.notebook.run(b) -``` - -to something like this: - -```python -a = "./leaf1.py" -dbutils.notebook.run(a) -``` - -[[back to top](#databricks-labs-ucx)] - -## `python-udf-in-shared-clusters` - -`applyInPandas` requires DBR 14.3 LTS or above on Unity Catalog clusters in Shared access mode. Example: - -```python -df.groupby("id").applyInPandas(subtract_mean, schema="id long, v double").show() -``` - -Arrow UDFs require DBR 14.3 LTS or above on Unity Catalog clusters in Shared access mode. - -```python -@udf(returnType='int', useArrow=True) -def arrow_slen(s): - return len(s) -``` - -It is not possible to register Java UDF from Python code on Unity Catalog clusters in Shared access mode. Use a -`%scala` cell to register the Scala UDF using `spark.udf.register`. Example code that triggers this message: - -```python -spark.udf.registerJavaFunction("func", "org.example.func", IntegerType()) -``` - -[[back to top](#databricks-labs-ucx)] - -## `rdd-in-shared-clusters` - -RDD APIs are not supported on Unity Catalog clusters in Shared access mode. Use mapInArrow() or Pandas UDFs instead. - -```python -df.rdd.mapPartitions(myUdf) -``` - -[[back to top](#databricks-labs-ucx)] - -## `spark-logging-in-shared-clusters` - -Cannot set Spark log level directly from code on Unity Catalog clusters in Shared access mode. Remove the call and set -the cluster spark conf `spark.log.level` instead: - -```python -sc.setLogLevel("INFO") -setLogLevel("WARN") -``` - -Another example could be: - -```python -log4jLogger = sc._jvm.org.apache.log4j -LOGGER = log4jLogger.LogManager.getLogger(__name__) -``` - -or - -```python -sc._jvm.org.apache.log4j.LogManager.getLogger(__name__).info("test") -``` - -[[back to top](#databricks-labs-ucx)] - -## `sql-parse-error` - -This is a generic message indicating that the SQL query could not be parsed. The user must manually check the SQL query. - -[[back to top](#databricks-labs-ucx)] - -## `sys-path-cannot-compute-value` - -Path for `sys.path.append` cannot be computed and requires adjusting the path. It is not clear for automated code -analysis where the path is located. - -[[back to top](#databricks-labs-ucx)] - -## `table-migrated-to-uc` - -This message indicates that the linter has found a table that has been migrated to Unity Catalog. The user must ensure -that the table is available in Unity Catalog. - -[[back to top](#databricks-labs-ucx)] - -## `to-json-in-shared-clusters` - -`toJson()` is not available on Unity Catalog clusters in Shared access mode. Use `toSafeJson()` on DBR 13.3 LTS or -above to get a subset of command context information. Example code that triggers this message: - -```python -dbutils.notebook.entry_point.getDbutils().notebook().getContext().toSafeJson() -``` - -[[back to top](#databricks-labs-ucx)] - -## `unsupported-magic-line` - -This message indicates the code that could not be analysed by UCX. User must check the code manually. - -[[back to top](#databricks-labs-ucx)] - -# Utility commands - -## `logs` command - -```text -$ databricks labs ucx logs [--workflow WORKFLOW_NAME] [--debug] -``` - -This command displays the logs of the last run of the specified workflow. If no workflow is specified, it displays -the logs of the workflow that was run the last. This command is useful for developers and administrators who want to -check the logs of the last run of a workflow and ensure that it was executed as expected. It can also be used for -debugging purposes when a workflow is not behaving as expected. By default, only `INFO`, `WARNING`, and `ERROR` logs -are displayed. To display `DEBUG` logs, use the `--debug` flag. - -[[back to top](#databricks-labs-ucx)] - -## `ensure-assessment-run` command - -```commandline -databricks labs ucx ensure-assessment-run -``` - -This command ensures that the [assessment workflow](#assessment-workflow) was run on a workspace. -This command will block until job finishes. -Failed workflows can be fixed with the [`repair-run` command](#repair-run-command). Workflows and their status can be -listed with the [`workflows` command](#workflows-command). - -[[back to top](#databricks-labs-ucx)] - -## `update-migration-progress` command - -```commandline -databricks labs ucx update-migration-progress -``` - -This command runs the [(experimental) migration progress workflow](#experimental-migration-progress-workflow) to update -the migration status of workspace resources that need to be migrated. It does this by triggering -the `migration-progress-experimental` workflow to run on a workspace and waiting for -it to complete. - -Workflows and their status can be listed with the [`workflows` command](#workflows-commandr), while failed workflows can -be fixed with the [`repair-run` command](#repair-run-command). - -[[back to top](#databricks-labs-ucx)] - -## `repair-run` command - -```commandline -databricks labs ucx repair-run --step WORKFLOW_NAME -``` - -This command repairs a failed [UCX Workflow](#workflows). This command is useful for developers and administrators who -want to repair a failed job. It can also be used to debug issues related to job failures. This operation can also be -done via [user interface](https://docs.databricks.com/en/workflows/jobs/repair-job-failures.html). Workflows and their -status can be listed with the [`workflows` command](#workflows-command). - -[[back to top](#databricks-labs-ucx)] - -## `workflows` command - -See the [migration process diagram](#migration-process) to understand the role of each workflow in the migration process. - -```text -$ databricks labs ucx workflows -Step State Started -assessment RUNNING 1 hour 2 minutes ago -099-destroy-schema UNKNOWN -migrate-groups UNKNOWN -remove-workspace-local-backup-groups UNKNOWN -validate-groups-permissions UNKNOWN -``` - -This command displays the [deployed workflows](#workflows) and their state in the current workspace. It fetches the latest -job status from the workspace and prints it in a table format. This command is useful for developers and administrators -who want to check the status of UCX workflows and ensure that they have been executed as expected. It can also be used -for debugging purposes when a workflow is not behaving as expected. Failed workflows can be fixed with -the [`repair-run` command](#repair-run-command). - -[[back to top](#databricks-labs-ucx)] - -## `open-remote-config` command - -```commandline -databricks labs ucx open-remote-config -``` - -This command opens the remote configuration file in the default web browser. It generates a link to the configuration file -and opens it using the `webbrowser.open()` method. This command is useful for developers and administrators who want to view or -edit the remote configuration file without having to manually navigate to it in the workspace. It can also be used to quickly -access the configuration file from the command line. Here's the description of configuration properties: - - * `inventory_database`: A string representing the name of the inventory database. - * `workspace_group_regex`: An optional string representing the regular expression to match workspace group names. - * `workspace_group_replace`: An optional string to replace the matched group names with. - * `account_group_regex`: An optional string representing the regular expression to match account group names. - * `group_match_by_external_id`: A boolean value indicating whether to match groups by their external IDs. - * `include_group_names`: An optional list of strings representing the names of groups to include for migration. - * `renamed_group_prefix`: An optional string representing the prefix to add to renamed group names. - * `instance_pool_id`: An optional string representing the ID of the instance pool. - * `warehouse_id`: An optional string representing the ID of the warehouse. - * `connect`: An optional `Config` object representing the configuration for connecting to the warehouse. - * `num_threads`: An optional integer representing the number of threads to use for migration. - * `database_to_catalog_mapping`: An optional dictionary mapping source database names to target catalog names. - * `default_catalog`: An optional string representing the default catalog name. - * `skip_tacl_migration`: Optional flag, allow skipping TACL migration when migrating tables or creating catalogs and schemas. - * `default_owner_group`: Assigns this group to all migrated objects (catalogs, databases, tables, views, etc.). The group has to be an account group and the user running the migration has to be a member of this group. - * `log_level`: An optional string representing the log level. - * `workspace_start_path`: A string representing the starting path for notebooks and directories crawler in the workspace. - * `instance_profile`: An optional string representing the name of the instance profile. - * `spark_conf`: An optional dictionary of Spark configuration properties. - * `override_clusters`: An optional dictionary mapping job cluster names to existing cluster IDs. - * `policy_id`: An optional string representing the ID of the cluster policy. - * `include_databases`: An optional list of strings representing the names of databases to include for migration. - -[[back to top](#databricks-labs-ucx)] - -## `installations` command - -```text -$ databricks labs ucx installations -... -13:49:16 INFO [d.labs.ucx] Fetching installations... -13:49:17 INFO [d.l.blueprint.parallel][finding_ucx_installations_5] finding ucx installations 10/88, rps: 22.838/sec -13:49:17 INFO [d.l.blueprint.parallel][finding_ucx_installations_9] finding ucx installations 20/88, rps: 35.002/sec -13:49:17 INFO [d.l.blueprint.parallel][finding_ucx_installations_2] finding ucx installations 30/88, rps: 51.556/sec -13:49:18 INFO [d.l.blueprint.parallel][finding_ucx_installations_9] finding ucx installations 40/88, rps: 56.272/sec -13:49:18 INFO [d.l.blueprint.parallel][finding_ucx_installations_19] finding ucx installations 50/88, rps: 67.382/sec -... -Path Database Warehouse -/Users/serge.smertin@databricks.com/.ucx ucx 675eaf1ff976aa98 -``` - -This command displays the [installations](#installation) by different users on the same workspace. It fetches all -the installations where the `ucx` package is installed and prints their details in JSON format. This command is useful -for administrators who want to see which users have installed `ucx` and where. It can also be used to debug issues -related to multiple installations of `ucx` on the same workspace. - -[[back to top](#databricks-labs-ucx)] - - -## `report-account-compatibility` command - -```text -databricks labs ucx report-account-compatibility --profile labs-azure-account -12:56:09 INFO [databricks.sdk] Using Azure CLI authentication with AAD tokens -12:56:09 INFO [d.l.u.account.aggregate] Generating readiness report -12:56:10 INFO [databricks.sdk] Using Azure CLI authentication with AAD tokens -12:56:10 INFO [databricks.sdk] Using Azure CLI authentication with AAD tokens -12:56:15 INFO [databricks.sdk] Using Azure CLI authentication with AAD tokens -12:56:15 INFO [d.l.u.account.aggregate] Querying Schema ucx -12:56:21 WARN [d.l.u.account.aggregate] Workspace 4045495039142306 does not have UCX installed -12:56:21 INFO [d.l.u.account.aggregate] UC compatibility: 30.303030303030297% (69/99) -12:56:21 INFO [d.l.u.account.aggregate] cluster type not supported : LEGACY_TABLE_ACL: 22 objects -12:56:21 INFO [d.l.u.account.aggregate] cluster type not supported : LEGACY_SINGLE_USER: 24 objects -12:56:21 INFO [d.l.u.account.aggregate] unsupported config: spark.hadoop.javax.jdo.option.ConnectionURL: 10 objects -12:56:21 INFO [d.l.u.account.aggregate] Uses azure service principal credentials config in cluster.: 1 objects -12:56:21 INFO [d.l.u.account.aggregate] No isolation shared clusters not supported in UC: 1 objects -12:56:21 INFO [d.l.u.account.aggregate] Data is in DBFS Root: 23 objects -12:56:21 INFO [d.l.u.account.aggregate] Non-DELTA format: UNKNOWN: 5 objects -``` - -[[back to top](#databricks-labs-ucx)] -## `export-assessment` command - -```commandline -databricks labs ucx export-assessment -``` -The export-assessment command is used to export UCX assessment results to a specified location. When you run this command, you will be prompted to provide details on the destination path and the type of report you wish to generate. If you do not specify these details, the command will default to exporting the main results to the current directory. The exported file will be named based on the selection made in the format. Eg: export_{query_choice}_results.zip -- **Choose a path to save the UCX Assessment results:** - - **Description:** Specify the path where the results should be saved. If not provided, results will be saved in the current directory. - -- **Choose which assessment results to export:** - - **Description:** Select the type of results to export. Options include: - - `azure` - - `estimates` - - `interactive` - - `main` - - **Default:** `main` - -[[back to top](#databricks-labs-ucx)] - -# Metastore related commands - -These commands are used to assign a Unity Catalog metastore to a workspace. The metastore assignment is a pre-requisite -for any further migration steps. - -[[back to top](#databricks-labs-ucx)] - -## `show-all-metastores` command - -```text -databricks labs ucx show-all-metastores [--workspace-id ] -``` - -This command lists all the metastores available to be assigned to a workspace. If no workspace is specified, it lists -all the metastores available in the account. This command is useful when there are multiple metastores available within -a region, and you want to see which ones are available for assignment. - -[[back to top](#databricks-labs-ucx)] - -## `assign-metastore` command - -```text -databricks labs ucx assign-metastore --workspace-id [--metastore-id ] -``` - -This command assigns a metastore to a workspace with `--workspace-id`. If there is only a single metastore in the -workspace region, the command automatically assigns that metastore to the workspace. If there are multiple metastores -available, the command prompts for specification of the metastore (id) you want to assign to the workspace. - -[[back to top](#databricks-labs-ucx)] - -## `create-ucx-catalog` command - -```commandline -databricks labs ucx create-ucx-catalog -16:12:59 INFO [d.l.u.hive_metastore.catalog_schema] Validating UC catalog: ucx -Please provide storage location url for catalog: ucx (default: metastore): ... -16:13:01 INFO [d.l.u.hive_metastore.catalog_schema] Creating UC catalog: ucx -``` - -Create and setup UCX artifact catalog. Amongst other things, the artifacts are used for tracking the migration progress -across workspaces. - -# Table migration commands - -These commands are vital part of [table migration process](#Table-Migration) process and require -the [assessment workflow](#assessment-workflow) and -[group migration workflow](#group-migration-workflow) to be completed. -See the [migration process diagram](#migration-process) to understand the role of the table migration commands in -the migration process. - -The first step is to run the [`principal-prefix-access` command](#principal-prefix-access-command) to identify all -the storage accounts used by tables in the workspace and their permissions on each storage account. - -If you don't have any storage credentials and external locations configured, you'll need to run -the [`migrate-credentials` command](#migrate-credentials-command) to migrate the service principals -and [`migrate-locations` command](#migrate-locations-command) to create the external locations. -If some of the external locations already exist, you should run -the [`validate-external-locations` command](#validate-external-locations-command). -You'll need to create the [uber principal](#create-uber-principal-command) with the _**access to all storage**_ used to tables in -the workspace, so that you can migrate all the tables. If you already have the principal, you can skip this step. - -Ask your Databricks Account admin to run the [`sync-workspace-info` command](#sync-workspace-info-command) to sync the -workspace information with the UCX installations. Once the workspace information is synced, you can run the -[`create-table-mapping` command](#create-table-mapping-command) to align your tables with the Unity Catalog, -[create catalogs and schemas](#create-catalogs-schemas-command) and start the migration using [`migrate-tables` command](#migrate-tables-command). During multiple runs of -the table migration workflow, you can use the [`revert-migrated-tables` command](#revert-migrated-tables-command) to -revert the tables that were migrated in the previous run. You can also skip the tables that you don't want to migrate -using the [`skip` command](#skip-command). - -Once you're done with the table migration, proceed to the [code migration](#code-migration-commands). - -[[back to top](#databricks-labs-ucx)] - -## `principal-prefix-access` command - -```text -databricks labs ucx principal-prefix-access [--subscription-ids ] [--aws-profile ] -``` - -This command depends on results from the [assessment workflow](#assessment-workflow) and requires [AWS CLI](#access-for-aws-s3-buckets) -or [Azure CLI](#access-for-azure-storage-accounts) to be installed and authenticated for the given machine. This command -identifies all the storage accounts used by tables in the workspace and their permissions on each storage account. -Once you're done running this command, proceed to the [`migrate-credentials` command](#migrate-credentials-command). - -The "prefix" refers to the start - i.e. prefix - of table locations that point to the cloud storage location. - -[[back to top](#databricks-labs-ucx)] - -### Access for AWS S3 Buckets - -```commandline -databricks labs ucx principal-prefix-access --aws-profile test-profile -``` - -Use to identify all instance profiles in the workspace, and map their access to S3 buckets. -Also captures the IAM roles which has UC arn listed, and map their access to S3 buckets -This requires `aws` CLI to be installed and configured. - -For AWS this command produces a file named `aws_instance_profile_info.csv`. -It has the following format: - -| **role_arn** | **resource_type** | **privilege** | **resource_path** | -|------------------------------------------------------|-------------------|---------------|-----------------------| -| arn:aws:iam::1234:instance-profile/instance-profile1 | s3 | WRITE_FILES | s3://s3_bucket1/path1 | - - -Once done, proceed to the [`migrate-credentials` command](#migrate-credentials-command). - -[[back to top](#databricks-labs-ucx)] - -### Access for Azure Storage Accounts - -```commandline -databricks labs ucx principal-prefix-access --subscription-ids test-subscription-id -``` - -Use to identify all storage account used by tables, identify the relevant Azure service principals and their permissions -on each storage account. The command is used to identify Azure Service Principals, which have `Storage Blob Data Contributor`, -`Storage Blob Data Reader`, `Storage Blob Data Owner` roles, or custom read/write roles on ADLS Gen2 locations that are being -used in Databricks. This requires Azure CLI to be installed and configured via `az login`. It outputs azure_storage_account_info.csv -which will be later used by migrate-credentials command to create UC storage credentials. -Note: This cmd only lists azure storage account gen2, storage format wasb:// or adl:// are not supported in UC and those storage info -will be skipped. - -Once done, proceed to the [`migrate-credentials` command](#migrate-credentials-command). - -[[back to top](#databricks-labs-ucx)] - -## `create-missing-principals` command (AWS Only) -```bash -databricks labs ucx create-missing-principals --aws-profile --single-role -``` -This command identifies all the S3 locations that are missing a UC compatible role and creates them. -It takes single-role optional parameter. -If set to True, it will create a single role for all the S3 locations. -Otherwise, it will create a role for each S3 location. - -Two optional parameter are available for this command: -`--role-name` - This parameter is used to set the prefix for the role name. The default value is `UCX-ROLE`. -`--role-policy` - This parameter is used to set the prefix for the role policy name. The default value is `UCX-POLICY`. - -[[back to top](#databricks-labs-ucx)] - -## `delete-missing-principals` command (AWS Only) -```bash -databricks labs ucx delete-missing-principals --aws-profile -``` -This command helps to delete the IAM role created by UCX. It lists all the IAM Roles generated by the principal-prefix-access -command and allows user to select multiple roles to delete. It also checks if selected roles are mapped to any storage credentials -and asks for confirmation from user. Once confirmed, it deletes the role and its associated inline policy. - -[[back to top](#databricks-labs-ucx)] - -## `create-uber-principal` command - -```text -databricks labs ucx create-uber-principal [--subscription-ids X] -``` - -**Requires Cloud IAM admin privileges.** - -Once the [`assessment` workflow](#assessment-workflow) complete, you should run this command to create a service principal with the -_**read-only access to all storage**_ used by tables in this workspace. It will also configure the -[UCX Cluster Policy](#installation) & SQL Warehouse data access configuration to use this service principal for migration -workflows. Once migration is complete, this service principal should be unprovisioned. - -On Azure, it creates a principal with `Storage Blob Data Contributor` role assignment on every storage account using -Azure Resource Manager APIs. - -This command is one of prerequisites for the [table migration process](#table-migration). - -[[back to top](#databricks-labs-ucx)] - -## `migrate-credentials` command - -```commandline -databricks labs ucx migrate-credentials -``` - -For Azure, this command prompts to confirm performing the following credential migration steps: -1. [RECOMMENDED] For each storage account, create access connectors with managed identities that have the - `Storage Blob Data Contributor` role on the respective storage account. A storage credential is created for each - access connector. -2. Migrate Azure Service Principals, which have `Storage Blob Data Contributor`, - `Storage Blob Data Reader`, `Storage Blob Data Owner`, or custom roles on ADLS Gen2 locations that are being used in - Databricks, to UC storage credentials. The Azure Service Principals to location mapping are listed - in `/Users/{user_name}/.ucx/azure_storage_account_info.csv` which is generated by - [`principal-prefix-access` command](#principal-prefix-access-command). Please review the file and delete the Service - Principals you do not want to be migrated. The command will only migrate the Service Principals that have client - secret stored in Databricks Secret. - - **Warning**: Service principals used to access storage accounts behind firewalls might cause connectivity issues. We - recommend to use access connectors instead. - -For AWS, this command migrates AWS Instance Profiles that are being used in Databricks, to UC storage credentials. -The AWS Instance Profiles to location mapping are listed in -{workspace ucx folder}/aws_instance_profile_info.csv which is generated by principal_prefix_access command. -Please review the file and delete the Instance Profiles you do not want to be migrated.
The aws_profile parameter indicates the aws profile to use. - -Once you're done with this command, run [`validate-external-locations` command](#validate-external-locations-command) after this one. - -[[back to top](#databricks-labs-ucx)] - -## `validate-external-locations` command - -```text -databricks labs ucx validate-external-locations -``` - -Once the [`assessment` workflow](#assessment-workflow) finished successfully, [storage credentials](#migrate-credentials-command) are configured, -run this command to validate and report the missing Unity Catalog external locations to be created. - -This command validates and provides mapping to external tables to external locations, also as Terraform configurations. - -Once you're done with this command, proceed to the [`migrate-locations` command](#migrate-locations-command). - -[[back to top](#databricks-labs-ucx)] - -## `migrate-locations` command - -```text -databricks labs ucx migrate-locations -``` - -Once the [`assessment` workflow](#assessment-workflow) finished successfully, and [storage credentials](#migrate-credentials-command) are configured, -run this command to have Unity Catalog external locations created. The candidate locations to be created are extracted from guess_external_locations -task in the assessment job. You can run [`validate-external-locations` command](#validate-external-locations-command) to check the candidate locations. - -**Location ACLs:** -`migrate-locations` command applies any location ACL from existing cluster. -For Azure, it checks if there are any interactive cluster or SQL warehouse -which has service principals configured to access storage. It maps the storage account to the external location created and grants `CREATE_EXTERNAL_TABLE`, -`CREATE_EXTERNAL_VOLUME` and `READ_FILES` permission on the location to all the user who have access to the interactive cluster or SQL warehouse -For AWS, it checks any instance profiles mapped to the interactive cluster or SQL warehouse. It checks the mapping of instance profiles to the bucket. It then -maps the bucket to the external locations created and grants `CREATE_EXTERNAL_TABLE`, `CREATE_EXTERNAL_VOLUME` and `READ_FILES` permission on the location to all the user who have access to the interactive cluster -or SQL warehouse - -Once you're done with this command, proceed to the [`create-table-mapping` command](#create-table-mapping-command). - -[[back to top](#databricks-labs-ucx)] - -## `create-table-mapping` command - -```text -databricks labs ucx create-table-mapping -``` - -Once the [`assessment` workflow](#assessment-workflow) finished successfully -[workspace info is synchronized](#sync-workspace-info-command), run this command to create the initial -table mapping for review in CSV format in the Databricks Workspace: - -```text -workspace_name,catalog_name,src_schema,dst_schema,src_table,dst_table -labs-azure,labs_azure,default,default,ucx_tybzs,ucx_tybzs -``` - -The format of the mapping file is as follows: - -| **columns:** | **workspace_name** | **catalog_name** | **src_schema** | **dst_schema** | **src_table** | **dst_table** | -|--------------|---------------------|--------------|----------------|----------------|---------------|---------------| -| values: | data_engineering_ws | de_catalog | database1 | database1 | table1 | table1 | - -You are supposed to review this mapping and adjust it if necessary. This file is in CSV format, so that you can edit it -easier in your favorite spreadsheet application. - -Once you're done with this command, [create catalogs and schemas](#create-catalogs-schemas-command). During -multiple runs of the table migration workflow, you can use the [`revert-migrated-tables` command](#revert-migrated-tables-command) -to revert the tables that were migrated in the previous run. You can also skip the tables that you don't want to migrate -using the [`skip` command](#skip-command). - -This command is one of prerequisites for the [table migration process](#Table-Migration). - -Once you're done with table migration, proceed to the [code migration](#code-migration-commands). - -[[back to top](#databricks-labs-ucx)] - -## `skip` command - -```text -databricks labs ucx skip --schema X [--table Y] [--view Z] -``` - -Anytime after [`create-table-mapping` command](#create-table-mapping-command) is executed, you can run this command. - -This command allows users to skip certain schemas, tables or views during the [table migration](#table-migration) process. -The command takes `--schema` and, optionally, `--table` and `--view` flags to specify the schema, table or view to skip. -If no `--table` flag is provided, all tables in the specified HMS database are skipped. The `--table` and `--view` can -only be used exclusively. This command is useful to temporarily disable migration of a particular schema, table or view. - -Once you're done with table migration, proceed to the [code migration](#code-migration-commands). - -[[back to top](#databricks-labs-ucx)] - -## `unskip` command - -```commandline -databricks labs ucx unskip --schema X [--table Y] [--view Z] -``` - -This command removes the mark set by the [`skip` command](#skip-command) on the given schema, table or view. - -[[back to top](#databricks-labs-ucx)] - -## `create-catalogs-schemas` command - -```text -databricks labs ucx create-catalogs-schemas -``` -After [`create-table-mapping` command](#create-table-mapping-command) is executed, you can run this command to have the required UC catalogs and schemas created. -This command is supposed to be run before migrating tables to UC using [table migration process](#Table-Migration). -Catalog & Schema ACL: -`create-catalogs-schemas` command also applies any catalog and schema ACL from existing clusters. -For Azure it checks if there are any interactive cluster or sql warehouse which has service principals configured to access storage. -It maps the storage account to the tables which has external location on those storage account created and grants `USAGE` access to -the schema and catalog if at least one such table is migrated to it. -For AWS, it checks any instance profiles mapped to the interactive cluster or sql warehouse. It checks the mapping of instance profiles -to the bucket. It then maps the bucket to the tables which has external location on those bucket created and grants `USAGE` access to -the schema and catalog if at least one such table is migrated to it. -[[back to top](#databricks-labs-ucx)] - -## `assign-owner-group` command - -```text -databricks labs ucx assign-owner-group -``` - -This command assigns the owner group to the UCX catalog and all migrated tables. -The owner group is an account group that will be designated as an owner to all migrated objects (catalogs, schemas, tables, views). -The principal running the command and later, the migration workflows, is required to be a member of this group. -The command will list all the groups the principal is a member of and allow the selection of the owner group. -It sets the default_owner_group property in the config.yml file. - -[[back to top](#databricks-labs-ucx)] - -## `migrate-tables` command - -```text -databricks labs ucx migrate-tables -``` - -Anytime after [`create-table-mapping` command](#create-table-mapping-command) is executed, you can run this command. - -This command kicks off the [table migration](#Table-Migration) process. It triggers the `migrate-tables` workflow, -and if there are HiveSerDe tables detected, prompt whether to trigger the `migrate-external-hiveserde-tables-in-place-experimental` workflow. - -Table and View ACL: -`migrate-tables` command also applies any table and view ACL from existing clusters. -For Azure it checks if there are any interactive cluster or sql warehouse which has service principals configured to access storage. -It maps the storage account to the tables which has external location on those storage account created and grants either `SELECT` permission if -the service principal only has read access on the storage account and `ALL_PRIVILEGES` if the service principal has write access on the storage account -For AWS, it checks any instance profiles mapped to the interactive cluster or sql warehouse. It checks the mapping of instance profiles -to the bucket. It then maps the bucket to the tables which has external location on those bucket created and grants either `SELECT` permission if -the instance profile only has read access on the bucket and `ALL_PRIVILEGES` if the instance profile has write access on the bucket. - -[[back to top](#databricks-labs-ucx)] - -## `revert-migrated-tables` command - -```text -databricks labs ucx revert-migrated-tables --schema X --table Y [--delete-managed] -``` - -Anytime after [`create-table-mapping` command](#create-table-mapping-command) is executed, you can run this command. - -This command removes the `upgraded_from` property on a migrated table for re-migration in the [table migration](#Table-Migration) process. -This command is useful for developers and administrators who want to revert the migration of a table. It can also be used -to debug issues related to table migration. - -Go back to the [`create-table-mapping` command](#create-table-mapping-command) after you're done with this command. - -[[back to top](#databricks-labs-ucx)] - -## `move` command - -```text -databricks labs ucx move --from-catalog A --from-schema B --from-table C --to-catalog D --to-schema E -``` - -This command moves a UC table/tables from one schema to another schema after -the [table migration](#Table-Migration) process. This is useful for developers and administrators who want -to adjust their catalog structure after tables upgrade. - -Users will be prompted whether tables/view are dropped after moving to new schema. This only applies to `MANAGED` tables and views. - -This command moves different table types differently: -- `MANAGED` tables are deep-cloned to the new schema. -- `EXTERNAL` tables are dropped from the original schema, then created in the target schema using the same location. -This is due to Unity Catalog not supporting multiple tables with overlapping paths -- `VIEW` are recreated using the same view definition. - -This command supports moving multiple tables at once, by specifying `*` as the table name. - -[[back to top](#databricks-labs-ucx)] - -## `alias` command - -```text -databricks labs ucx alias --from-catalog A --from-schema B --from-table C --to-catalog D --to-schema E -``` - -This command aliases a UC table/tables from one schema to another schema in the same or different catalog. -It takes a `WorkspaceClient` object and `from` and `to` parameters as parameters and aliases the tables using -the `TableMove` class. This command is useful for developers and administrators who want to create an alias for a table. -It can also be used to debug issues related to table aliasing. - -[[back to top](#databricks-labs-ucx)] - -# Code migration commands - -See the [migration process diagram](#migration-process) to understand the role of the code migration commands in the migration process. - -After you're done with the [table migration](#Table-Migration), you can proceed to the code migration. - -Once you're done with the code migration, you can run the [`cluster-remap` command](#cluster-remap-command) to remap the -clusters to be UC compatible. - -[[back to top](#databricks-labs-ucx)] - -## `lint-local-code` command - -```text -databricks labs ucx lint-local-code -``` - -At any time, you can run this command to assess all migrations required in a local directory or a file. It only takes seconds to run and it -gives you an initial overview of what needs to be migrated without actually performing any migration. A great way to start a migration! - -This command detects all dependencies, and analyzes them. It is still experimental and at the moment only supports Python and SQL files. -We expect this command to run within a minute on code bases up to 50.000 lines of code. -Future versions of `ucx` will add support for more source types, and more migration details. - -When run from an IDE terminal, this command generates output as follows: -![img.png](docs/lint-local-code-output.png) -With modern IDEs, clicking on the file link opens the file at the problematic line - -[[back to top](#databricks-labs-ucx)] - -## `migrate-local-code` command - -```text -databricks labs ucx migrate-local-code -``` - -**(Experimental)** Once [table migration](#Table-Migration) is complete, you can run this command to -migrate all python and SQL files in the current working directory. This command is highly experimental and -at the moment only supports Python and SQL files and discards code comments and formatting during -the automated transformation process. - -[[back to top](#databricks-labs-ucx)] - -## `migrate-dbsql-dashboards` command - -```text -databricks labs ucx migrate-dbsql-dashboards [--dashboard-id ] -``` - -**(Experimental)** Once [table migration](#Table-Migration) is complete, you can run this command to -migrate all Databricks SQL dashboards in the workspace. At this moment, this command is highly experimental and discards -formatting during the automated transformation process. - -This command tags dashboards & queries that have been migrated with `migrated by UCX` tag. The original queries are -also backed up in the ucx installation folder, to allow for easy rollback (see [`revert-dbsql-dashboards` command](#revert-dbsql-dashboards-command)). - -This command can be run with `--dashboard-id` flag to migrate a specific dashboard. - -This command is incremental and can be run multiple times to migrate new dashboards. - -[[back to top](#databricks-labs-ucx)] - -## `revert-dbsql-dashboards` command - -```text -databricks labs ucx revert-dbsql-dashboards [--dashboard-id ] -``` - -**(Experimental)** This command reverts the migration of Databricks SQL dashboards in the workspace, after -`migrate-dbsql-dashboards` command is executed. - -This command can be run with `--dashboard-id` flag to migrate a specific dashboard. - -[[back to top](#databricks-labs-ucx)] - -# Cross-workspace installations - -When installing UCX across multiple workspaces, administrators need to keep UCX configurations in sync. -UCX will prompt you to select an account profile that has been defined in `~/.databrickscfg`. If you don't have one, -authenticate your machine with: - -* `databricks auth login --host https://accounts.cloud.databricks.com/` (AWS) -* `databricks auth login --host https://accounts.azuredatabricks.net/` (Azure) - -Ask your Databricks Account admin to run the [`sync-workspace-info` command](#sync-workspace-info-command) to sync the -workspace information with the UCX installations. Once the workspace information is synced, you can run the -[`create-table-mapping` command](#create-table-mapping-command) to align your tables with the Unity Catalog. - -[[back to top](#databricks-labs-ucx)] - -## `sync-workspace-info` command - -```text -databricks --profile ACCOUNTS labs ucx sync-workspace-info -14:07:07 INFO [databricks.sdk] Using Azure CLI authentication with AAD tokens -14:07:07 INFO [d.labs.ucx] Account ID: ... -14:07:10 INFO [d.l.blueprint.parallel][finding_ucx_installations_16] finding ucx installations 10/88, rps: 16.415/sec -14:07:10 INFO [d.l.blueprint.parallel][finding_ucx_installations_0] finding ucx installations 20/88, rps: 32.110/sec -14:07:11 INFO [d.l.blueprint.parallel][finding_ucx_installations_18] finding ucx installations 30/88, rps: 39.786/sec -... -``` - -> Requires Databricks Account Administrator privileges. Use `--profile` to select the Databricks cli profile configured -> with access to the Databricks account console (with endpoint "https://accounts.cloud.databricks.com/" -> or "https://accounts.azuredatabricks.net"). - -This command uploads the workspace config to all workspaces in the account where `ucx` is installed. This command is -necessary to create an immutable default catalog mapping for [table migration](#Table-Migration) process and is the prerequisite -for [`create-table-mapping` command](#create-table-mapping-command). - -If you cannot get account administrator privileges in reasonable time, you can take the risk and -run [`manual-workspace-info` command](#manual-workspace-info-command) to enter Databricks Workspace IDs and Databricks -Workspace names. - -[[back to top](#databricks-labs-ucx)] - -## `manual-workspace-info` command - -```text -$ databricks labs ucx manual-workspace-info -14:20:36 WARN [d.l.ucx.account] You are strongly recommended to run "databricks labs ucx sync-workspace-info" by account admin, - ... otherwise there is a significant risk of inconsistencies between different workspaces. This command will overwrite all UCX - ... installations on this given workspace. Result may be consistent only within https://adb-987654321.10.azuredatabricks.net -Workspace name for 987654321 (default: workspace-987654321): labs-workspace -Next workspace id (default: stop): 12345 -Workspace name for 12345 (default: workspace-12345): other-workspace -Next workspace id (default: stop): -14:21:19 INFO [d.l.blueprint.parallel][finding_ucx_installations_11] finding ucx installations 10/89, rps: 24.577/sec -14:21:19 INFO [d.l.blueprint.parallel][finding_ucx_installations_15] finding ucx installations 20/89, rps: 48.305/sec -... -14:21:20 INFO [d.l.ucx.account] Synchronised workspace id mapping for installations on current workspace -``` - -This command is only supposed to be run if the [`sync-workspace-info` command](#sync-workspace-info-command) cannot be -run. It prompts the user to enter the required information manually and creates the workspace info. This command is -useful for workspace administrators who are unable to use the `sync-workspace-info` command, because they are not -Databricks Account Administrators. It can also be used to manually create the workspace info in a new workspace. - -[[back to top](#databricks-labs-ucx)] - -## `create-account-groups` command - -```text -$ databricks labs ucx create-account-groups [--workspace-ids 123,456,789] -``` - -**Requires Databricks Account Administrator privileges.** This command creates account-level groups if a workspace local -group is not present in the account. It crawls all workspaces configured in `--workspace-ids` flag, then creates -account level groups if a WS local group is not present in the account. If `--workspace-ids` flag is not specified, UCX -will create account groups for all workspaces configured in the account. - -The following scenarios are supported, if a group X: -- Exist in workspaces A,B,C, and it has same members in there, it will be created in the account -- Exist in workspaces A,B but not in C, it will be created in the account -- Exist in workspaces A,B,C. It has same members in A,B, but not in C. Then, X and C_X will be created in the account - -This command is useful for the setups, that don't have SCIM provisioning in place. - -Once you're done with this command, proceed to the [group migration workflow](#group-migration-workflow). - -[[back to top](#databricks-labs-ucx)] - -## `validate-groups-membership` command - -```text -$ databricks labs ucx validate-groups-membership -... -14:30:36 INFO [d.l.u.workspace_access.groups] Found 483 account groups -14:30:36 INFO [d.l.u.workspace_access.groups] No group listing provided, all matching groups will be migrated -14:30:36 INFO [d.l.u.workspace_access.groups] There are no groups with different membership between account and workspace -Workspace Group Name Members Count Account Group Name Members Count Difference -``` - -This command validates the groups to see if the groups at the account level and workspace level have different membership. -This command is useful for administrators who want to ensure that the groups have the correct membership. It can also be -used to debug issues related to group membership. See [group migration](docs/local-group-migration.md) and -[group migration](#group-migration-workflow) for more details. - -Valid group membership is important to ensure users has correct access after legacy table ACL is migrated in [table migration process](#Table-Migration) - -[[back to top](#databricks-labs-ucx)] - -## `validate-table-locations` command - -```text -$ databricks labs ucx validate-table-locations [--workspace-ids 123,456,789] -... -11:39:36 WARN [d.l.u.account.aggregate] Workspace 99999999 does not have UCX installed -11:39:37 WARN [d.l.u.account.aggregate] Overlapping table locations: 123456789:hive_metastore.database.table and 987654321:hive_metastore.database.table -11:39:37 WARN [d.l.u.account.aggregate] Overlapping table locations: 123456789:hive_metastore.database.table and 123456789:hive_metastore.another_database.table -``` - -This command validates the table locations by checking for overlapping table locations in the workspace and across -workspaces. Unity catalog does not allow overlapping table locations, also not between tables in different catalogs. -Overlapping table locations need to be resolved by the user before running the table migration. - -Options to resolve tables with overlapping locations are: -- Move one table and [skip](#skip-command) the other(s). -- Duplicate the tables by copying the data into a managed table and [skip](#skip-command) the original tables. - -Considerations when resolving tables with overlapping locations are: -- Migrate the tables one workspace at a time: - - Let later migrated workspaces read tables from the earlier migrated workspace catalogs. - - [Move](#move-command) tables between schemas and catalogs when it fits the data management model. -- The tables might have different: - - Metadata, like: - - Column schema (names, types, order) - - Description - - Tags - - ACLs - -[[back to top](#databricks-labs-ucx)] - -## `cluster-remap` command - -```text -$ databricks labs ucx cluster-remap -21:29:38 INFO [d.labs.ucx] Remapping the Clusters to UC -Cluster Name Cluster Id -Field Eng Shared UC LTS Cluster 0601-182128-dcbte59m -Shared Autoscaling Americas cluster 0329-145545-rugby794 -``` -```text -Please provide the cluster id's as comma separated value from the above list (default: ): -``` - -Once you're done with the [code migration](#code-migration-commands), you can run this command to remap the clusters to UC enabled. - -This command will remap the cluster to uc enabled one. When we run this command it will list all the clusters -and its id's and asks to provide the cluster id's as comma separated value which has to be remapped, by default it will take all cluster ids. -Once we provide the cluster id's it will update these clusters to UC enabled.Back up of the existing cluster -config will be stored in backup folder inside the installed location(backup/clusters/cluster_id.json) as a json file.This will help -to revert the cluster remapping. - -You can revert the cluster remapping using the [`revert-cluster-remap` command](#revert-cluster-remap-command). - -[[back to top](#databricks-labs-ucx)] - -## `revert-cluster-remap` command - -```text -$ databricks labs ucx revert-cluster-remap -21:31:29 INFO [d.labs.ucx] Reverting the Remapping of the Clusters from UC -21:31:33 INFO [d.labs.ucx] 0301-055912-4ske39iq -21:31:33 INFO [d.labs.ucx] 0306-121015-v1llqff6 -Please provide the cluster id's as comma separated value from the above list (default: ): -``` - -If a customer want's to revert the cluster remap done using the [`cluster-remap` command](#cluster-remap-command) they can use this command to revert -its configuration from UC to original one.It will iterate through the list of clusters from the backup folder and reverts the -cluster configurations to original one.This will also ask the user to provide the list of clusters that has to be reverted as a prompt. -By default, it will revert all the clusters present in the backup folder - -[[back to top](#databricks-labs-ucx)] - -## `upload` command - -```text -$ databricks labs ucx upload --file --run-as-collection True -21:31:29 WARNING [d.labs.ucx] The schema of CSV files is NOT validated, ensure it is correct -21:31:29 INFO [d.labs.ucx] Finished uploading: -``` - -Upload a file to a single workspace (`--run-as-collection False`) or a collection of workspaces -(`--run-as-collection True`). This command is especially useful when uploading the same file to multiple workspaces. - -## `download` command - -```text -$ databricks labs ucx download --file --run-as-collection True -21:31:29 INFO [d.labs.ucx] Finished downloading: -``` - -Download a csv file from a single workspace (`--run-as-collection False`) or a collection of workspaces -(`--run-as-collection True`). This command is especially useful when downloading the same file from multiple workspaces. - -## `join-collection` command - -```text -$ databricks labs ucx join-collection --workspace-ids --profile -``` - -`join-collection` command joins 2 or more workspaces into a collection. This helps in running supported cli commands as a collection -`join-collection` command updates config.yml file on each workspace ucx installation with installed_workspace_ids attribute. -In order to run `join-collectioon` command a user should: - - be an Account admin on the Databricks account - - be a Workspace admin on all the workspaces to be joined as a collection) or a collection of workspaces - - have installed UCX on the workspace -The `join-collection` command will fail and throw an error msg if the above conditions are not met. - -## collection eligible command - -Once `join-collection` command is run, it allows user to run multiple cli commands as a collection. The following cli commands -are eligible to be run as a collection. User can run the below commands as collection by passing an additional flag `--run-as-collection=True` -- `ensure-assessment-run` -- `create-table-mapping` -- `principal-prefix-access` -- `migrate-credentials` -- `create-uber-principal` -- `create-missing-principals` -- `validate-external-location` -- `migrate-locations` -- `create-catalog-schemas` -- `migrate-tables` -- `migrate-acls` -- `migrate-dbsql-dashboards` -- `validate-group-membership` -Ex: `databricks labs ucx ensure-assessment-run --run-as-collection=True` - - -# Common Challenges and the Solutions -Users might encounter some challenges while installing and executing UCX. Please find the listing of some common challenges and the solutions below. - -[[back to top](#databricks-labs-ucx)] - -### Network Connectivity Issues - -**From local machine to the Databricks Account and Workspace:** UCX -installation process has to be run from the local laptop using -Databricks CLI and it will deploy the latest version of UCX into the -Databricks workspace. For this reason, the Databricks account and -workspace needs to be accessible from the laptop. Sometimes, the -workspace might have a network isolation, like it can only be reached -from a VPC, or from a specific IP range. - -**Solution:** Please check that your laptop has network connectivity to -the Databricks account and workspace. If not, you might need to be -connected to a VPN or configure an HTTP proxy to access your workspace. - -**From local machine to GitHub:** UCX needs internet access to connect to GitHub (https://api.github.com -and https://raw.githubusercontent.com) for downloading the tool from the machine running the installation. The -installation will fail if there is no internet connectivity to these URLs. - -**Solution:** Ensure that GitHub is reachable from the local machine. If -not, make necessary changes to the network/firewall settings. - -**From Databricks workspace to PyPi:** There are some dependent -libraries which need to be installed from -[pypi.org](https://pypi.org/) to run the UCX workflows from the -Databricks workspace. If the workspace doesn’t have network -connectivity, then the job might fail with -NO_MATCHING_DISTRIBUTION_ERROR. - -**Solution:** Version 0.24.0 of UCX supports workspace with no internet -access. Please upgrade UCX and rerun the installation. Reply *yes* to -the question "Does the given workspace block Internet access?" asked -during installation. It will then upload all necessary dependencies to -the workspace. Also, please note that UCX uses both UC and non-UC -enabled clusters. If you have different proxy settings for each, then -please update the necessary proxies (eg. with init scripts) for each -cluster type. - -**Local machine to Databricks Account and Workspace connection failed due to proxy and self-signed cert:** -When customer uses web proxy and self-singed certification, UCX may not be able to connect to Account and Workspace -with following errors: -``` -File "/Users/userabc/.databricks/labs/ucx/state/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 466, in _make_request - self._validate_conn(conn) - File "/Users/userabc/.databricks/labs/ucx/state/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 1095, in _validate_conn - conn.connect() - File "/Users/userabc/.databricks/labs/ucx/state/venv/lib/python3.10/site-packages/urllib3/connection.py", line 652, in connect - sock_and_verified = _ssl_wrap_socket_and_match_hostname( - File "/Users/userabc/.databricks/labs/ucx/state/venv/lib/python3.10/site-packages/urllib3/connection.py", line 805, in _ssl_wrap_socket_and_match_hostname - ssl_sock = ssl_wrap_socket( - File "/Users/userabc/.databricks/labs/ucx/state/venv/lib/python3.10/site-packages/urllib3/util/ssl_.py", line 465, in ssl_wrap_socket - ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) - File "/Users/userabc/.databricks/labs/ucx/state/venv/lib/python3.10/site-packages/urllib3/util/ssl_.py", line 509, in _ssl_wrap_socket_impl - return ssl_context.wrap_socket(sock, server_hostname=server_hostname) - File "/opt/homebrew/Cellar/python@3.10/3.10.14/Frameworks/Python.framework/Versions/3.10/lib/python3.10/ssl.py", line 513, in wrap_socket - return self.sslsocket_class._create( - File "/opt/homebrew/Cellar/python@3.10/3.10.14/Frameworks/Python.framework/Versions/3.10/lib/python3.10/ssl.py", line 1104, in _create - self.do_handshake() - File "/opt/homebrew/Cellar/python@3.10/3.10.14/Frameworks/Python.framework/Versions/3.10/lib/python3.10/ssl.py", line 1375, in do_handshake - self._sslobj.do_handshake() -ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1007) -``` - -**Solution:** set both `REQUESTS_CA_BUNDLE` and `CURL_CA_BUNDLE` -[to force requests library to set verify=False](https://github.com/psf/requests/blob/8c211a96cdbe9fe320d63d9e1ae15c5c07e179f8/requests/sessions.py#L718) -as well as set `SSL_CERT_DIR` env var pointing to the proxy CA cert for the urllib3 library. - -[[back to top](#databricks-labs-ucx)] - -### Insufficient Privileges - -**User is not a Databricks workspace administrator:** User running the -installation needs to be a workspace administrator as the CLI will -deploy the UCX tool into the workspace, create jobs, and dashboards. - -**Solution:** Identify a workspace admin from your team and ask them to -install UCX with their authentication, or request a workspace -administrator to grant you temporary administrator privileges to run the -installation. More details on the issues that you can run into if -you are not an admin (and some possible solutions) can be found -[here](docs/troubleshooting.md#resolving-common-errors-on-ucx-install). - -**User is not a Cloud IAM Administrator:** Cloud CLI needs to be -installed in the local machine for certain cloud related activities, -like creating an [uber -principal](#create-uber-principal-command). -For this, the user needs Cloud IAM Administrator privileges. - -**Solution:** Work with a cloud administrator in your organization to -run the commands that need cloud administrator rights. - -Admin privileges required for commands: - -| **CLI command** | **Admin privileges** | -|--------------------------------------------------------------------------------------------------|----| -| [install](#install-ucx) | Workspace Admin | -| [account install](#advanced-installing-ucx-on-all-workspaces-within-a-databricks-account) | Account Admin | -| [create-account-groups](#create-account-groups-command) | Account Admin | -| [validate-groups-membership](#validate-groups-membership-command) | Account Admin | -| [create-uber-principal](#create-uber-principal-command) | Cloud Admin | -| [principal-prefix-access](#principal-prefix-access-command) | Cloud Admin | -| [create-missing-principals](#create-missing-principals-command-aws-only) | Cloud Admin | -| [delete-missing-principals](#delete-missing-principals-command-aws-only) | Cloud Admin | -| [migrate-credentials](#migrate-credentials-command) | Cloud Admin, Account Admin / Metastore Admin / CREATE STORAGE CREDENTIAL privilege | -| [migrate-location](#migrate-locations-command) | Metastore Admin / CREATE EXTERNAL LOCATION privilege | -| [create-catalogs-schemas](#create-catalogs-schemas-command) | Metastore Admin / CREATE CATALOG privilege | -| [sync-workspace-info](#sync-workspace-info-command) | Account Admin | -| [manual-workspace-info](#manual-workspace-info-command) | Workspace Admin | - -[[back to top](#databricks-labs-ucx)] - -### Version Issues - -**Python:** UCX needs Python version 3.10 or later. - -**Solution:** Check the current version using `python --version`. If the -version is lower than 3.10, upgrade the local Python version to 3.10 or -higher. - -**Databricks CLI:** Databricks CLI v0.213 or higher is needed. - -**Solution:** Check the current version with `databricks --version`. For -lower versions of CLI, -[update](https://docs.databricks.com/en/dev-tools/cli/install.html#update) -the Databricks CLI on the local machine. - -**UCX:** When you install UCX, you get the latest version. But since UCX -is being actively developed, new versions are released frequently. There -might be issues if you have run the assessment with a much earlier -version, and then trying to run the migration workflows with the latest -UCX version. - -**Solution:** Upgrade UCX, and rerun the assessment job before running -the migration workflows. For some reason, if you want to install a -specific version of UCX, you can do it using the command -`databricks labs install ucx@\`, for example, -`databricks labs install ucx@v0.21.0`. - -[[back to top](#databricks-labs-ucx)] - -### Authentication Issues - -**Workspace Level:** If you are facing authentication issues while -setting up Databricks CLI, please refer to the -[Cryptic errors on authentication](docs/troubleshooting.md#cryptic-errors-on-authentication) -section to resolve the common errors related to authentication, -profiles, and tokens. - -**Account Level:** Not only workspace, but account level authentication -is also needed for installing UCX. If you do not have an account -configured in .databrickscfg, you will get an error message -".databrickscfg does not contain account profiles; please create one -first". - -**Solution:** To authenticate with a Databricks account, consider using -one of the following authentication types: [OAuth machine-to-machine -(M2M) -authentication](https://docs.databricks.com/en/dev-tools/cli/authentication.html#m2m-auth), -[OAuth user-to-machine (U2M) -authentication](https://docs.databricks.com/en/dev-tools/cli/authentication.html#u2m-auth), -[Basic authentication -(legacy)](https://docs.databricks.com/en/dev-tools/cli/authentication.html#basic-auth). - -[[back to top](#databricks-labs-ucx)] - -### Multiple Profiles in Databricks CLI - -**Workspace Level:** More than one workspace profile can be configured -in the .databrickscfg file. For example, you can have profiles set for -Dev and Prod workspaces. You want to install UCX only for the Prod -workspace. - -**Solution:** The Databricks CLI provides an option to select the -[profile](https://docs.databricks.com/en/dev-tools/cli/profiles.html) -using `--profile \` or `-p \`. You can -test that the correct workspace is getting selected by running any -Databricks CLI command. For example, you can run `databricks clusters -list -p prod` and check that the Prod clusters are being returned. Once -the profile is verified, you can run UCX install for that specific -profile: `databricks labs install ucx -p prod`. - -**Account Level:** Multiple account level profiles are set in the -.databrickscfg file. - -**Solution:** The installation command `databricks labs install ucx` -will provide an option to select one account profile. - -[[back to top](#databricks-labs-ucx)] - -### Workspace has an external Hive Metastore (HMS) - -**External HMS connectivity from UCX clusters:** If the workspace has an -external HMS, the clusters running the UCX jobs need to have specific -configurations to connect to the external HMS. Otherwise, UCX assessment -will not be able to assess the tables on HMS. - -**Solution:** Use a cluster policy before installation to set the -required Spark config for connecting to the external HMS, or manually -edit the cluster post-installation to have the correct configurations. -Detailed steps can be found -[here](docs/external_hms_glue.md). +# Documentation -**External HMS connectivity from UCX SQL warehouse:** UCX requires a SQL -warehouse to create tables, run queries, create and refresh dashboards. -If you already have a Pro or Serverless warehouse connected to the -external HMS, you can select the same warehouse for UCX. You will also -be given an option (during installation) to create a new warehouse for -UCX. If you have never used a warehouse before, the new warehouse -created might not have proper configuration set to connect to the -external HMS. +Please refer to the [UCX documentation](https://databrickslabs.github.io/ucx/) for detailed information on how to use UCX. -**Solution:** Set Spark configuration for connecting to external HMS in -the Admin Settings of SQL warehouse. This will only be needed if the -admin settings do not have the configurations already set. For example, -add *spark.hadoop.javax.jdo.option.ConnectionURL \* -under Data Access Configuration of SQL Warehouse Admin Settings. -[[back to top](#databricks-labs-ucx)] +# Developer documentation -### Verify the Installation +Please refer to the [developer documentation](https://databrickslabs.github.io/ucx/docs/dev/) for information on how to contribute to UCX. -Once the UCX command `databricks labs install ucx` has completed -successfully, the installation can be verified with the following steps: +# Troubleshooting -1. Go to the Databricks Catalog Explorer and check if a new schema for - ucx is available in Hive Metastore with all empty tables. +For questions, troubleshooting or bug fixes, please see our [troubleshooting guide](https://databrickslabs.github.io/ucx/docs/reference/troubleshooting/). -2. Check that the UCX jobs are visible under Workflows. +# Feedback -3. Run the assessment. This will start the UCX clusters, crawl through - the workspace, and display results in the UCX dashboards. In case of - external HMS, verify from the results that the assessment has - analyzed the external HMS tables. +We welcome your feedback! Please submit an [an issue](https://github.com/databrickslabs/ucx/issues). -[[back to top](#databricks-labs-ucx)] # Star History [![Star History Chart](https://api.star-history.com/svg?repos=databrickslabs/ucx&type=Date)](https://star-history.com/#databrickslabs/ucx) -[[back to top](#databricks-labs-ucx)] # Project Support Please note that all projects in the databrickslabs GitHub account are provided for your exploration only, and are not formally supported by Databricks with Service Level Agreements (SLAs). They are provided AS-IS, and we do not make any guarantees of any kind. Please do not submit a support ticket relating to any issues arising from the use of these projects. diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000000..25a7f0cd5f --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,3 @@ +.hugo_build.lock +node_modules/ +public/ \ No newline at end of file diff --git a/docs/archetypes/default.md b/docs/archetypes/default.md new file mode 100644 index 0000000000..0d5eebd0dd --- /dev/null +++ b/docs/archetypes/default.md @@ -0,0 +1,5 @@ +--- +date: '{{ .Date }}' +draft: true +title: '{{ replace .File.ContentBaseName "-" " " | title }}' +--- diff --git a/docs/assets/css/custom.css b/docs/assets/css/custom.css new file mode 100644 index 0000000000..b3b48d25be --- /dev/null +++ b/docs/assets/css/custom.css @@ -0,0 +1,22 @@ +@import url('https://fonts.googleapis.com/css2?family=DM+Mono:ital,wght@0,300;0,400;0,500;1,300;1,400;1,500&family=DM+Sans:ital,opsz,wght@0,9..40,100..1000;1,9..40,100..1000&display=swap'); + +.font-mono { + font-family: "DM Mono", monospace; + font-style: normal; +} + +.font-sans { + font-family: "DM Sans", sans-serif; + font-style: normal; +} + +.content { + @apply font-sans; +} + + +:root { + --primary-hue: 2; + --primary-saturation: 98%; + --primary-lightness: 41.8%; +} \ No newline at end of file diff --git a/docs/assets/css/tailwind.css b/docs/assets/css/tailwind.css new file mode 100644 index 0000000000..a461c505f1 --- /dev/null +++ b/docs/assets/css/tailwind.css @@ -0,0 +1 @@ +@import "tailwindcss"; \ No newline at end of file diff --git a/docs/content/_index.md b/docs/content/_index.md new file mode 100644 index 0000000000..75e0ed49e9 --- /dev/null +++ b/docs/content/_index.md @@ -0,0 +1,7 @@ +--- +date: '2024-11-15T15:13:18+01:00' +draft: true +title: '' +description: 'Just a placeholder, actual content is in hextra-home.html' +layout: 'hextra-home' +--- diff --git a/docs/content/docs/_index.md b/docs/content/docs/_index.md new file mode 100644 index 0000000000..0bbaa17079 --- /dev/null +++ b/docs/content/docs/_index.md @@ -0,0 +1,43 @@ +--- +linkTitle: "Documentation" +title: Introduction +--- + +👋 Hello! Welcome to the UCX documentation! + +## What is UCX? + +UCX is a companion toolkit to assist users with upgrading to Unity Catalog (UC). It is provided in a form of CLI tool that can be used to automate the process of upgrading to UC. + + +Usual steps to upgrade to UC include infra-level configuration, which is followed by tables and code migration. + +Infra-level configuration includes: +- Setting up a UC metastore +- Assess the current state of your workspaces by running UCX assessment +- Attach workspaces to UC metastore +- Migrate workspace-level groups to account-level groups + +Tables and code migration includes: +- Setting up cloud-level infra for UC (storage credentials, external locations, volumes) +- Migrate tables to UC catalogs +- Adjust code to be compatible with UC-enabled compute +- Switch clusters to UC-enabled compute +- Change downstream consumers (e.g. BI tools) to use UC-enabled compute and UC catalogs + + +## Questions or Feedback? + +{{< callout emoji="💻" type="info" >}} + UCX is still in active development.\ + Have a question or feedback? Feel free to [open an issue](https://github.com/databrickslabs/ucx/issues) +{{< /callout >}} + +## Next + +Dive right into the following section to get started: + +{{< cards >}} + {{< card link="installation" title="Installation" icon="document-text" subtitle="Learn how to install UCX" >}} + {{< card link="process/overview" title="Overview" icon="document-text" subtitle="Check out the migration process overview" >}} +{{< /cards >}} \ No newline at end of file diff --git a/docs/content/docs/dev/_index.md b/docs/content/docs/dev/_index.md new file mode 100644 index 0000000000..2d4bb83117 --- /dev/null +++ b/docs/content/docs/dev/_index.md @@ -0,0 +1,15 @@ +--- +linkTitle: "Developer Documentation" +title: Introduction into developing UCX +weight: 5 +--- + +The sections here are dedicated mainly to developers who are interested in contributing to UCX. If you are a user looking for information on how to use UCX, please refer to the [User Documentation](docs/). + +## Next + +Dive right into the following section to get started: +{{< cards >}} + {{< card link="contributing" title="Contributing" icon="document-text" subtitle="Learn how to contribute to UCX" >}} + {{< card link="docs" title="Write docs" icon="document-text" subtitle="Learn how to write documentation for UCX" >}} +{{< /cards >}} \ No newline at end of file diff --git a/CONTRIBUTING.md b/docs/content/docs/dev/contributing.md similarity index 97% rename from CONTRIBUTING.md rename to docs/content/docs/dev/contributing.md index 3baf859c77..d4d1943aba 100644 --- a/CONTRIBUTING.md +++ b/docs/content/docs/dev/contributing.md @@ -1,4 +1,8 @@ -# Contributing +--- +linkTitle: "Contributing to UCX" +title: "Contributing to UCX" +weight: 1 +--- ## First Principles @@ -27,7 +31,7 @@ or specialized functionality unavailable in standard libraries. ## Adding entries to `known.json` file -When adding a new entry to the [`known.json` file](src/databricks/labs/ucx/source_code/known.json), ensure that the entry is unique and follows the correct format. The `known.json` file +When adding a new entry to the [`known.json` file](https://github.com/databrickslabs/ucx/blob/main/src/databricks/labs/ucx/source_code/known.json), ensure that the entry is unique and follows the correct format. The `known.json` file stores information about known (in)compatibilities with Unity Catalog. The file speeds up static code analysis and prevents false positives. 1. load into virtual environment @@ -127,7 +131,7 @@ for the latest reference of environment variables related to authentication. indicate the cloud provider being used, such as "aws" for Amazon Web Services and "azure" for Microsoft Azure. - `DATABRICKS_ACCOUNT_ID`: This variable stores the unique identifier for your Databricks account. - `DATABRICKS_HOST`: This variable contains the URL of your Databricks workspace. It is the web address you use to access - your Databricks environment and typically looks like "https://dbc-....cloud.databricks.com." + your Databricks environment and typically looks like `https://dbc-....cloud.databricks.com`. - `TEST_DEFAULT_CLUSTER_ID`: This variable holds the identifier for the default cluster used in testing. The value resembles a unique cluster ID, like "0824-163015-tdtagl1h." - `TEST_DEFAULT_WAREHOUSE_DATASOURCE_ID`: This environment variable stores the identifier for the default warehouse data @@ -214,7 +218,7 @@ with the [Python plugin (Community Edition)](https://plugins.jetbrains.com/plugi IntelliJ CE, then it would work in PyCharm. Debugging capabilities are essential for troubleshooting and diagnosing issues during development. Please make sure that your test setup allows for easy debugging by following best practices. -![debugging tests](docs/debugging-tests.gif) +![debugging tests](/images/debugging-tests.gif) Adhering to these guidelines ensures that our integration tests are robust, efficient, and easily maintainable. This, in turn, contributes to the overall reliability and quality of our software. @@ -263,7 +267,7 @@ HATCH_PYTHON="$(which python3.10)" make clean dev test ``` Configure your IDE to use `.venv/bin/python` from the virtual environment when developing the project: -![IDE Setup](docs/hatch-intellij.gif) +![IDE Setup](/images/hatch-intellij.gif) Verify installation with diff --git a/docs/content/docs/dev/docs.md b/docs/content/docs/dev/docs.md new file mode 100644 index 0000000000..f5c0c1e9d5 --- /dev/null +++ b/docs/content/docs/dev/docs.md @@ -0,0 +1,155 @@ +--- +linkTitle: "Writing documentation" +title: "Writing documentation" +weight: 2 +--- + +UCX documentation uses the following set of technologies: + +- [Hugo](https://gohugo.io/) - Static site generator written in Go +- [Hextra](https://imfing.github.io/hextra/) - Hugo theme for documentation +- [TailwindCSS](https://tailwindcss.com/) - Utility-first CSS framework, mainly used for custom styling + + +## Prerequisites + +To build the documentation, you need to have the following tools installed: +- [Hugo](https://gohugo.io/getting-started/installing/) +- [Node.js](https://nodejs.org/en/download/) +- [Yarn](https://classic.yarnpkg.com/en/docs/install/) + +On MacOS, you can install the prerequisites using [Homebrew](https://brew.sh/): + +```bash +brew install hugo node +npm install -g yarn +``` + +{{}} +The documentation is built using Hugo, a static site generator. The content is written in Markdown and the site is styled using TailwindCSS. We only need Node.JS and `yarn` to make TailwindCSS work locally during development. +{{}} + +## Installing tailwindcss + +To install the dependencies, run the following commands: + +```bash +cd docs && yarn install +``` + +## Installing Hugo + +To install Hugo, follow the instructions on the [Hugo website](https://gohugo.io/getting-started/installing/). + +## Running the documentation locally + +To run the documentation locally, run the following command: + +```bash +cd docs && hugo server +``` + +Follow the instructions in the terminal to open the documentation in your browser. The default URL is `http://localhost:1313/ucx`. + +## Adding static files or images + +To add static files or images, place them in the `docs/static` directory. + +{{< callout type="info" >}} +All images, including `.gif` files should be stored in the `docs/static/images` directory. +{{< /callout >}} + +You can reference them in your markdown files using the following syntax: + +```markdown +![Alt text](/images/image.png) +``` + +{{< callout type="info" >}} +Please note that image paths should start with `/images/` and be relative to the `docs/static` directory. + +{{< /callout >}} + +## Using shortcodes + +Shortcodes are used to add reusable components to the documentation. You can find the list of available shortcodes in the [Hextra documentation](https://imfing.github.io/hextra/docs/guide/shortcodes/). + +We mainly use the following shortcodes: +- [callout](https://imfing.github.io/hextra/docs/guide/shortcodes/callout/) +- [tabs](https://imfing.github.io/hextra/docs/guide/shortcodes/tabs/) +- [steps](https://imfing.github.io/hextra/docs/guide/shortcodes/steps/) + + +## Using links + +When linking to other pages in the documentation, use the following syntax: + +```markdown +[Link text](docs/reference/workflows/group_migration.md) +``` + +Please note that paths should be relative to the `docs/content` directory. + +{{< callout type="info" >}} +Please make sure to use correct paths when linking to other pages in the documentation. This way, the links will work correctly when the documentation is deployed. +{{< /callout >}} + +To reference the index for a folder (e.g. `_index.md` file), use the following syntax: + +```markdown +[Link text](docs/reference/workflows/) +``` + +This will link to the `_index.md` file in the `docs/content/docs/reference/workflows` directory. + +Links to specific anchors in other files should be formatted as follows: + +```markdown +[Link text](docs/reference/workflows/group_migration.md#group-migration-workflow) +``` + +Links to local anchors should be formatted as follows: + +```markdown +[Link text](#using-links) +``` + + + +## Writing content + +We follow a strict structure for writing content in the documentation. Each markdown file should have the following front matter: + +```markdown +--- +linkTitle: "Link title" +title: "Title" +weight: 1 # optional, defines the order of the page in the sidebar +--- +``` + +Our documentation is split into the following 4 pieces: + +- Installation: guidance on how to install UCX +- Process: guidance on how to use UCX in simple, step-by-step instructions +- Reference: detailed information about UCX features, commands, codes, configurations, etc. +- Developer documentation: anythign related to the development of UCX, as well as technical implementation details + + +{{< callout type="warning" >}} +Please don't put any technical implementation details in the installation, process, or reference sections. These sections are meant to be user-friendly and easy to understand. +When in doubt, please ask for a review. +{{< /callout >}} + +Please make sure (especially in Process and Reference sections) to correctly order the pages in the sidebar. The order should be logical and easy to follow for the user. + +Please don't use nesting when not necessary. 3 levels of nesting are maximum. If you need more, please consider restructuring the content. + + +## File naming + +Please follow the `snake_case.md` naming convention for markdown files. This makes it easier to find and reference files in the documentation. For example: + +- `group_migration.md` +- `table.md` + diff --git a/docs/content/docs/dev/implementation/_index.md b/docs/content/docs/dev/implementation/_index.md new file mode 100644 index 0000000000..ed834aa463 --- /dev/null +++ b/docs/content/docs/dev/implementation/_index.md @@ -0,0 +1,7 @@ +--- +title: "Implementation details" +linkTitle: "Implementation details" +weight: 3 +--- + +This section provides detailed information about the implementation of UCX. \ No newline at end of file diff --git a/docs/group_name_conflict.md b/docs/content/docs/dev/implementation/group_name_conflict.md similarity index 91% rename from docs/group_name_conflict.md rename to docs/content/docs/dev/implementation/group_name_conflict.md index d9223434ca..1258ea22c6 100644 --- a/docs/group_name_conflict.md +++ b/docs/content/docs/dev/implementation/group_name_conflict.md @@ -1,7 +1,7 @@ Group Name Conflict Resolution === -See [this document](local-group-migration.md) for workspace group migration. +See [this document](docs/dev/implementation/local_group_migration.md) for workspace group migration. When migrating multiple workspaces we can run into conflicts. These conflicts occur when groups with the same name in different workspaces have different membership and different @@ -26,7 +26,7 @@ The user then input the Prefix/Suffix/Regular Expression. The installation process will validate the regular expression. The installation process will register the selection as regular expression in the configuration YAML file. -We introduce 3 more parameters to the [configuration](../README.md#open-remote-config-command) and the group manager: +We introduce 3 more parameters to the [configuration](docs/reference/commands.md#open-remote-config) and the group manager: - workspace_group_regex - workspace_group_replace diff --git a/docs/content/docs/dev/implementation/installation.md b/docs/content/docs/dev/implementation/installation.md new file mode 100644 index 0000000000..2f1307032b --- /dev/null +++ b/docs/content/docs/dev/implementation/installation.md @@ -0,0 +1,26 @@ +--- +title: Installation process +linkTitle: Installation process +--- + +The `WorkspaceInstaller` class is used to create a new configuration for Unity Catalog migration in a Databricks workspace. + +It guides the user through a series of prompts to gather necessary information, such as: +- selecting an inventory database +- choosing a PRO or SERVERLESS SQL warehouse +- specifying a log level and number of threads +- setting up an external Hive Metastore (if necessary) + + + +The [`WorkspaceInstallation`](https://github.com/databrickslabs/ucx/blob/main/src/databricks/labs/ucx/install.py) manages the installation and uninstallation of UCX in a workspace. + +It handles the configuration and exception management during the installation process. +The installation process creates dashboards, databases, and jobs. + +It also includes the creation of a database with given configuration and the deployment of workflows with specific settings. The installation process can handle exceptions and infer errors from job runs and task runs. + +The workspace installation uploads wheels, creates cluster policies,and wheel runners to the workspace. + +It can also handle the creation of job tasks for a given task, such as job dashboard tasks, job notebook tasks, and job wheel tasks. The class handles the installation of UCX, including configuring the workspace, installing necessary libraries, and verifying the installation, making it easier for users to migrate their workspaces to UCX. + diff --git a/docs/local-group-migration.md b/docs/content/docs/dev/implementation/local_group_migration.md similarity index 89% rename from docs/local-group-migration.md rename to docs/content/docs/dev/implementation/local_group_migration.md index 9a6b07f254..18278a7c0c 100644 --- a/docs/local-group-migration.md +++ b/docs/content/docs/dev/implementation/local_group_migration.md @@ -1,22 +1,13 @@ -Workspace Group Migration -=== - - -* [Workspace Group Migration](#workspace-group-migration) -* [Design](#design) - * [Group Manager](#group-manager) - * [Permission Manager](#permission-manager) - * [ACL Support](#acl-support) - * [Generic Permissions](#generic-permissions) - * [Dashboard Permissions](#dashboard-permissions) - * [Entitlements and Roles](#entitlements-and-roles) - * [Secret Scope Permissions](#secret-scope-permissions) - * [Legacy Table Access Controls](#legacy-table-access-controls) -* [Troubleshooting](#troubleshooting) - +--- +title: Local Group Migration +linkTitle: Local Group Migration +--- + +Upon the first installation, user is prompted for a workspace local [group migration strategy](docs/dev/implementation/group_name_conflict.md). +Based on user input, the class creates a new cluster policy with the specified configuration. The user can review and confirm the configuration, which is saved to the workspace and can be opened in a web browser. This feature introduces the ability to migrate groups from workspace level to account level in -the [group migration workflow](../README.md#group-migration-workflow). It helps you to upgrade all Databricks workspace assets: +the [group migration workflow](docs/reference/workflows/group_migration.md). It helps you to upgrade all Databricks workspace assets: Legacy Table ACLs, Entitlements, AWS instance profiles, Clusters, Cluster policies, Instance Pools, Databricks SQL warehouses, Delta Live Tables, Jobs, MLflow experiments, MLflow registry, SQL Dashboards & Queries, SQL Alerts, Token and Password usage permissions that are set on the workspace level, Secret scopes, Notebooks, @@ -35,7 +26,7 @@ The group migration workflow can be executed multiple times to ensure that all t 5. `validate_groups_permissions`: This task validates that all the crawled permissions are applied correctly to the destination groups. 6. `delete_backup_groups`: This task removes all workspace-level backup groups, along with their permissions. This should only be executed after confirming that the workspace-local migration worked successfully for all the groups involved. This step is necessary to clean up the workspace and remove any unnecessary groups and permissions. -[[back to top](#workspace-group-migration)] + # Design @@ -44,7 +35,7 @@ the original and new names, as well as the group's members, external ID, and rol the state of the migration process and provides methods for getting the target principal and temporary name for a given group name. -[[back to top](#workspace-group-migration)] + ## Group Manager @@ -58,12 +49,12 @@ of `MigratedGroup` objects based on a mapping between workspace and account grou The `MatchingNamesStrategy`, `MatchByExternalIdStrategy`, `RegexSubStrategy`, and `RegexMatchStrategy` classes are concrete implementations of this interface. See [group name conflicts](group_name_conflict.md) for more details. -The `ConfigureGroups` class provides a command-line interface for configuring the group migration process during [installation](../README.md#installation). +The `ConfigureGroups` class provides a command-line interface for configuring the group migration process during [installation](docs/installation). It prompts the user to enter information about the group migration strategy, such as the renamed group prefix, regular expressions for matching and substitution, and a list of groups to migrate. The class also provides methods for validating user input and setting class variables based on the user's responses. -[[back to top](#workspace-group-migration)] + ## Permission Manager @@ -77,9 +68,9 @@ is created, you can call the `inventorize_permissions` method to crawl and save the inventory database in the `permissions` table. The `apply_group_permissions` method allows you to apply the permissions to a list of account groups, while -the [`verify_group_permissions` method](../README.md#validate-groups-membership-command) verifies that the permissions are valid. +the [`verify_group_permissions` method](docs/reference/commands.md#validate-groups-membership) verifies that the permissions are valid. + -[[back to top](#workspace-group-migration)] ## ACL Support @@ -93,7 +84,7 @@ The `AclSupport` objects define how to crawl, save, and apply permissions for sp The `Permissions` dataclass is used to represent the permissions for a specific object type and ID. The dataclass includes a `raw` attribute that contains the raw permission data as a string, providing a convenient way to work with the underlying permission data. -[[back to top](#workspace-group-migration)] + ### Generic Permissions @@ -130,13 +121,13 @@ he `_safe_get_permissions` and `_safe_updatepermissions` methods are used to saf a given object type and ID, respectively. These methods handle exceptions that may occur during the API calls and log appropriate warning messages. -[[back to top](#workspace-group-migration)] + ### Dashboard Permissions -Reflected in [RedashPermissionsSupport](../src/databricks/labs/ucx/workspace_access/redash.py). See [examples](../tests/integration/workspace_access/test_redash.py) for more details on how to use it as a library. +Reflected in [RedashPermissionsSupport](https://github.com/databrickslabs/ucx/blob/main/src/databricks/labs/ucx/workspace_access/redash.py). See [examples](https://github.com/databrickslabs/ucx/blob/main/tests/integration/workspace_access/test_redash.py) for more details on how to use it as a library. + -[[back to top](#workspace-group-migration)] ### Entitlements and Roles @@ -144,9 +135,9 @@ The `ScimSupport` is [`AclSupport`](#acl-support) that creates a snapshot of all The `_is_item_relevant` method checks if a permission item is relevant to the current migration state. The `get_crawler_tasks` method returns an iterator of partial functions for crawling the permissions of each group in the snapshot. It checks if the group has any roles or entitlements and returns a partial function to crawl the corresponding property. -See [examples](../tests/integration/workspace_access/test_scim.py) for more details on how to use it as a library. +See [examples](https://github.com/databrickslabs/ucx/blob/main/tests/integration/workspace_access/test_scim.py) for more details on how to use it as a library. + -[[back to top](#workspace-group-migration)] ### Secret Scope Permissions @@ -155,7 +146,7 @@ all secret scopes, applying and verifying ACLs, and checking if a `Permissions` migration state. It simplifies the process of managing permissions on secret scopes by checking if the ACLs have been applied correctly, and if not, automatically reapplying them. -[[back to top](#workspace-group-migration)] + ### Legacy Table Access Controls @@ -167,11 +158,11 @@ Furthermore, the `_apply_grant` method applies the ACL permission to the target the `Grant` object matches the ACL permission for that object and principal in the database. The `get_verify_task` method calls the `_verify` method with the object type, object ID, and `Grant` object from the `Permissions` object. -[[back to top](#workspace-group-migration)] + # Troubleshooting -Use [`DEBUG` notebook](../README.md#debug-notebook) to troubleshoot anything. +Use [`DEBUG` notebook](docs/installation/resources.md#debug-notebook) to troubleshoot anything. Below are some useful code snippets that can be useful for troubleshooting. Make sure to install [databricks-sdk](https://docs.databricks.com/en/dev-tools/sdk-python.html) on the cluster to run @@ -291,4 +282,4 @@ finally: file_handler.close() ``` -[[back to top](#workspace-group-migration)] + diff --git a/docs/table_upgrade.md b/docs/content/docs/dev/implementation/table_upgrade.md similarity index 89% rename from docs/table_upgrade.md rename to docs/content/docs/dev/implementation/table_upgrade.md index 315f1e144d..4e87630a6d 100644 --- a/docs/table_upgrade.md +++ b/docs/content/docs/dev/implementation/table_upgrade.md @@ -1,27 +1,6 @@ Table Upgrade === - -* [Table Upgrade](#table-upgrade) -* [Migration dashboard](#migration-dashboard) -* [Common considerations](#common-considerations) - * [Tables (Parquet/Delta) on DBFS root](#tables-parquetdelta-on-dbfs-root) - * [Tables (Parquet/Delta) on Cloud Storage](#tables-parquetdelta-on-cloud-storage) - * [Tables (None Parquet/Delta)](#tables-none-parquetdelta) - * [Views](#views) - * [Functions](#functions) - * [Account Consideration](#account-consideration) - * [Open Questions](#open-questions) -* [Design](#design) - * [Data Access Permissions](#data-access-permissions) - * [External Storage](#external-storage) - * [Table Mapping](#table-mapping) - * [Migrating Tables](#migrating-tables) - * [Moving tables](#moving-tables) - * [Table Size Estimation](#table-size-estimation) - * [Table Crawler](#table-crawler) - * [Table](#table) - The Hive Metastore migration process will upgrade the following Assets: @@ -33,18 +12,18 @@ We don't expect this process to be a "one and done" process. The table migration and may require a few runs. The migration process is implemented within a workflow that can be invoked multiple times. Each time it will upgrade tables it can and report the ones it can't. -[[back to top](#table-upgrade)] -# Migration dashboard + +## Migration dashboard We keep track of the migration and provide the user continuous feedback of the progress and status of the upgrade. This feedback is presented in the migration dashboard: -![report](migration-report.png) +![report](/images/migration-report.png) + -[[back to top](#table-upgrade)] -# Common considerations +## Common considerations 1. One view per workspace summarizing all the table inventory and various counters 1. By default, we create a single catalog per HMS (_), happens at the account level. @@ -82,9 +61,9 @@ upgrade. This feedback is presented in the migration dashboard: 1. We should consider automating ACLs based on Instance Profiles / Service Principals and other legacy security mechanisms -[[back to top](#table-upgrade)] +## Tables overview -## Tables (Parquet/Delta) on DBFS root +### Tables (Parquet/Delta) on DBFS root 1. By default, we copy the table content (CTAS) 1. Allow skipping individual tables/databases @@ -93,25 +72,25 @@ upgrade. This feedback is presented in the migration dashboard: 1. Allow overriding target to an external table 1. Allow an exception list in case we want to skip certain tables -[[back to top](#table-upgrade)] -## Tables (Parquet/Delta) on Cloud Storage + +### Tables (Parquet/Delta) on Cloud Storage 1. Verify that we have the external locations for these tables 1. Automate creation of External Locations (Future) 1. Use sync to upgrade these tables "in place". Use the default or override catalog.database destination. 1. Update the source table with "upgraded_to" property -[[back to top](#table-upgrade)] -## Tables (None Parquet/Delta) + +### Tables (None Parquet/Delta) 1. Copy this table using CTAS or Deep Clone. (Consider bringing history) 1. Copy the Metadata 1. Skip tables as needed based on size threshold or an exception list 1. Update the source table with "upgraded_to" property -[[back to top](#table-upgrade)] + ## Views @@ -122,14 +101,14 @@ upgrade. This feedback is presented in the migration dashboard: 1. Handle or highlight other cases (functions/storage references/ETC) 1. Create an exception list with views failures -[[back to top](#table-upgrade)] + ## Functions 1. We should migrate (if possible) functions 1. Address incompatibilities -[[back to top](#table-upgrade)] + ## Account Consideration @@ -147,7 +126,7 @@ upgrade. This feedback is presented in the migration dashboard: 1. Consider upgrading a workspace at a time. Highlight the conflict with prior upgrades. 1. Allow workspace admins to upgrade more than one workspace. -[[back to top](#table-upgrade)] + ## Open Questions @@ -156,7 +135,7 @@ upgrade. This feedback is presented in the migration dashboard: 1. What mechanism do we use to map source to target databases 1. How to list workspaces in Azure/AWS -[[back to top](#table-upgrade)] + # Design @@ -174,7 +153,7 @@ that normalizes the input parameters and returns a tuple of the object type and the specified object. The code also includes methods for generating SQL statements to grant and revoke privileges in Hive and Unity Catalog (UC) systems. -[[back to top](#table-upgrade)] + ## External Storage @@ -189,7 +168,7 @@ rocesses the external locations based on certain conditions. as well as a method for creating a snapshot of the current mounts. The `_deduplicate_mounts` method removes any duplicate mounts based on their name and source. -[[back to top](#table-upgrade)] + ## Table Mapping @@ -204,7 +183,7 @@ the mappings to a file, skipping tables and schemas, and checking if a table is The `TableMapping` class is initialized with an `Installation` object, a `WorkspaceClient` object, and a `SqlBackend` object, which are used to interact with the Unity Catalog, the workspace, and to execute SQL queries. -[[back to top](#table-upgrade)] + ## Migrating Tables @@ -224,7 +203,7 @@ The `_revert_migrated_table` method is responsible for reverting the migration o The `is_upgraded` method checks if a table has been upgraded or not. The `print_revert_report` method generates a report of the tables that can be reverted. -[[back to top](#table-upgrade)] + ## Moving tables @@ -239,7 +218,7 @@ associated with the object. The `_reapply_grants` method reapplies the grants on that the necessary permissions are maintained. The `_recreate_table` and `_recreate_view` methods recreate the table or view in the destination schema, including any dependencies or permissions associated with the object. -[[back to top](#table-upgrade)] + ## Table Size Estimation @@ -254,7 +233,7 @@ size. The `_try_load` method tries to load table information from the database a error if the table cannot be found. The `_crawl` method crawls and lists tables using the `tables_crawler` object and calculates the size of DBFS root tables, skipping tables that are not of type `TABLE` or are not DBFS root tables. -[[back to top](#table-upgrade)] + ## Table Crawler @@ -262,7 +241,7 @@ The `TablesCrawler` is designed for crawling and listing tables within Hive Meta about each table, including the table's name, external location, and storage format. This information can be used to better understand the structure and contents of the tables in the Databricks workspace. -[[back to top](#table-upgrade)] + ## Table @@ -282,4 +261,4 @@ The `Table` class has the following methods: * `sql_migrate_dbfs`: Returns an SQL command to migrate a table located in DBFS. * `sql_migrate_view`: Returns an SQL command to migrate a view. -[[back to top](#table-upgrade)] + diff --git a/docs/content/docs/installation/_index.md b/docs/content/docs/installation/_index.md new file mode 100644 index 0000000000..2ddf3100ce --- /dev/null +++ b/docs/content/docs/installation/_index.md @@ -0,0 +1,32 @@ +--- +linkTitle: "Installation" +title: Installation +weight: 1 +--- + +This guide will help you to check the prerequisites and install UCX. + +## Overview + +Briefly, the process involves the following steps: + +{{% steps %}} + +### Setting up the environment + +Please fullfill the [prerequisites](docs/installation/prerequisites/) before installing UCX. + +### Install and authenticate Databricks CLI + +Please [install and authenticate Databricks CLI](docs/installation/databricks_cli/) on your local machine. + +### Install the toolkit + +Install UCX via Databricks CLI. Please follow the instructions to [install UCX](docs/installation/install_ucx). + +### [Optional] Joining a collection + +Workspaces can be grouped into collections to manage UCX migration at collection level. +Please follow the instructions to [join a collection](docs/installation/collection) if you want to join an existing collection or create a new one. + +{{% /steps %}} \ No newline at end of file diff --git a/docs/content/docs/installation/advanced.md b/docs/content/docs/installation/advanced.md new file mode 100644 index 0000000000..071076a528 --- /dev/null +++ b/docs/content/docs/installation/advanced.md @@ -0,0 +1,104 @@ +--- +title: Advanced +linkTitle: Advanced +weight: 5 +--- + +Advanced installation options and other actions are detailed below. + +## Force install over existing UCX +Using an environment variable `UCX_FORCE_INSTALL` you can force the installation of UCX over an existing installation. + +The values for the environment variable are `global` and `user`. + +- Global Install: When UCX is installed at `/Applications/ucx` +- User Install: When UCX is installed at `/Users//.ucx` + +If there is an existing global installation of UCX, you can force a user installation of UCX over the existing installation by setting the environment variable `UCX_FORCE_INSTALL` to 'global'. + +At this moment there is no global override over a user installation of UCX. As this requires migration and can break existing installations. + + +| global | user | expected install location | install_folder | mode | +|--------|------|---------------------------|---------------------|-------------------------------------------------| +| no | no | default | `/Applications/ucx` | install | +| yes | no | default | `/Applications/ucx` | upgrade | +| no | yes | default | `/Users/X/.ucx` | upgrade (existing installations must not break) | +| yes | yes | default | `/Users/X/.ucx` | upgrade | +| yes | no | **USER** | `/Users/X/.ucx` | install (show prompt) | +| no | yes | **GLOBAL** | ... | migrate | + + +* `UCX_FORCE_INSTALL=user databricks labs install ucx` - will force the installation to be for user only +* `UCX_FORCE_INSTALL=global databricks labs install ucx` - will force the installation to be for root only + + + + +## Installing UCX on all workspaces within a Databricks account +Setting the environment variable `UCX_FORCE_INSTALL` to 'account' will install UCX on all workspaces within a Databricks account. + +* `UCX_FORCE_INSTALL=account databricks labs install ucx` + +After the first installation, UCX will prompt the user to confirm whether to install UCX on the remaining workspaces with the same answers. If confirmed, the remaining installations will be completed silently. + +This installation mode will automatically select the following options: +* Automatically create and enable HMS lineage init script +* Automatically create a new SQL warehouse for UCX assessment + + + +## Installing UCX with company hosted pypi mirror +{{< callout type="info" >}} +If you're using custom Pypi, please reply *yes* to the question "Does the given workspace block Internet access"? +{{< /callout >}} + +Some enterprise block the public pypi index and host a company controlled pypi mirror. + +To install UCX while using a company hosted pypi mirror for finding its dependencies, add all UCX dependencies to the company hosted pypi mirror (see +"dependencies" in [`pyproject.toml`](https://github.com/databrickslabs/ucx/blob/main/pyproject.toml)) and set the environment variable `PIP_INDEX_URL` to the company hosted PYPI mirror URL while installing UCX: + +```bash +PIP_INDEX_URL="https://url-to-company-hosted-pypi.internal" databricks labs install ucx +``` + + + + + +## Upgrading UCX for newer versions + +Verify that UCX is installed + +```bash +databricks labs installed + +Name Description Version +ucx Unity Catalog Migration Toolkit (UCX) +``` + +Upgrade UCX via Databricks CLI: + +```bash +databricks labs upgrade ucx +``` + +The prompts will be similar to [Installation](docs/installation/install_ucx) + +![macos_upgrade_ucx](/images/macos_3_databrickslabsmac_upgradeucx.gif) + + + +## Uninstall UCX + +Uninstall UCX via Databricks CLI: + +```bash +databricks labs uninstall ucx +``` + +Databricks CLI will confirm a few options: +- Whether you want to remove all ucx artefacts from the workspace as well. Defaults to no. +- Whether you want to delete the inventory database in `hive_metastore`. Defaults to no. + +![macos_uninstall_ucx](/images/macos_4_databrickslabsmac_uninstallucx.gif) \ No newline at end of file diff --git a/docs/content/docs/installation/collection.md b/docs/content/docs/installation/collection.md new file mode 100644 index 0000000000..27728b437c --- /dev/null +++ b/docs/content/docs/installation/collection.md @@ -0,0 +1,17 @@ +--- +title: Collection +weight: 4 +--- + +At the end of the installation, the user will be prompted if the current installation needs to join an existing collection (or create new collection if none present). + +For large organization with many workspaces, grouping workspaces into collection helps in managing UCX migration at collection level (instead of workspaces level) + +{{< callout type="warning" >}} +User should be an account admin to be able to join a collection. +{{< /callout >}} + + +{{}} +You can read more about cross-workspace installation [here](docs/reference/commands.md#cross-workspace-installations). +{{< /callout >}} diff --git a/docs/content/docs/installation/databricks_cli.md b/docs/content/docs/installation/databricks_cli.md new file mode 100644 index 0000000000..a10e3626eb --- /dev/null +++ b/docs/content/docs/installation/databricks_cli.md @@ -0,0 +1,34 @@ +--- +linkTitle: "Install and authenticate Databricks CLI" +title: "Install and authenticate Databricks CLI" +weight: 2 +--- + + +{{< callout type="info" >}} +We only support installations and upgrades through [Databricks CLI](https://docs.databricks.com/en/dev-tools/cli/index.html), as UCX requires an installation script run to make sure all the necessary and correct configurations are in place. +{{< /callout >}} + +Please follow the instructions below to install Databricks CLI on your local machine: + +{{< tabs items="macOS,Windows" >}} + {{< tab >}} + For MacOS we recommend using `brew`: + ![macos_install_databricks](/images/macos_1_databrickslabsmac_installdatabricks.gif) + {{< /tab >}} + + {{< tab >}} + For Windows we recommend using `winget`: + ![windows_install_databricks.png](/images/windows_install_databricks.png) + {{< /tab >}} + +{{< /tabs >}} + + +Once you install Databricks CLI, authenticate your current machine to a Databricks Workspace: + +```bash +databricks auth login --host WORKSPACE_HOST +``` + +To enable debug logs, simply add `--debug` flag to any command. \ No newline at end of file diff --git a/docs/content/docs/installation/install_ucx.md b/docs/content/docs/installation/install_ucx.md new file mode 100644 index 0000000000..996aa65e33 --- /dev/null +++ b/docs/content/docs/installation/install_ucx.md @@ -0,0 +1,29 @@ +--- +linkTitle: "Install UCX" +title: Install UCX +weight: 3 +--- + + +Install UCX via Databricks CLI: + +```bash +databricks labs install ucx +``` + +You'll be prompted to select a [configuration profile](https://docs.databricks.com/en/dev-tools/auth.html#databricks-client-unified-authentication) created by `databricks auth login` command. + +After running this command, UCX will be installed locally and a number of assets will be deployed in the selected workspace. + +These assets are available under the installation folder, i.e. `/Applications/ucx` is the default installation folder. Please check [here](docs/installation/advanced.md#force-install-over-existing-ucx) for more details. + +{{< callout type="info" >}} +You can also install a specific version by specifying it like `@vX.Y.Z`: +```bash +databricks labs install ucx@vX.Y.Z +``` +{{< /callout >}} + +Here is a demo of the installation process: +![macos_install_ucx](/images/macos_2_databrickslabsmac_installucx.gif) + diff --git a/docs/content/docs/installation/prerequisites.md b/docs/content/docs/installation/prerequisites.md new file mode 100644 index 0000000000..272e534634 --- /dev/null +++ b/docs/content/docs/installation/prerequisites.md @@ -0,0 +1,35 @@ +--- +linkTitle: Prerequisites +title: Prerequisites +weight: 1 +--- + +{{< callout type="info" >}} +To install UCX, user must have Databricks Workspace Administrator privileges. +{{< /callout >}} + +{{< callout type="warning" >}} +Running UCX as a Service Principal is not supported. +{{< /callout >}} + +### Local machine +- Databricks CLI v0.213 or later. See [instructions](./databricks_cli.md). +- Python 3.10 or later. See [Windows](https://www.python.org/downloads/windows/) instructions. + + +### Network access from local machine +- Network access to your Databricks Workspace used for the [installation process](./install_ucx.md). +- Network access to the Internet for [pypi.org](https://pypi.org) and [github.com](https://github.com) from machine running the installation. + +### Account-level settings +- Account level Identity Setup. See instructions for [AWS](https://docs.databricks.com/en/administration-guide/users-groups/best-practices.html), [Azure](https://learn.microsoft.com/en-us/azure/databricks/administration-guide/users-groups/best-practices), and [GCP](https://docs.gcp.databricks.com/administration-guide/users-groups/best-practices.html). +- Unity Catalog Metastore Created (per region). See instructions for [AWS](https://docs.databricks.com/en/data-governance/unity-catalog/create-metastore.html), [Azure](https://learn.microsoft.com/en-us/azure/databricks/data-governance/unity-catalog/create-metastore), and [GCP](https://docs.gcp.databricks.com/data-governance/unity-catalog/create-metastore.html). + + +### Workspace-level settings +- Workspace should have Premium or Enterprise tier. +- A PRO or Serverless SQL Warehouse to render the [report](docs/reference/assessment.md) for the [assessment workflow](docs/reference/workflows/assessment.md). + +{{< callout type="info" >}} +If your Databricks Workspace relies on an external Hive Metastore (such as AWS Glue), make sure to read [this guide](docs/reference/external_hms_glue.md). +{{< /callout >}} diff --git a/docs/content/docs/installation/resources.md b/docs/content/docs/installation/resources.md new file mode 100644 index 0000000000..0d68fd7d5e --- /dev/null +++ b/docs/content/docs/installation/resources.md @@ -0,0 +1,47 @@ +## Installation resources + +The following resources are installed by UCX: + +| Installed UCX resources | Description | +|---------------------------------------------------|--------------------------------------------------------------------------------------------------| +| [Inventory database](docs/reference/table_persistence.md) | A Hive metastore database/schema in which UCX persist inventory required for the upgrade process | +| [Workflows](docs/reference/workflows/) | Workflows to execute UCX | +| [Dashboards](docs/reference/dashboards.md) | Dashboards to visualize UCX outcomes | +| [Installation folder](#installation-folder) | A workspace folder containing UCX files in `/Applications/ucx/`. | + +## Installation folder + +UCX is in installed in the workspace folder `/Applications/ucx/`. This folder contains UCX's code resources, like the +[source code](https://github.com/databrickslabs/ucx/tree/main/src/databricks) from UCX GitHub repository and the [dashboard](docs/reference/dashboards). Generally, these resources are not +*directly* used by UCX users. Resources that can be of importance to users are detailed in the subsections below. + +### Readme notebook + +![readme](/images/readme-notebook.png) + +Every installation creates a `README` notebook with a detailed description of all deployed workflows and their tasks, +providing quick links to the relevant workflows and dashboards. + + + +### Debug notebook + +![debug](/images/debug-notebook.png) + +Every installation creates a `DEBUG` notebook, that initializes UCX as a library for you to execute interactively. + + + +### Debug logs + +![debug](/images/debug-logs.png) + +The [workflow](docs/reference/workflows.md) runs store debug logs in the `logs` folder of the installation folder. The logs are flushed every minute in a separate file. Debug logs for [the command-line interface](docs/installation/databricks_cli.md) are shown by adding the `--debug` flag: + +```bash +databricks --debug labs ucx +``` + +## Installation configuration persistence + +The UCX configuration is stored in the installation folder. diff --git a/docs/content/docs/process/_index.md b/docs/content/docs/process/_index.md new file mode 100644 index 0000000000..da744729c2 --- /dev/null +++ b/docs/content/docs/process/_index.md @@ -0,0 +1,10 @@ +--- +title: "Process" +linkTitle: "Process" +weight: 2 +--- + + +This section describes different processes related to UC migration. + +To see an overview, please click [here](./overview.md). \ No newline at end of file diff --git a/docs/content/docs/process/overview.md b/docs/content/docs/process/overview.md new file mode 100644 index 0000000000..b7345ae2fd --- /dev/null +++ b/docs/content/docs/process/overview.md @@ -0,0 +1,68 @@ +--- +title: Overview +linkTitle: Overview +weight: 1 +--- + +## Steps + +On a high level, the steps in migration process are: + +{{% steps %}} + +### Assessment + +Please follow the [assessment workflow](docs/reference/workflows/assessment.md) + +### Group Migration + +Please follow the [group migration workflow](docs/reference/workflows/group_migration.md) + +### Table Migration + +Please follow the [table migration process](docs/process/table_migration.md) + +### Post Migration Data Reconciliation + +Please follow the [post-migration data reconciliation workflow](docs/reference/workflows/reconciliation.md) + +### Code Migration + +Please follow the [code migration commands](docs/reference/commands.md#code-migration-commands) + +{{% /steps %}} + +## Diagram + +The migration process can be schematic visualized as: + +```mermaid +flowchart TD + subgraph workspace-admin + assessment --> group-migration + group-migration --> table-migration + table-migration --> code-migration + assessment --> create-table-mapping + create-table-mapping --> table-migration + create-table-mapping --> code-migration + validate-external-locations --> table-migration + assessment --> validate-table-locations + validate-table-locations --> table-migration + table-migration --> revert-migrated-tables + revert-migrated-tables --> table-migration + end + subgraph account-admin + create-account-groups --> group-migration + sync-workspace-info --> create-table-mapping + group-migration --> validate-groups-membership + end + subgraph iam-admin + setup-account-scim --> create-account-groups + assessment --> create-uber-principal + create-uber-principal --> table-migration + assessment --> principal-prefix-access + principal-prefix-access --> migrate-credentials + migrate-credentials --> validate-external-locations + setup-account-scim + end +``` \ No newline at end of file diff --git a/docs/content/docs/process/table_migration.md b/docs/content/docs/process/table_migration.md new file mode 100644 index 0000000000..116418bea1 --- /dev/null +++ b/docs/content/docs/process/table_migration.md @@ -0,0 +1,282 @@ +--- +title: Table Migration Process +linkTitle: Table Migration +--- + +{{}} +You are required to complete the [assessment workflow](docs/reference/workflows/assessment.md) before starting the table migration workflow. +{{}} + +## Overview + +This section explains how to migrate Hive metastore data objects to Unity Catalog. + +The table migration process consists of more steps than only a [workflow](docs/reference/workflows/table_migration.md), these steps are: + +{{% steps %}} + +### Step 1 - table mapping + +Create a file that maps Hive metastore data objects to Unity Catalog locations. For details, check [table mapping](#table-mapping) section. + +### Step 2 - data access + +Setup cloud principals to access data in Unity Catalog and during the table migration. For details, check [data access](#data-access) section. + +### Step 3 - Create new Unity Catalog resources + +Create new Unity Catalog resources that do not require touching existing Hive metastore resources. For details, check [new Unity Catalog resources](#new-unity-catalog-resources) section. + +### Step 4 - Migrate Hive metastore data objects + +Migrate Hive metastore data objects to Unity Catalog. For details, check [migrate Hive metastore data objects](#migrate-hive-metastore-data-objects) section. + +{{% /steps %}} + + +## Table mapping + +This section details how to create the table mapping file. This file points existing Hive metastore tables and views to +Unity Catalog locations. When [migrating the tables and views](#migrate-hive-metastore-data-objects), the file is read +to decide where to migrate the tables and views to. + +#### Step 1 : Create the mapping file + +Create the mapping file in the [UCX installation folder](docs/installation/resources.md#installation-folder) by running the +[`create-table-mapping` command](docs/reference/commands.md#create-table-mapping). +By default, the file contains all the Hive metastore tables and views mapped to a single UC catalog, while maintaining the original schema and table names. + +#### Step 2: Update the mapping file + +Edit the mapping file from the previous step to: +- Exclude tables and/or views by removing the lines +- Change UC location by editing the destination catalog and/or schema + +The mapping file is in CSV format and can be edited using any text editor or Excel. If using Excel, save the file in CSV +format. + +Example changes: + +**Before editing** + +| **workspace_name** | **catalog_name** | **src_schema** | **dst_schema** | **src_table** | **dst_table** | +|----------------------|--------------|----------------|----------------|---------------|---------------| +| data_engineering_ws | 123333333 | sales_analysis | sales_analysis | ytd_sales | ytd_sales | + +**After editing** + +| **workspace_name** | **catalog_name** | **src_schema** | **dst_schema** | **src_table** | **dst_table** | +|----------------------|------------------|----------------|----------------|---------------|---------------| +| data_engineering_ws | data_engineering | sales_analysis | sales | ytd_sales | ytd_analysis | + +## Cloud credentials configuration + +{{}} +Throughout this guide, we refer to : +- IAM Roles/Instance profiles in AWS +- Service Principals/Managed Identities in Azure + +as **cloud principals**. +{{}} + +This section creates the cloud principals to access data in Unity Catalog and during the table data during migration. + +To understand the motivation for this step, please read how Databricks accesses cloud locations: +- [AWS - Create External Locations](https://docs.databricks.com/en/connect/unity-catalog/external-locations.html) +- [Azure - Create External Locations](https://learn.microsoft.com/en-us/azure/databricks/connect/unity-catalog/external-locations) + +{{% steps %}} + +### Step 1 : Map cloud principals to cloud storage locations + +Map the cloud principals to cloud storage by running the +[`principal-prefix-access` command](docs/reference/commands.md#principal-prefix-access). + +### Step 2 : Create or modify cloud principals and credentials + +Manually create the cloud principals to access data from Unity Catalog: + +- AWS: + - [`create-missing-principals` command](docs/reference/commands.md#create-missing-principals) creates new AWS roles for Unity Catalog to access data. + - Or, [Manually create storage credentials](https://docs.databricks.com/en/connect/unity-catalog/storage-credentials.html) +- Azure: + - [Manually create storage Credentials](https://learn.microsoft.com/en-us/azure/databricks/sql/language-manual/sql-ref-storage-credentials) + +Then, run the [`migrate-credentials` command](docs/reference/commands.md#migrate-credentials) to migrate the cloud principal +credentials to Unity Catalog. + +### Step 3: Create the "uber" Principal + +Create the "uber" principal by running the [`create-uber-principal` command](docs/reference/commands.md#create-uber-principal). +The table migration requires a cloud principal with access to table data stored in cloud storage. These tables are known +as external tables. UCX name this cloud principal the "uber" principal as it has access to **all** in-scope cloud +storage. This principal is only required for the table migration process and not for ongoing UC usage. Once the upgrade +is completed, this principal should be deleted. + +{{% /steps %}} + +## New Unity Catalog resources + +This section details the **new** Unity Catalog resources required for migration the data objects. These resources can be +created without touching the existing Hive metastore objecs. + +{{% steps %}} + +### Step 0: Attach a metastore + +If skipping the [group migration](docs/reference/workflows/group_migration.md), then a metastore should be attached to the workspace by following [these instructions](https://docs.databricks.com/en/data-governance/unity-catalog/create-metastore.html) or running the [`assign-metastore` command](docs/reference/commands.md#assign-metastore). + +### Step 1: Create external Locations + +Create UC external locations by running the [`migration-locations` command](docs/reference/commands.md#migrate-locations). The command +creates a location for each location found during the [assessment](docs/reference/workflows/assessment.md). It uses the credentials created in the previous steps. + +### Step 2: Create Catalogs and Schemas + +Create Unity Catalog [catalogs](https://learn.microsoft.com/en-us/azure/databricks/catalogs/) and +[schemas](https://learn.microsoft.com/en-us/azure/databricks/schemas/) to organize the destination tables and views in +by running the +[`create-catalogs-schemas` command](docs/reference/commands.md#create-catalogs-schemas). The command creates the UC catalogs and +schemas based on the [table mapping file](#table-mapping). Additionally, it migrates Hive metastore database +permissions if present. + +This step requires considering how to [physically separate data in storage](https://docs.databricks.com/en/data-governance/unity-catalog/best-practices.html#data-is-physically-separated-in-storage) +within UC. As [Databricks recommends storing managed data at the catalog level](https://docs.databricks.com/en/data-governance/unity-catalog/best-practices.html#configure-a-unity-catalog-metastore), +we advise to prepare the external locations for the to-be created catalogs before running the `create-catalogs-schemas` +command. Either, reuse [previously created external locations](#step-23-create-external-locations) or create additional +external locations outside of UCX if data separation restrictions requires that. Note that external locations can be +reused when using subpaths, for example, a folder in a cloud storage +(`abfss://container@storage.dfs.core.windows.net/folder`) can reuse the external location of the cloud storage +(`abfss://container@storage.dfs.core.windows.net/`). (The previous example also holds for other clouds.) + +{{% /steps %}} + +## Migrate Hive metastore data objects + +In this step, tables and views are migrated using the following workflows: + +```mermaid +flowchart LR + subgraph CLI + migrate-tables[migrate-tables] + end + + subgraph mt_workflow[workflow: migrate-tables] + dbfs_root_delta_mt_task[migrate_dbfs_root_delta_tables] + migrate_dbfs_root_non_delta_tables[migrate_dbfs_root_non_delta_tables] + external_tables_sync_mt_task[migrate_external_tables_sync] + view_mt_task[migrate_views] + refresh_migration_status[refresh_migration_status] + external_tables_sync_mt_task --> view_mt_task + dbfs_root_delta_mt_task --> view_mt_task + migrate_dbfs_root_non_delta_tables --> view_mt_task + view_mt_task --> refresh_migration_status + end + + subgraph mt_serde_inplace_wf[workflow: migrate-external-hiveserde-tables-in-place-experimental] + serde_inplace_mt_task[migrate_hive_serde_in_place] --> view_mt_task_inplace[migrate_views] + view_mt_task_inplace --> refresh_migration_status_hiveserde[refresh_migration_status] + end + + subgraph mt_ctas_wf[workflow: migrate-external-tables-ctas] + ctas_mt_task[migrate_other_external_ctas] + migrate_hive_serde_ctas[migrate_hive_serde_ctas] + view_mt_ctas_task[migrate_views] + refresh_migration_status_ctas[refresh_migration_status] + ctas_mt_task --> view_mt_ctas_task + migrate_hive_serde_ctas --> view_mt_ctas_task + view_mt_ctas_task --> refresh_migration_status_ctas + end + + migrate-tables -- 1st --> mt_workflow + migrate-tables -- 2nd (optional) table migrated here will be excluded in ctas workflow --> mt_serde_inplace_wf + migrate-tables -- 3rd --> mt_ctas_wf +``` + +The [table migration workflows](docs/reference/workflows/table_migration.md) can be triggered using the [`migrate-tables` command](docs/reference/commands.md#migrate-tables) or by +starting the workflows from the workspace UI manually. + +{{< callout type="info" >}} +The table migration workflows are designed to minimize data copying and to maintain metadata while allowing users to choose preferred strategies where possible. +{{< /callout >}} + +Chose the strategies for migrating tables using the table below: + +| Object Type | Description | Upgrade Method | +|-----------------------|--------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `DBFS_ROOT_DELTA` | Delta tables persisted on the Databricks file system (dbfs). | Create a copy of the table with the [`DEEP CLONE` command](https://docs.databricks.com/en/sql/language-manual/delta-clone.html). | +| `DBFS_ROOT_NON_DELTA` | Non-delta tables persisted on the Databricks file system (dbfs). | Create a copy of the table with a [`CREATE TABLE AS SELECT * FROM` command](https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-ddl-create-table-using.html). The UC table will be a Delta table. | | +| `MANAGED` | Managed Hive metastore tables. | Depending on the managed table migration strategy chosen during installation:
1. `CLONE`: Create a copy of the table with a [`CREATE TABLE LOCATION '' AS SELECT * FROM` command](https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-ddl-create-table-using.html).
2. `SYNC_AS_EXTERNAL`, synchronize the table metadata to UC with the [`SYNC` command](https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-aux-sync.html). **Warning**: If the managed Hive metastore table is dropped, the drop deletes the underlying data affecting the synchronised UC table as well.
3. `CONVERT_TO_EXTERNAL`: First, in-place convert the managed Hive metastore to a non-managed table. Then, synchronize the table metadata to UC with the [`SYNC` command](https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-aux-sync.html). **Warning**: This strategy has the advantage over `SYNC_AS_EXTERNAL` that dropping the Hive metastore table does not delete the underlying data, however, impact should be carefully validated in existing workloads as the strategy converts the managed Hive metastore to an external table **in-place**. | +| `EXTERNAL_SYNC` | External tables that the `SYNC` command supports: Delta, Parquet, CSV, JSON, ORC, text or Avro tables. | Synchronize the table metadata to UC with the [`SYNC` command](https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-aux-sync.html). | +| `EXTERNAL_NO_SYNC` | External tables that the `SYNC` command does not support. | Create a copy of the table with a [`CREATE TABLE AS SELECT * FROM` command](https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-ddl-create-table-using.html). The UC table will be a Delta table. | +| `EXTERNAL_HIVESERDE` | External Hive SerDe tables that the `SYNC` command does not support. | Depending on the migration workflow chosen:
1. `migrate-external-tables-ctas` (officially supported) : Create a copy of the table with a [`CREATE TABLE AS SELECT * FROM` command](https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-ddl-create-table-using.html) The UC table will be a Delta table.
2. `migrate-external-hiveserde-tables-in-place-experimental` (experimental) : Recreate the [Hive SerDe](https://cwiki.apache.org/confluence/display/Hive/SerDe) tables using the serialization and deserialization protocols. **Warning:** Although this strategy is supported, there is a risk that the old files created by Hive SerDe may not be processed correctly by Spark datasource in corner cases. | +| `VIEW` | Views | Recreate [views](https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-ddl-create-view.html) using their definitions after repointing their dependencies to UC objects. The migration process supports views depending on other views. | + +The workflows may be executed multiple times. Each step is designed as a standalone operation that migrates all +in-scope tables. + +After migration, each object is marked with an `upgraded_to` property containing the UC identifier to +which the object is migrated. This property signals that the object is out-of-scope for other migration operations and +that the view dependency exists within UC. + +All created UC objects are marked with an `upgraded_from` property containing +the Hive metastore identifier from which the object was migrated. + +Finally, the table migration workflows also migrate Hive metastore permissions to Unity Catalog. + +## Considerations + +General considerations are as follows: +- You may want to run the workflows multiple times to migrate tables in phases. +- If your Delta tables in DBFS root have a large number of files, consider: + - Setting higher `Min` and `Max workers for auto-scale` when being asked during the UCX installation. More cores in the cluster means more concurrency for calling cloud storage API to copy files when deep cloning the Delta tables. + - Setting higher `Parallelism for migrating DBFS root Delta tables with deep clone` (default 200) when being asked during the UCX installation. This controls the number of Spark tasks/partitions to be created for deep clone. +- Consider creating an instance pool, and setting its id when prompted during the UCX installation. This instance pool will be specified in the cluster policy used by all UCX workflows job clusters. +- You may also manually edit the job cluster configration per job or per task after the workflows are deployed. + +## Additional references + +- [Databricks file system (dbfs)](https://docs.databricks.com/en/dbfs/index.html) +- [External tables](https://docs.databricks.com/en/tables/external.html) + +## Odds and Ends + +The following sections detail how to repair/amend the UC metastore after the upgrade process. + +#### Skip migrating schemas, tables or views + +```bash +databricks labs ucx skip --schema X [--table Y] [--view Zj] +``` + +The [`skip` command](docs/reference/commands.md#skip) marks a schema, table or view as to-be skipped during the migration processes. + +#### Move data objects + +```bash +databricks labs ucx move --from-catalog A --from-schema B --from-table C --to-catalog D --to-schema E +``` + +The [`move` command](docs/reference/commands.md#move) moves the object from the source to the target location. The `upgraded_from` +property are updated to reflect the new location on the source object. Use this command if the data object is migrated +to the wrong destination. + +#### Alias data objects + +```bash +databricks labs ucx alias --from-catalog A --from-schema B --from-table C --to-catalog D --to-schema E +``` + +This [`alias` command](docs/reference/commands.md#alias) creates an alias for the object in the target location by creating a view reading +from the table that needs aliasing. It will create a mirror view to view that is marked as alias. +Use this command if Hive metastore tables point to the same location as UC does not support UC does not support tables +with overlapping data locations. + +#### Revert migrated data objects + +```bash +databricks labs ucx revert-migrated-tables --schema X --table Y [--delete-managed] +``` + +The [`revert-migrated-tables` command](docs/reference/commands.md#revert-migrated-tables) drops the Unity Catalog table or view and reset the `upgraded_to` property on the source object. Use this command to allow for migrating a table or view again. + diff --git a/docs/content/docs/reference/_index.md b/docs/content/docs/reference/_index.md new file mode 100644 index 0000000000..080245cff5 --- /dev/null +++ b/docs/content/docs/reference/_index.md @@ -0,0 +1,8 @@ +--- +title: "Reference" +linkTitle: "Reference" +weight: 3 +--- +The reference documentation is a collection of documents that describe specific details about UCX. + +Please navigate to the sub-sections to find relevant content. \ No newline at end of file diff --git a/docs/assessment.md b/docs/content/docs/reference/assessment.md similarity index 68% rename from docs/assessment.md rename to docs/content/docs/reference/assessment.md index 741be2c9a4..b4d913c8c4 100644 --- a/docs/assessment.md +++ b/docs/content/docs/reference/assessment.md @@ -1,182 +1,47 @@ -Migration Assessment Report -=== - - -* [Migration Assessment Report](#migration-assessment-report) -* [Assessment Report Summary](#assessment-report-summary) -* [Assessment Widgets](#assessment-widgets) - * [Readiness](#readiness) - * [Total Databases](#total-databases) - * [Metastore Crawl Failures](#metastore-crawl-failures) - * [Total Tables](#total-tables) - * [Storage Locations](#storage-locations) -* [Assessment Widgets](#assessment-widgets-1) - * [Readiness](#readiness-1) - * [Assessment Summary](#assessment-summary) - * [Table counts by storage](#table-counts-by-storage) - * [Table counts by schema and format](#table-counts-by-schema-and-format) - * [Database Summary](#database-summary) - * [External Locations](#external-locations) - * [Mount Points](#mount-points) - * [Table Types](#table-types) - * [Incompatible Clusters](#incompatible-clusters) - * [Incompatible Jobs](#incompatible-jobs) - * [Incompatible Delta Live Tables](#incompatible-delta-live-tables) - * [Incompatible Global Init Scripts](#incompatible-global-init-scripts) -* [Assessment Finding Index](#assessment-finding-index) - * [AF101 - not supported DBR: ##.#.x-scala2.12](#af101---not-supported-dbr-x-scala212) - * [AF102 - not supported DBR: ##.#.x-cpu-ml-scala2.12](#af102---not-supported-dbr-x-cpu-ml-scala212) - * [AF103 - not supported DBR: ##.#.x-gpu-ml-scala2.12](#af103---not-supported-dbr-x-gpu-ml-scala212) - * [AF111 - Uses azure service principal credentials config in cluster.](#af111---uses-azure-service-principal-credentials-config-in-cluster) - * [AF112 - Uses azure service principal credentials config in Job cluster.](#af112---uses-azure-service-principal-credentials-config-in-job-cluster) - * [AF113 - Uses azure service principal credentials config in pipeline.](#af113---uses-azure-service-principal-credentials-config-in-pipeline) - * [AF114 - unsupported config](#af114---unsupported-config) - * [AF115 - unsupported config: spark.databricks.passthrough.enabled](#af115---unsupported-config-sparkdatabrickspassthroughenabled) - * [AF116 - No isolation shared clusters not supported in UC](#af116---no-isolation-shared-clusters-not-supported-in-uc) - * [AF117 - cluster type not supported](#af117---cluster-type-not-supported) - * [AF201 - Inplace Sync](#af201---inplace-sync) - * [AF202 - Asset Replication Required](#af202---asset-replication-required) - * [AF203 - Data in DBFS Root](#af203---data-in-dbfs-root) - * [AF204 - Data is in DBFS Mount](#af204---data-is-in-dbfs-mount) - * [AF210 - Non-DELTA format: CSV](#af210---non-delta-format-csv) - * [AF211 - Non-DELTA format: DELTA](#af211---non-delta-format-delta) - * [AF212 - Non-DELTA format](#af212---non-delta-format) - * [AF221 - Unsupported Storage Type](#af221---unsupported-storage-type) - * [AF300 - AF399](#af300---af399) - * [AF300.6 - 3 level namespace](#af3006---3-level-namespace) - * [AF300.1 - r language support](#af3001---r-language-support) - * [AF300.2 - scala language support](#af3002---scala-language-support) - * [AF300.3 - Minimum DBR version](#af3003---minimum-dbr-version) - * [AF300.4 - ML Runtime cpu](#af3004---ml-runtime-cpu) - * [AF300.5 - ML Runtime gpu](#af3005---ml-runtime-gpu) - * [AF301.1 - spark.catalog.x](#af3011---sparkcatalogx) - * [AF301.2 - spark.catalog.x (spark._jsparkSession.catalog)](#af3012---sparkcatalogx-spark_jsparksessioncatalog) - * [AF302.x - Arbitrary Java](#af302x---arbitrary-java) - * [AF302.1 - Arbitrary Java (`spark._jspark`)](#af3021---arbitrary-java-spark_jspark) - * [AF302.2 - Arbitrary Java (`spark._jvm`)](#af3022---arbitrary-java-spark_jvm) - * [AF302.3 - Arbitrary Java (`._jdf`)](#af3023---arbitrary-java-_jdf) - * [AF302.4 - Arbitrary Java (`._jcol`)](#af3024---arbitrary-java-_jcol) - * [AF302.5 - Arbitrary Java (`._jvm`)](#af3025---arbitrary-java-_jvm) - * [AF302.6 - Arbitrary Java (`._jvm.org.apache.log4j`)](#af3026---arbitrary-java-_jvmorgapachelog4j) - * [AF303.1 - Java UDF (`spark.udf.registerJavaFunction`)](#af3031---java-udf-sparkudfregisterjavafunction) - * [AF304.1 - JDBC datasource (`spark.read.format("jdbc")`)](#af3041---jdbc-datasource-sparkreadformatjdbc) - * [AF305.1 - boto3](#af3051---boto3) - * [AF305.2 - s3fs](#af3052---s3fs) - * [AF306.1 - dbutils...getContext (`.toJson()`)](#af3061---dbutilsgetcontext-tojson) - * [AF306.2 - dbutils...getContext](#af3062---dbutilsgetcontext) - * [AF310.1 - credential passthrough (`dbutils.credentials.`)](#af3101---credential-passthrough-dbutilscredentials) - * [AF311.x - dbutils (`dbutils`)](#af311x---dbutils-dbutils) - * [AF311.1 - dbutils.fs (`dbutils.fs.`)](#af3111---dbutilsfs-dbutilsfs) - * [AF311.2 - dbutils mount(s) (`dbutils.fs.mount`)](#af3112---dbutils-mounts-dbutilsfsmount) - * [AF311.3 - dbutils mount(s) (`dbutils.fs.refreshMounts`)](#af3113---dbutils-mounts-dbutilsfsrefreshmounts) - * [AF311.4 - dbutils mount(s) (`dbutils.fs.unmount`)](#af3114---dbutils-mounts-dbutilsfsunmount) - * [AF311.5 - mount points (`dbfs:/mnt`)](#af3115---mount-points-dbfsmnt) - * [AF311.6 - dbfs usage (`dbfs:/`)](#af3116---dbfs-usage-dbfs) - * [AF311.7 - dbfs usage (`/dbfs/`)](#af3117---dbfs-usage-dbfs) - * [AF313.x - SparkContext](#af313x---sparkcontext) - * [AF313.1 - SparkContext (`spark.sparkContext`)](#af3131---sparkcontext-sparksparkcontext) - * [AF313.2 - SparkContext (`from pyspark.sql import SQLContext`)](#af3132---sparkcontext-from-pysparksql-import-sqlcontext) - * [AF313.3 - SparkContext (`.binaryFiles`)](#af3133---sparkcontext-binaryfiles) - * [AF313.4 - SparkContext (`.binaryRecords`)](#af3134---sparkcontext-binaryrecords) - * [AF313.5 - SparkContext (`.emptyRDD`)](#af3135---sparkcontext-emptyrdd) - * [AF313.6 - SparkContext (`.getConf`)](#af3136---sparkcontext-getconf) - * [AF313.7 - SparkContext ( `.hadoopFile` )](#af3137---sparkcontext--hadoopfile-) - * [AF313.8 - SparkContext ( `.hadoopRDD` )](#af3138---sparkcontext--hadooprdd-) - * [AF313.9 - SparkContext ( `.init_batched_serializer` )](#af3139---sparkcontext--init_batched_serializer-) - * [AF313.10 - SparkContext ( `.newAPIHadoopFile` )](#af31310---sparkcontext--newapihadoopfile-) - * [AF313.11 - SparkContext ( `.newAPIHadoopRDD` )](#af31311---sparkcontext--newapihadooprdd-) - * [AF313.12 - SparkContext ( `.parallelize` )](#af31312---sparkcontext--parallelize-) - * [AF313.13 - SparkContext ( `.pickleFile` )](#af31313---sparkcontext--picklefile-) - * [AF313.14 - SparkContext ( `.range` )](#af31314---sparkcontext--range-) - * [AF313.15 - SparkContext ( `.rdd` )](#af31315---sparkcontext--rdd-) - * [AF313.16 - SparkContext ( `.runJob` )](#af31316---sparkcontext--runjob-) - * [AF313.17 - SparkContext ( `.sequenceFile` )](#af31317---sparkcontext--sequencefile-) - * [AF313.18 - SparkContext ( `.setJobGroup` )](#af31318---sparkcontext--setjobgroup-) - * [AF313.19 - SparkContext ( `.setLocalProperty` )](#af31319---sparkcontext--setlocalproperty-) - * [AF313.20 - SparkContext ( `.setSystemProperty` )](#af31320---sparkcontext--setsystemproperty-) - * [AF313.21 - SparkContext ( `.stop` )](#af31321---sparkcontext--stop-) - * [AF313.22 - SparkContext ( `.textFile` )](#af31322---sparkcontext--textfile-) - * [AF313.23 - SparkContext ( `.uiWebUrl`)](#af31323---sparkcontext--uiweburl) - * [AF313.24 - SparkContext (`.union`)](#af31324---sparkcontext-union) - * [AF313.25 - SparkContext (`.wholeTextFiles`)](#af31325---sparkcontext-wholetextfiles) - * [AF314.x - Distributed ML](#af314x---distributed-ml) - * [AF314.1 - Distributed ML (`sparknlp`)](#af3141---distributed-ml-sparknlp) - * [AF314.2 - Distributed ML (`xgboost.spark`)](#af3142---distributed-ml-xgboostspark) - * [AF314.3 - Distributed ML (`catboost_spark`)](#af3143---distributed-ml-catboost_spark) - * [AF314.4 - Distributed ML (`ai.catboost:catboost-spark`)](#af3144---distributed-ml-aicatboostcatboost-spark) - * [AF314.5 - Distributed ML (`hyperopt`)](#af3145---distributed-ml-hyperopt) - * [AF314.6 - Distributed ML (`SparkTrials`)](#af3146---distributed-ml-sparktrials) - * [AF314.7 - Distributed ML (`horovod.spark`)](#af3147---distributed-ml-horovodspark) - * [AF314.8 - Distributed ML (`ray.util.spark`)](#af3148---distributed-ml-rayutilspark) - * [AF314.9 - Distributed ML (`databricks.automl`)](#af3149---distributed-ml-databricksautoml) - * [AF308.1 - Graphframes (`from graphframes`)](#af3081---graphframes-from-graphframes) - * [AF309.1 - Spark ML (`pyspark.ml.`)](#af3091---spark-ml-pysparkml) - * [AF315.1 - UDAF scala issue (`UserDefinedAggregateFunction`)](#af3151---udaf-scala-issue-userdefinedaggregatefunction) - * [AF315.2 - applyInPandas (`applyInPandas`)](#af3152---applyinpandas-applyinpandas) - * [AF315.3 - mapInPandas (`mapInPandas`)](#af3153---mapinpandas-mapinpandas) - * [AF330.x - Streaming](#af330x---streaming) - * [AF330.1 - Streaming (`.trigger(continuous`)](#af3301---streaming-triggercontinuous) - * [AF330.2 - Streaming (`kafka.sasl.client.callback.handler.class`)](#af3302---streaming-kafkasaslclientcallbackhandlerclass) - * [AF330.3 - Streaming (`kafka.sasl.login.callback.handler.class`)](#af3303---streaming-kafkasasllogincallbackhandlerclass) - * [AF330.4 - Streaming (`kafka.sasl.login.class`)](#af3304---streaming-kafkasaslloginclass) - * [AF330.5 - Streaming (`kafka.partition.assignment.strategy`)](#af3305---streaming-kafkapartitionassignmentstrategy) - * [AF330.6 - Streaming (`kafka.ssl.truststore.location`)](#af3306---streaming-kafkassltruststorelocation) - * [AF330.7 - Streaming (`kafka.ssl.keystore.location`)](#af3307---streaming-kafkasslkeystorelocation) - * [AF330.8 - Streaming (`cleanSource`)](#af3308---streaming-cleansource) - * [AF330.9 - Streaming (`sourceArchiveDir`)](#af3309---streaming-sourcearchivedir) - * [AF330.10 - Streaming (`applyInPandasWithState`)](#af33010---streaming-applyinpandaswithstate) - * [AF330.11 - Streaming (`.format("socket")`)](#af33011---streaming-formatsocket) - * [AF330.12 - Streaming (`StreamingQueryListener`)](#af33012---streaming-streamingquerylistener) - * [AF330.13 - Streaming (`applyInPandasWithState`)](#af33013---streaming-applyinpandaswithstate) -* [Common Terms](#common-terms) - * [UC](#uc) - * [DELTA](#delta) - * [CTAS](#ctas) - * [DEEP CLONE](#deep-clone) - * [EXTERNAL LOCATION](#external-location) - * [STORAGE CREDENTIAL](#storage-credential) - * [Assigned Clusters or Single User Clusters](#assigned-clusters-or-single-user-clusters) - * [Shared Clusters](#shared-clusters) - +--- +title: "Migration Assessment Report" +linkTitle: "Migration Assessment Report" +--- + This document describes the Assessment Report generated from the UCX tools. The main assessment report includes dashlets, widgets and details of the assessment findings and common recommendations made based on the Assessment Finding (AF) Index entry. -![report](assessment-report.png) +![report](/images/assessment-report.png) # Assessment Report Summary The Assessment Report (Main) is the output of the Databricks Labs UCX assessment workflow. This report queries the $inventory database (e.g. `ucx`) and summarizes the findings of the assessment. The link to the Assessment Report (Main) can be found in your home folder, under `.ucx` in the README.py file. The user may also directly navigate to the Assessment report by clicking on `Dashboards` icon on the left to find the Dashboard. -[[back to top](#migration-assessment-report)] + # Assessment Widgets image -[[back to top](#migration-assessment-report)] + ## Readiness This is an overall summary of readiness detailed in the Readiness dashlet. This value is based on the ratio of findings divided by the total number of assets scanned. -[[back to top](#migration-assessment-report)] + ## Total Databases The total number of `hive_metastore` databases found during the assessment. -[[back to top](#migration-assessment-report)] + ## Metastore Crawl Failures Total number of failures encountered by the crawler while extracting metadata from the Hive Metastore and REST APIs. -[[back to top](#migration-assessment-report)] + ## Total Tables Total number of hive metastore tables discovered -[[back to top](#migration-assessment-report)] + ## Storage Locations Total number of identified storage locations based on scanning Hive Metastore tables and schemas -[[back to top](#migration-assessment-report)] + # Assessment Widgets Assessment widgets query tables in the $inventory database and summarize or detail out findings. @@ -185,25 +50,25 @@ The second row of the report starts with "Job Count", "Readiness", "Assessment S image -[[back to top](#migration-assessment-report)] + ## Readiness This is a rough summary of the workspace readiness to run Unity Catalog governed workloads. Each line item is the percent of compatible items divided by the total items in the class. -[[back to top](#migration-assessment-report)] + ## Assessment Summary This is a summary count, per finding type of all of the findings identified during the assessment workflow. The assessment summary will help identify areas that need focus (e.g. Tables on DBFS or Clusters that need DBR upgrades) -[[back to top](#migration-assessment-report)] + ## Table counts by storage This is a summary count of Hive Metastore tables, per storage type (DBFS Root, DBFS Mount, Cloud Storage (referred as External)). This also gives a summary count of tables using storage types which are unsupported (such as WASB or ADL in Azure) in Unity Catalog. Count of tables created using Databricks Demo Datasets are also identified here -[[back to top](#migration-assessment-report)] + ## Table counts by schema and format @@ -212,7 +77,7 @@ This is a summary count by Hive Metastore (HMS) table formats (Delta and Non Del The third row continues with "Database Summary" image -[[back to top](#migration-assessment-report)] + ## Database Summary @@ -222,13 +87,13 @@ This is a Hive Metastore based Database by Database assessment summary along wit And the fourth row contains "External Locations" and "Mount Points" image -[[back to top](#migration-assessment-report)] + ## External Locations Tables were scanned for `LOCATION` attributes and that list was distilled down to External Locations. In Unity Catalog, create a STORAGE CREDENTIAL that can access the External Locations, then define Unity Catalog `EXTERNAL LOCATION`s for these items. -[[back to top](#migration-assessment-report)] + ## Mount Points @@ -240,7 +105,7 @@ Unfortunately, as of January 2024, cross cloud external locations are not suppor The next row contains the "Table Types" widget image -[[back to top](#migration-assessment-report)] + ## Table Types @@ -254,13 +119,13 @@ This widget is a detailed list of each table, it's format, storage type, locatio The following row includes "Incompatible Clusters and "Incompatible Jobs" image -[[back to top](#migration-assessment-report)] + ## Incompatible Clusters This widget is a list of findings (reasons) and clusters that may need upgrading. See Assessment Finding Index (below) for specific recommendations. -[[back to top](#migration-assessment-report)] + ## Incompatible Jobs @@ -269,25 +134,25 @@ This is a list of findings (reasons) and jobs that may need upgrading. See Asses The final row includes "Incompatible Delta Live Tables" and "Incompatible Global Init Scripts" image -[[back to top](#migration-assessment-report)] + ## Incompatible Object Privileges These are permissions on objects that are not supported by Unit Catalog. -[[back to top](#migration-assessment-report)] + ## Incompatible Delta Live Tables These are Delta Live Table jobs that may be incompatible with Unity Catalog. -[[back to top](#migration-assessment-report)] + ## Incompatible Global Init Scripts These are Global Init Scripts that are incompatible with Unity Catalog compute. As a reminder, global init scripts need to be on secure storage (Volumes or a Cloud Storage account and not DBFS) -[[back to top](#migration-assessment-report)] + # Assessment Finding Index @@ -297,7 +162,7 @@ The assessment finding index is grouped by: - The 200 series findings are centered around data related observations. - The 300 series findings relate to [Compute Access mode limitations for Unity Catalog](https://docs.databricks.com/en/compute/access-mode-limitations.html#spark-api-limitations-for-unity-catalog-shared-access-mode) -[[back to top](#migration-assessment-report)] + ### AF101 - not supported DBR: ##.#.x-scala2.12 @@ -305,19 +170,19 @@ Short description: The compute runtime does not meet the requirements to use Uni Explanation: Unity Catalog capabilities are fully enabled on Databricks Runtime 13.3 LTS. This is the current recommended runtime for production interactive clusters and jobs. This finding is noting the cluster or job compute configuration does not meet this threshold. recommendation: Upgrade the DBR version to 13.3 LTS or later. -[[back to top](#migration-assessment-report)] + ### AF102 - not supported DBR: ##.#.x-cpu-ml-scala2.12 Currently, MLR (Machine Learning Runtime) and GPU *SHARED* clusters are not supported with Unity Catalog. Use *Assigned* or *Job* clusters instead. -[[back to top](#migration-assessment-report)] + ### AF103 - not supported DBR: ##.#.x-gpu-ml-scala2.12 Currently, MLR (Machine Learning Runtime) and GPU *SHARED* clusters are not supported with Unity Catalog. Use *Assigned* or *Job* clusters instead. -[[back to top](#migration-assessment-report)] + ### AF111 - Uses azure service principal credentials config in cluster. @@ -325,7 +190,7 @@ Azure service principles are replaced by Storage Credentials to access cloud sto Create a storage CREDENTIAL, then an EXTERNAL LOCATION and possibly external tables to provide data access. If the service principal is used to access additional azure cloud services, convert the cluster to a `Assigned` cluster type which *may* work. -[[back to top](#migration-assessment-report)] + ### AF112 - Uses azure service principal credentials config in Job cluster. @@ -333,14 +198,14 @@ Azure service principles are replaced by Storage Credentials to access cloud sto Create a storage CREDENTIAL, then an EXTERNAL LOCATION and possibly external tables to provide data access. If the service principal is used to access additional azure cloud services, convert the job cluster to a `Assigned` cluster type which *may* work. -[[back to top](#migration-assessment-report)] + ### AF113 - Uses azure service principal credentials config in pipeline. Azure service principles are replaced by Storage Credentials to access cloud storage accounts. Create a storage CREDENTIAL, then an EXTERNAL LOCATION and possibly external tables to provide data access. -[[back to top](#migration-assessment-report)] + ### AF114 - Uses external Hive metastore config: spark.hadoop.javax.jdo.option.ConnectionURL @@ -350,34 +215,34 @@ using UCX. As a transition strategy, "No Isolation Shared" clusters or "Assigned - `spark.hadoop.javax.jdo.option.ConnectionURL` an external Hive Metastore is in use. Recommend migrating the these tables and schemas to Unity Catalog external tables where they can be shared across workspaces. - `spark.databricks.hive.metastore.glueCatalog.enabled` Glue is used as external Hive Metastore. Recommend migrating the these tables and schemas to Unity Catalog external tables where they can be shared across workspaces. -[[back to top](#migration-assessment-report)] + ### AF115 - Uses passthrough config: spark.databricks.passthrough.enabled. Passthrough security model is not supported by Unity Catalog. Passthrough mode relied upon file based authorization which is incompatible with Fine Grained Access Controls supported by Unity Catalog. Recommend mapping your Passthrough security model to an External Location/Volume/Table/View based security model compatible with Unity Catalog. -[[back to top](#migration-assessment-report)] + ### AF116 - No isolation shared clusters not supported in UC Unity Catalog data cannot be accessed from No Isolation clusters, they should not be used. -[[back to top](#migration-assessment-report)] + ### AF117 - cluster type not supported Only Assigned and Shared access mode are supported in UC. You must change your cluster configuration to match UC compliant access modes. -[[back to top](#migration-assessment-report)] + ### AF201 - Inplace Sync Short description: We found that the table or database can be SYNC'd without moving data because the data is stored directly on cloud storage specified via a mount or a cloud storage URL (not DBFS). How: Run the SYNC command on the table or schema. If the tables (or source database) is 'managed' first set this spark setting in your session or in the interactive cluster configuration: `spark.databricks.sync.command.enableManagedTable=true` -[[back to top](#migration-assessment-report)] + ### AF202 - Asset Replication Required @@ -388,33 +253,33 @@ CREATE TABLE [IF NOT EXISTS] table_name [SHALLOW | DEEP] CLONE source_table_name [TBLPROPERTIES clause] [LOCATION path] ``` -[[back to top](#migration-assessment-report)] + ### AF203 - Data in DBFS Root A table or schema refers to a location in DBFS and not a cloud storage location. The data must be moved from DBFS to a cloud storage location or to a Unity Catalog managed storage. -[[back to top](#migration-assessment-report)] + ### AF204 - Data is in DBFS Mount A table or schema refers to a location in DBFS mount and not a direct cloud storage location. Mounts are not suppored in Unity Catalog so the mount source location must be de-referenced and the table/schema objects mapped to a UC external location. -[[back to top](#migration-assessment-report)] + ### AF210 - Non-DELTA format: CSV Unity Catalog does not support managed CSV tables. Recommend converting the table to DELTA format or migrating the table to an External table. -[[back to top](#migration-assessment-report)] + ### AF211 - Non-DELTA format: DELTA This was a known [issue](https://github.com/databrickslabs/ucx/issues/788) of the UCX assessment job. This bug should be fixed with release `0.10.0` -[[back to top](#migration-assessment-report)] + ### AF212 - Non-DELTA format @@ -434,7 +299,7 @@ Workaround: Granting ANY FILE permissions will allow users to access untrusted databases. Note that ANY FILE will still enforce ACLs on any tables or external (storage) locations governed by Unity Catalog. Upgrade the DBR runtime to 13.3 LTS or higher to avoid cluster level firewall restrictions. -[[back to top](#migration-assessment-report)] + ### AF221 - Unsupported Storage Type @@ -446,13 +311,13 @@ CREATE TABLE [IF NOT EXISTS] table_name [SHALLOW | DEEP] CLONE source_table_name [TBLPROPERTIES clause] [LOCATION path] ``` -[[back to top](#migration-assessment-report)] + ### AF222 - Explicitly DENYing privileges is not supported in UC Unity Catalog does not support `DENY` permissions on securable objects. These permissions will be updated during group migration, but won't be transferred during catalog migration. -[[back to top](#migration-assessment-report)] + ## AF300 - AF399 The 300 series findings relate to [Compute Access mode limitations for Unity Catalog](https://docs.databricks.com/en/compute/access-mode-limitations.html#spark-api-limitations-for-unity-catalog-shared-access-mode) @@ -504,21 +369,21 @@ PWD= Catalog=my_catalog ``` -[[back to top](#migration-assessment-report)] + ### AF300.1 - r language support When using `%r` command cells, the user will receive `Your administrator has only allowed sql and python and scala commands on this cluster. This execution contained at least one disallowed language.` message. Recommend using Assigned (single user clusters). -[[back to top](#migration-assessment-report)] + ### AF300.2 - scala language support Scala is supported on Databricks Runtime 13.3 and above. Recommend upgrading your shared cluster DBR to 13.3 LTS or greater or using Assigned data security mode (single user clusters). -[[back to top](#migration-assessment-report)] + ### AF300.3 - Minimum DBR version The minimum DBR version to access Unity Catalog was not met. The recommendation is to upgrade to the latest Long Term Supported (LTS) version of the Databricks Runtime. @@ -533,13 +398,13 @@ The Databricks ML Runtime is not supported on Shared Compute mode clusters. Reco The `spark.catalog.` pattern was found. Commonly used functions in `spark.catalog`, such as `tableExists`, `listTables`, `setCurrentCatalog` are not allowed on shared clusters due to security reasons. `spark.sql(")` may be a better alternative. DBR 14.1 and above have made these commands available. Upgrade your DBR version. -[[back to top](#migration-assessment-report)] + ### AF301.2 - spark.catalog.x (spark._jsparkSession.catalog) The `spark._jsparkSession.catalog` pattern was found. Commonly used functions in `spark.catalog`, such as `tableExists`, `listTables`, `setCurrentCatalog` are not allowed on shared clusters due to security reasons. `spark.sql(")` may be a better alternative. The corresponding `spark.catalog.x` methods may work on DBR 14.1 and above. -[[back to top](#migration-assessment-report)] + ## AF302.x - Arbitrary Java With Spark Connect on Shared clusters it is no longer possible to directly access the host JVM from the Python process. This means it is no longer possible to interact with Java classes or instantiate arbitrary Java classes directly from Python similar to the code below. @@ -550,43 +415,43 @@ Recommend finding the equivalent PySpark or Scala api. The `spark._jspark` is used to execute arbitrary Java code. -[[back to top](#migration-assessment-report)] + ### AF302.2 - Arbitrary Java (`spark._jvm`) The `spark._jvm` is used to execute arbitrary Java code. -[[back to top](#migration-assessment-report)] + ### AF302.3 - Arbitrary Java (`._jdf`) The `._jdf` is used to execute arbitrary Java code. -[[back to top](#migration-assessment-report)] + ### AF302.4 - Arbitrary Java (`._jcol`) The `._jcol` is used to execute arbitrary Java code. -[[back to top](#migration-assessment-report)] + ### AF302.5 - Arbitrary Java (`._jvm`) The `._jvm` is used to execute arbitrary Java code. -[[back to top](#migration-assessment-report)] + ### AF302.6 - Arbitrary Java (`._jvm.org.apache.log4j`) The `._jvm.org.apache.log4j` is used to execute arbitrary Java code. -[[back to top](#migration-assessment-report)] + ### AF303.1 - Java UDF (`spark.udf.registerJavaFunction`) The `spark.udf.registerJavaFunction` is used to register a Java UDF. -[[back to top](#migration-assessment-report)] + ### AF304.1 - JDBC datasource (`spark.read.format("jdbc")`) @@ -598,7 +463,7 @@ Workaround: Granting ANY FILE permissions will allow users to access untrusted databases. Note that ANY FILE will still enforce ACLs on any tables or external (storage) locations governed by Unity Catalog. This requires DBR 12.2 or later (DBR 12.1 or before is blocked on the network layer) -[[back to top](#migration-assessment-report)] + ### AF305.1 - boto3 @@ -611,13 +476,13 @@ For accessing cloud storage (S3), use storage credentials and external locations (AWS) Consider other ways to authenticate with boto3, e.g., by passing credentials from Databricks secrets directly to boto3 as a parameter, or loading them as environment variables. This page contains more information. Please note that unlike instance profiles, those methods do not provide short-lived credentials out of the box, and customers are responsible for rotating secrets according to their security needs. -[[back to top](#migration-assessment-report)] + ### AF305.2 - s3fs The `s3fs` library is used which provides posix type sematics for S3 access. s3fs is based on boto3 library and has similar restrictions. The recommendation is to use EXTERNAL VOLUMES mapped to the fixed s3 storage location or MANAGED VOLUMES. -[[back to top](#migration-assessment-report)] + ### AF306.1 - dbutils...getContext (`.toJson()`) @@ -630,7 +495,7 @@ context = get_context() context.__dict__ ``` -[[back to top](#migration-assessment-report)] + ### AF306.2 - dbutils...getContext @@ -642,13 +507,13 @@ context = get_context() context.__dict__ ``` -[[back to top](#migration-assessment-report)] + ### AF310.1 - credential passthrough (`dbutils.credentials.`) The `dbutils.credentials.` is used for credential passthrough. This is not supported by Unity Catalog. -[[back to top](#migration-assessment-report)] + ## AF311.x - dbutils (`dbutils`) @@ -659,31 +524,31 @@ Use [Volumes](https://docs.databricks.com/en/connect/unity-catalog/volumes.html) The `dbutils.fs.` pattern was found. DBUtils and other clients that directly read the data from cloud storage are not supported. Please note that `dbutils.fs` calls with /Volumes ([Volumes](https://docs.databricks.com/en/connect/unity-catalog/volumes.html)) will work. -[[back to top](#migration-assessment-report)] + ### AF311.2 - dbutils mount(s) (`dbutils.fs.mount`) The `dbutils.fs.mount` pattern was found. This is not supported by Unity Catalog. Use instead EXTERNAL LOCATIONS and VOLUMES. -[[back to top](#migration-assessment-report)] + ### AF311.3 - dbutils mount(s) (`dbutils.fs.refreshMounts`) The `dbutils.fs.refreshMounts` pattern was found. This is not supported by Unity Catalog. Use instead EXTERNAL LOCATIONS and VOLUMES. -[[back to top](#migration-assessment-report)] + ### AF311.4 - dbutils mount(s) (`dbutils.fs.unmount`) The `dbutils.fs.unmount` pattern was found. This is not supported by Unity Catalog. Use instead EXTERNAL LOCATIONS and VOLUMES. -[[back to top](#migration-assessment-report)] + ### AF311.5 - mount points (`dbfs:/mnt`) The `dbfs:/mnt` is used as a mount point. This is not supported by Unity Catalog. Use instead EXTERNAL LOCATIONS and VOLUMES. -[[back to top](#migration-assessment-report)] + ### AF311.6 - dbfs usage (`dbfs:/`) @@ -691,13 +556,13 @@ The `dbfs:/` pattern was found. DBFS is not supported by Unity Catalog. Use inst Please Note: `dbfs:/Volumes///` is a supported access pattern for spark. -[[back to top](#migration-assessment-report)] + ### AF311.7 - dbfs usage (`/dbfs/`) The `/dbfs/` pattern was found. DBFS is not supported by Unity Catalog. Use instead EXTERNAL LOCATIONS and VOLUMES. -[[back to top](#migration-assessment-report)] + ## AF313.x - SparkContext @@ -713,26 +578,26 @@ The following sc functions are also not supported in "Shared" Access Mode: empty The `spark.sparkContext` pattern was found, use the `spark` variable directly. -[[back to top](#migration-assessment-report)] + ### AF313.2 - SparkContext (`from pyspark.sql import SQLContext`) The `from pyspark.sql import SQLContext` and `import org.apache.spark.sql.SQLContext` are used. These are not supported in Unity Catalog. Possibly, the `spark` variable will suffice. -[[back to top](#migration-assessment-report)] + ### AF313.3 - SparkContext (`.binaryFiles`) The `.binaryFiles` pattern was found, this is not supported by Unity Catalog. Instead, please consider using `spark.read.format('binaryFiles')`. -[[back to top](#migration-assessment-report)] + ### AF313.4 - SparkContext (`.binaryRecords`) The `.binaryRecords` pattern was found, which is not supported by Unity Catalog Shared Compute access mode. -[[back to top](#migration-assessment-report)] + ### AF313.5 - SparkContext (`.emptyRDD`) @@ -746,7 +611,7 @@ schema = StructType([StructField("k", StringType(), True)]) spark.createDataFrame([], schema) ``` -[[back to top](#migration-assessment-report)] + ### AF313.6 - SparkContext (`.getConf`) @@ -756,37 +621,37 @@ The `.getConf` pattern was found. There may be significant false positives with - spark.conf.set() # sets a single value (If allowed) - spark.conf.getAll() unfortunately does not exist -[[back to top](#migration-assessment-report)] + ### AF313.7 - SparkContext ( `.hadoopFile` ) The `.hadoopFile` pattern was found. Use "Assigned" access mode compute or make permanent file type changes. -[[back to top](#migration-assessment-report)] + ### AF313.8 - SparkContext ( `.hadoopRDD` ) The `.hadoopRDD` pattern was found. Use "Assigned" access mode compute or make permanent file type changes. -[[back to top](#migration-assessment-report)] + ### AF313.9 - SparkContext ( `.init_batched_serializer` ) The `.init_batched_serializer` pattern was found. No suggestions available at this time. -[[back to top](#migration-assessment-report)] + ### AF313.10 - SparkContext ( `.newAPIHadoopFile` ) The `.newAPIHadoopFile` pattern was found. Use "Assigned" access mode compute or make permanent file type changes. -[[back to top](#migration-assessment-report)] + ### AF313.11 - SparkContext ( `.newAPIHadoopRDD` ) The `.newAPIHadoopRDD` pattern was found. Use "Assigned" access mode compute or make permanent file type changes. -[[back to top](#migration-assessment-report)] + ### AF313.12 - SparkContext ( `.parallelize` ) @@ -818,86 +683,86 @@ df = spark.createDataFrame(rows) df.display() ``` -[[back to top](#migration-assessment-report)] + ### AF313.13 - SparkContext ( `.pickleFile` ) The `.pickleFile` pattern was found. Use "Assigned" access mode compute or make permanent file type changes. -[[back to top](#migration-assessment-report)] + ### AF313.14 - SparkContext ( `.range` ) The `.range` pattern was found. Use `spark.range()` instead of `sc.range()` -[[back to top](#migration-assessment-report)] + ### AF313.15 - SparkContext ( `.rdd` ) The `.rdd` pattern was found. Use "Assigned" access mode compute or upgrade to the faster Spark DataFrame api. -[[back to top](#migration-assessment-report)] + ### AF313.16 - SparkContext ( `.runJob` ) The `.runJob` pattern was found. Use "Assigned" access mode compute or upgrade to the faster Spark DataFrame api. -[[back to top](#migration-assessment-report)] + ### AF313.17 - SparkContext ( `.sequenceFile` ) The `.sequenceFile` pattern was found. Use "Assigned" access mode compute or make permanent file type changes. -[[back to top](#migration-assessment-report)] + ### AF313.18 - SparkContext ( `.setJobGroup` ) The `.setJobGroup` pattern was found. `spark.addTag()` can attach a tag, and `getTags()` and `interruptTag(tag)` can be used to act upon the presence/absence of a tag. These APIs only work with Spark Connect (Shared Compute Mode) and will not work in "Assigned" access mode. -[[back to top](#migration-assessment-report)] + ### AF313.19 - SparkContext ( `.setLocalProperty` ) The `.setLocalProperty` pattern was found. -[[back to top](#migration-assessment-report)] + ### AF313.20 - SparkContext ( `.setSystemProperty` ) The `.setSystemProperty` pattern was found. Use "Assigned" access mode compute or find alternative within the spark API. -[[back to top](#migration-assessment-report)] + ### AF313.21 - SparkContext ( `.stop` ) The `.stop` pattern was found. Use "Assigned" access mode compute or find alternative within the spark API. -[[back to top](#migration-assessment-report)] + ### AF313.22 - SparkContext ( `.textFile` ) The `.textFile` pattern was found. Use "Assigned" access mode compute or find alternative within the spark API. -[[back to top](#migration-assessment-report)] + ### AF313.23 - SparkContext ( `.uiWebUrl`) The `.uiWebUrl` pattern was found. Use "Assigned" access mode compute or find alternative within the spark API. -[[back to top](#migration-assessment-report)] + ### AF313.24 - SparkContext (`.union`) The `.union` pattern was found, Use "Assigned" access mode compute or find alternative within the spark API. -[[back to top](#migration-assessment-report)] + ### AF313.25 - SparkContext (`.wholeTextFiles`) The `.wholeTextFiles` pattern was found. Use "Assigned" access mode compute or use the `spark.read().text("file_name")` API. -[[back to top](#migration-assessment-report)] + ## AF314.x - Distributed ML Databricks Runtime ML and Spark Machine Learning Library (MLlib) are not supported on shared Unity Catalog compute. The recommendation is to use Assigned mode cluster; Use cluster policies and (warm) compute pools to improve compute and cost management. @@ -906,85 +771,85 @@ Databricks Runtime ML and Spark Machine Learning Library (MLlib) are not support The `sparknlp` pattern was found. Use "Assigned" access mode compute. -[[back to top](#migration-assessment-report)] + ### AF314.2 - Distributed ML (`xgboost.spark`) The `xgboost.spark` pattern was found. Use "Assigned" access mode compute. -[[back to top](#migration-assessment-report)] + ### AF314.3 - Distributed ML (`catboost_spark`) The `catboost_spark` pattern was found. Use "Assigned" access mode compute. -[[back to top](#migration-assessment-report)] + ### AF314.4 - Distributed ML (`ai.catboost:catboost-spark`) The `ai.catboost:catboost-spark` pattern was found. Use "Assigned" access mode compute. -[[back to top](#migration-assessment-report)] + ### AF314.5 - Distributed ML (`hyperopt`) The `hyperopt` pattern was found. Use "Assigned" access mode compute. -[[back to top](#migration-assessment-report)] + ### AF314.6 - Distributed ML (`SparkTrials`) The `SparkTrials` pattern was found. Use "Assigned" access mode compute. -[[back to top](#migration-assessment-report)] + ### AF314.7 - Distributed ML (`horovod.spark`) The `horovod.spark` pattern was found. Use "Assigned" access mode compute. -[[back to top](#migration-assessment-report)] + ### AF314.8 - Distributed ML (`ray.util.spark`) The `ray.util.spark` pattern was found. Use "Assigned" access mode compute. -[[back to top](#migration-assessment-report)] + ### AF314.9 - Distributed ML (`databricks.automl`) The `databricks.automl` pattern was found. Use "Assigned" access mode compute. -[[back to top](#migration-assessment-report)] + ### AF308.1 - Graphframes (`from graphframes`) The `from graphframes` pattern was found. Use "Assigned" access mode compute. -[[back to top](#migration-assessment-report)] + ### AF309.1 - Spark ML (`pyspark.ml.`) The `pyspark.ml.` pattern was found. Use "Assigned" access mode compute. -[[back to top](#migration-assessment-report)] + ### AF315.1 - UDAF scala issue (`UserDefinedAggregateFunction`) The `UserDefinedAggregateFunction` pattern was found. Use "Assigned" access mode compute. -[[back to top](#migration-assessment-report)] + ### AF315.2 - applyInPandas (`applyInPandas`) The `applyInPandas` pattern was found. Use "Assigned" access mode compute. -[[back to top](#migration-assessment-report)] + ### AF315.3 - mapInPandas (`mapInPandas`) The `mapInPandas` pattern was found. Use "Assigned" access mode compute. -[[back to top](#migration-assessment-report)] + ## AF330.x - Streaming @@ -999,79 +864,79 @@ The assessment patterns and specifics are as follows: The `.trigger(continuous` pattern was found. Continuous processing mode is not supported in Unity Catalog shared access mode. Apache Spark continuous processing mode is not supported. See [Continuous Processing](https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html#continuous-processing) in the Spark Structured Streaming Programming Guide. -[[back to top](#migration-assessment-report)] + ### AF330.2 - Streaming (`kafka.sasl.client.callback.handler.class`) The `kafka.sasl.client.callback.handler.class` pattern was found. SASL features are not supported in Unity Catalog shared access mode. -[[back to top](#migration-assessment-report)] + ### AF330.3 - Streaming (`kafka.sasl.login.callback.handler.class`) The `kafka.sasl.login.callback.handler.class` pattern was found. SASL features are not supported in Unity Catalog shared access mode. -[[back to top](#migration-assessment-report)] + ### AF330.4 - Streaming (`kafka.sasl.login.class`) The `kafka.sasl.login.class` pattern was found. SASL features are not supported in Unity Catalog shared access mode. -[[back to top](#migration-assessment-report)] + ### AF330.5 - Streaming (`kafka.partition.assignment.strategy`) The `kafka.partition.assignment.strategy` pattern was found. Kafka features are not supported in Unity Catalog shared access mode. -[[back to top](#migration-assessment-report)] + ### AF330.6 - Streaming (`kafka.ssl.truststore.location`) The `kafka.ssl.truststore.location` pattern was found. SSL features are not supported in Unity Catalog shared access mode. -[[back to top](#migration-assessment-report)] + ### AF330.7 - Streaming (`kafka.ssl.keystore.location`) The `kafka.ssl.keystore.location` pattern was found. SSL features are not supported in Unity Catalog shared access mode. -[[back to top](#migration-assessment-report)] + ### AF330.8 - Streaming (`cleanSource`) The `cleanSource` pattern was found. The cleanSource operation is not supported in Unity Catalog shared access mode. -[[back to top](#migration-assessment-report)] + ### AF330.9 - Streaming (`sourceArchiveDir`) The `sourceArchiveDir` pattern was found. The sourceArchiveDir operation is not supported in Unity Catalog shared access mode. -[[back to top](#migration-assessment-report)] + ### AF330.10 - Streaming (`applyInPandasWithState`) The `applyInPandasWithState` pattern was found. The applyInPandasWithState operation is not supported in Unity Catalog shared access mode. -[[back to top](#migration-assessment-report)] + ### AF330.11 - Streaming (`.format("socket")`) The `.format("socket")` pattern was found. Socket source is not supported in Unity Catalog shared access mode. -[[back to top](#migration-assessment-report)] + ### AF330.12 - Streaming (`StreamingQueryListener`) The `StreamingQueryListener` pattern was found. StreamingQueryListener is not supported in Unity Catalog shared access mode. -[[back to top](#migration-assessment-report)] + ### AF330.13 - Streaming (`applyInPandasWithState`) The `applyInPandasWithState` pattern was found. The applyInPandasWithState operation is not supported in Unity Catalog shared access mode. -[[back to top](#migration-assessment-report)] + # Common Terms @@ -1093,11 +958,11 @@ Is shortcut for CREATE TABLE DEEP CLONE which only ## EXTERNAL LOCATION -[EXTERNAL LOCATION]([url](https://docs.databricks.com/en/connect/unity-catalog/external-locations.html#create-an-external-location)) is a UC object type describing a url to a cloud storage bucket + folder or storage account + container and folder +[EXTERNAL LOCATION](https://docs.databricks.com/en/connect/unity-catalog/external-locations.html#create-an-external-location) is a UC object type describing a url to a cloud storage bucket + folder or storage account + container and folder ## STORAGE CREDENTIAL -[STORAGE CREDENTIAL]([url](https://docs.databricks.com/en/sql/language-manual/sql-ref-storage-credentials.html)https://docs.databricks.com/en/sql/language-manual/sql-ref-storage-credentials.html) are a UC object encapsulating the credentials necessary to access cloud storage. +[STORAGE CREDENTIAL](https://docs.databricks.com/en/sql/language-manual/sql-ref-storage-credentials.html) is a UC object encapsulating the credentials necessary to access cloud storage. ## Assigned Clusters or Single User Clusters "Assigned Clusters" are Interactive clusters assigned to a single principal. Implicit in this term is that these clusters are enabled for Unity Catalog. Publicly available today, "Assigned Clusters" can be assigned to a user and the user's identity is used to access data resources. The access to the cluster is restricted to that single user to ensure accountability and accuracy of the audit logs. @@ -1113,4 +978,4 @@ The `data_security_mode` for these clusters are `SINGLE_USER` The `data_security_mode` for these clusters are `USER_ISOLATION`. -[[back to top](#migration-assessment-report)] + diff --git a/docs/content/docs/reference/commands.md b/docs/content/docs/reference/commands.md new file mode 100644 index 0000000000..ed4cb32b68 --- /dev/null +++ b/docs/content/docs/reference/commands.md @@ -0,0 +1,923 @@ +--- +linkTitle: "Commands" +title: "Commands" +weight: 1 +--- +This section contains commands reference. + +## Code migration commands + +See the [migration process diagram](docs/process/overview.md#diagram) to understand the role of the code migration commands in the migration process. + +After you're done with the [table migration](docs/reference/workflows/table_migration.md), you can proceed to the code migration. + +Once you're done with the code migration, you can run the [`cluster-remap` command](docs/reference/commands.md#cluster-remap) to remap the clusters to be UC compatible. + + + +### `lint-local-code` + +```text +databricks labs ucx lint-local-code +``` + +At any time, you can run this command to assess all migrations required in a local directory or a file. It only takes seconds to run and it +gives you an initial overview of what needs to be migrated without actually performing any migration. A great way to start a migration! + +This command detects all dependencies, and analyzes them. It is still experimental and at the moment only supports Python and SQL files. +We expect this command to run within a minute on code bases up to 50.000 lines of code. +Future versions of `ucx` will add support for more source types, and more migration details. + +When run from an IDE terminal, this command generates output as follows: +![img.png](/images/lint-local-code-output.png) +With modern IDEs, clicking on the file link opens the file at the problematic line + + + +### `migrate-local-code` + +```text +databricks labs ucx migrate-local-code +``` + +**(Experimental)** Once [table migration](docs/process/table_migration.md) is complete, you can run this command to +migrate all python and SQL files in the current working directory. This command is highly experimental and +at the moment only supports Python and SQL files and discards code comments and formatting during +the automated transformation process. + + + +### `migrate-dbsql-dashboards` + +{{< callout type="warning">}} +**Experimental**: once [table migration](docs/reference/workflows/table_migration.md) is complete, you can run this command to +migrate all Databricks SQL dashboards in the workspace. At this moment, this command is highly experimental and discards +formatting during the automated transformation process. +{{< /callout >}} + +```text +databricks labs ucx migrate-dbsql-dashboards [--dashboard-id ] +``` + +This command tags dashboards & queries that have been migrated with `migrated by UCX` tag. The original queries are +also backed up in the ucx installation folder, to allow for easy rollback (see [`revert-dbsql-dashboards` command](#revert-dbsql-dashboards)). + +This command can be run with `--dashboard-id` flag to migrate a specific dashboard. + +This command is incremental and can be run multiple times to migrate new dashboards. + + + +### `revert-dbsql-dashboards` + +{{< callout type="warning">}} +**Experimental**: this command reverts the migration of Databricks SQL dashboards in the workspace, after +`migrate-dbsql-dashboards` command is executed. +{{< /callout >}} + +```text +databricks labs ucx revert-dbsql-dashboards [--dashboard-id ] +``` + + + +This command can be run with `--dashboard-id` flag to migrate a specific dashboard. + + +## Cross-workspace installations + +When installing UCX across multiple workspaces, administrators need to keep UCX configurations in sync. +UCX will prompt you to select an account profile that has been defined in `~/.databrickscfg`. If you don't have one, +authenticate your machine with: + +* `databricks auth login --host https://accounts.cloud.databricks.com/` (AWS) +* `databricks auth login --host https://accounts.azuredatabricks.net/` (Azure) + +Ask your Databricks Account admin to run the [`sync-workspace-info` command](#sync-workspace-info) to sync the +workspace information with the UCX installations. Once the workspace information is synced, you can run the +[`create-table-mapping` command](#create-table-mapping) to align your tables with the Unity Catalog. + + + +### `sync-workspace-info` + +{{< callout type="warning" >}} +This command requires Databricks Account Administrator privileges. + +Use `--profile` to select the Databricks cli profile configured +with access to the Databricks account console (with account-level endpoint "https://accounts.cloud.databricks.com/" +or "https://accounts.azuredatabricks.net"). +{{< /callout >}} + +```text +databricks --profile ACCOUNTS labs ucx sync-workspace-info +14:07:07 INFO [databricks.sdk] Using Azure CLI authentication with AAD tokens +14:07:07 INFO [d.labs.ucx] Account ID: ... +14:07:10 INFO [d.l.blueprint.parallel][finding_ucx_installations_16] finding ucx installations 10/88, rps: 16.415/sec +14:07:10 INFO [d.l.blueprint.parallel][finding_ucx_installations_0] finding ucx installations 20/88, rps: 32.110/sec +14:07:11 INFO [d.l.blueprint.parallel][finding_ucx_installations_18] finding ucx installations 30/88, rps: 39.786/sec +... +``` + + + +This command uploads the workspace config to all workspaces in the account where `ucx` is installed. This command is +necessary to create an immutable default catalog mapping for [table migration](docs/process/table_migration) process and is the prerequisite +for [`create-table-mapping` command](#create-table-mapping). + +If you cannot get account administrator privileges in reasonable time, you can take the risk and +run [`manual-workspace-info` command](#manual-workspace-info) to enter Databricks Workspace IDs and Databricks +Workspace names. + + + +### `manual-workspace-info` + +```text +$ databricks labs ucx manual-workspace-info +14:20:36 WARN [d.l.ucx.account] You are strongly recommended to run "databricks labs ucx sync-workspace-info" by account admin, + ... otherwise there is a significant risk of inconsistencies between different workspaces. This command will overwrite all UCX + ... installations on this given workspace. Result may be consistent only within https://adb-987654321.10.azuredatabricks.net +Workspace name for 987654321 (default: workspace-987654321): labs-workspace +Next workspace id (default: stop): 12345 +Workspace name for 12345 (default: workspace-12345): other-workspace +Next workspace id (default: stop): +14:21:19 INFO [d.l.blueprint.parallel][finding_ucx_installations_11] finding ucx installations 10/89, rps: 24.577/sec +14:21:19 INFO [d.l.blueprint.parallel][finding_ucx_installations_15] finding ucx installations 20/89, rps: 48.305/sec +... +14:21:20 INFO [d.l.ucx.account] Synchronised workspace id mapping for installations on current workspace +``` + +This command is only supposed to be run if the [`sync-workspace-info` command](#sync-workspace-info) cannot be +run. It prompts the user to enter the required information manually and creates the workspace info. This command is +useful for workspace administrators who are unable to use the `sync-workspace-info` command, because they are not +Databricks Account Administrators. It can also be used to manually create the workspace info in a new workspace. + + + +### `create-account-groups` + +{{< callout type="warning" >}} +This command requires Databricks Account Administrator privileges. +{{< /callout >}} + +```text +$ databricks labs ucx create-account-groups [--workspace-ids 123,456,789] +``` + + +This command creates account-level groups if a workspace local +group is not present in the account. It crawls all workspaces configured in `--workspace-ids` flag, then creates +account level groups if a WS local group is not present in the account. If `--workspace-ids` flag is not specified, UCX +will create account groups for all workspaces configured in the account. + +The following scenarios are supported, if a group X: +- Exist in workspaces A,B,C, and it has same members in there, it will be created in the account +- Exist in workspaces A,B but not in C, it will be created in the account +- Exist in workspaces A,B,C. It has same members in A,B, but not in C. Then, X and C_X will be created in the account + +This command is useful for the setups, that don't have SCIM provisioning in place. + +Once you're done with this command, proceed to the [group migration workflow](#group-migration-workflow). + + + +### `validate-groups-membership` + +```text +$ databricks labs ucx validate-groups-membership +... +14:30:36 INFO [d.l.u.workspace_access.groups] Found 483 account groups +14:30:36 INFO [d.l.u.workspace_access.groups] No group listing provided, all matching groups will be migrated +14:30:36 INFO [d.l.u.workspace_access.groups] There are no groups with different membership between account and workspace +Workspace Group Name Members Count Account Group Name Members Count Difference +``` + +This command validates the groups to see if the groups at the account level and workspace level have different membership. +This command is useful for administrators who want to ensure that the groups have the correct membership. It can also be +used to debug issues related to group membership. See [group migration](docs/dev/implementation/local_group_migration.md) and +[group migration](docs/reference/workflows/group_migration.md) for more details. + +Valid group membership is important to ensure users has correct access after legacy table ACL is migrated in [table migration process](docs/process/table_migration) + + + +### `validate-table-locations` + +```text +$ databricks labs ucx validate-table-locations [--workspace-ids 123,456,789] +... +11:39:36 WARN [d.l.u.account.aggregate] Workspace 99999999 does not have UCX installed +11:39:37 WARN [d.l.u.account.aggregate] Overlapping table locations: 123456789:hive_metastore.database.table and 987654321:hive_metastore.database.table +11:39:37 WARN [d.l.u.account.aggregate] Overlapping table locations: 123456789:hive_metastore.database.table and 123456789:hive_metastore.another_database.table +``` + +This command validates the table locations by checking for overlapping table locations in the workspace and across +workspaces. Unity catalog does not allow overlapping table locations, also not between tables in different catalogs. +Overlapping table locations need to be resolved by the user before running the table migration. + +Options to resolve tables with overlapping locations are: +- Move one table and [skip](#skip) the other(s). +- Duplicate the tables by copying the data into a managed table and [skip](#skip) the original tables. + +Considerations when resolving tables with overlapping locations are: +- Migrate the tables one workspace at a time: + - Let later migrated workspaces read tables from the earlier migrated workspace catalogs. + - [Move](#move) tables between schemas and catalogs when it fits the data management model. +- The tables might have different: + - Metadata, like: + - Column schema (names, types, order) + - Description + - Tags + - ACLs + + + +### `cluster-remap` + +```text +$ databricks labs ucx cluster-remap +21:29:38 INFO [d.labs.ucx] Remapping the Clusters to UC +Cluster Name Cluster Id +Field Eng Shared UC LTS Cluster 0601-182128-dcbte59m +Shared Autoscaling Americas cluster 0329-145545-rugby794 +``` +```text +Please provide the cluster id's as comma separated value from the above list (default: ): +``` + +Once you're done with the [code migration](#code-migration-commands), you can run this command to remap the clusters to UC enabled. + +This command will remap the cluster to uc enabled one. When we run this command it will list all the clusters +and its id's and asks to provide the cluster id's as comma separated value which has to be remapped, by default it will take all cluster ids. +Once we provide the cluster id's it will update these clusters to UC enabled.Back up of the existing cluster +config will be stored in backup folder inside the installed location(backup/clusters/cluster_id.json) as a json file.This will help +to revert the cluster remapping. + +You can revert the cluster remapping using the [`revert-cluster-remap` command](#revert-cluster-remap). + + + +### `revert-cluster-remap` + +```text +$ databricks labs ucx revert-cluster-remap +21:31:29 INFO [d.labs.ucx] Reverting the Remapping of the Clusters from UC +21:31:33 INFO [d.labs.ucx] 0301-055912-4ske39iq +21:31:33 INFO [d.labs.ucx] 0306-121015-v1llqff6 +Please provide the cluster id's as comma separated value from the above list (default: ): +``` + +If a customer want's to revert the cluster remap done using the [`cluster-remap` command](#cluster-remap) they can use this command to revert +its configuration from UC to original one.It will iterate through the list of clusters from the backup folder and reverts the +cluster configurations to original one.This will also ask the user to provide the list of clusters that has to be reverted as a prompt. +By default, it will revert all the clusters present in the backup folder + + + +### `upload` + +```text +$ databricks labs ucx upload --file --run-as-collection True +21:31:29 WARNING [d.labs.ucx] The schema of CSV files is NOT validated, ensure it is correct +21:31:29 INFO [d.labs.ucx] Finished uploading: +``` + +Upload a file to a single workspace (`--run-as-collection False`) or a collection of workspaces +(`--run-as-collection True`). This command is especially useful when uploading the same file to multiple workspaces. + +### `download` + +```text +$ databricks labs ucx download --file --run-as-collection True +21:31:29 INFO [d.labs.ucx] Finished downloading: +``` + +Download a csv file from a single workspace (`--run-as-collection False`) or a collection of workspaces +(`--run-as-collection True`). This command is especially useful when downloading the same file from multiple workspaces. + +### `join-collection` + +```text +$ databricks labs ucx join-collection --workspace-ids --profile +``` + +`join-collection` command joins 2 or more workspaces into a collection. This helps in running supported cli commands as a collection +`join-collection` command updates config.yml file on each workspace ucx installation with installed_workspace_ids attribute. +In order to run `join-collectioon` command a user should: + - be an Account admin on the Databricks account + - be a Workspace admin on all the workspaces to be joined as a collection) or a collection of workspaces + - have installed UCX on the workspace +The `join-collection` command will fail and throw an error msg if the above conditions are not met. + +### Check collection eligibility + +Once `join-collection` command is run, it allows user to run multiple cli commands as a collection. The following cli commands +are eligible to be run as a collection. User can run the below commands as collection by passing an additional flag `--run-as-collection=True` +- `ensure-assessment-run` +- `create-table-mapping` +- `principal-prefix-access` +- `migrate-credentials` +- `create-uber-principal` +- `create-missing-principals` +- `validate-external-location` +- `migrate-locations` +- `create-catalog-schemas` +- `migrate-tables` +- `migrate-acls` +- `migrate-dbsql-dashboards` +- `validate-group-membership` +Ex: `databricks labs ucx ensure-assessment-run --run-as-collection=True` + +## Metastore related commands + +These commands are used to assign a Unity Catalog metastore to a workspace. The metastore assignment is a pre-requisite +for any further migration steps. + + + +### `show-all-metastores` + +```text +databricks labs ucx show-all-metastores [--workspace-id ] +``` + +This command lists all the metastores available to be assigned to a workspace. If no workspace is specified, it lists +all the metastores available in the account. This command is useful when there are multiple metastores available within +a region, and you want to see which ones are available for assignment. + + + +### `assign-metastore` + +```text +databricks labs ucx assign-metastore --workspace-id [--metastore-id ] +``` + +This command assigns a metastore to a workspace with `--workspace-id`. If there is only a single metastore in the +workspace region, the command automatically assigns that metastore to the workspace. If there are multiple metastores +available, the command prompts for specification of the metastore (id) you want to assign to the workspace. + + + +### `create-ucx-catalog` + +```commandline +databricks labs ucx create-ucx-catalog +16:12:59 INFO [d.l.u.hive_metastore.catalog_schema] Validating UC catalog: ucx +Please provide storage location url for catalog: ucx (default: metastore): ... +16:13:01 INFO [d.l.u.hive_metastore.catalog_schema] Creating UC catalog: ucx +``` + +Create and setup UCX artifact catalog. Amongst other things, the artifacts are used for tracking the migration progress +across workspaces. + +## Table migration commands + +These commands are vital part of [table migration process](docs/process/table_migration.md) process and require +the [assessment workflow](docs/reference/workflows/assessment.md) and +[group migration workflow](docs/reference/workflows/group_migration.md) to be completed. +See the [migration process diagram](docs/process/overview.md#diagram) to understand the role of the table migration commands in +the migration process. + +The first step is to run the [`principal-prefix-access` command](#principal-prefix-access) to identify all +the storage accounts used by tables in the workspace and their permissions on each storage account. + +If you don't have any storage credentials and external locations configured, you'll need to run +the [`migrate-credentials` command](#migrate-credentials) to migrate the service principals +and [`migrate-locations` command](#migrate-locations) to create the external locations. +If some of the external locations already exist, you should run +the [`validate-external-locations` command](#validate-external-locations). +You'll need to create the [uber principal](#create-uber-principal) with the _**access to all storage**_ used to tables in +the workspace, so that you can migrate all the tables. If you already have the principal, you can skip this step. + +Ask your Databricks Account admin to run the [`sync-workspace-info` command](#sync-workspace-info) to sync the +workspace information with the UCX installations. Once the workspace information is synced, you can run the +[`create-table-mapping` command](#create-table-mapping) to align your tables with the Unity Catalog, +[create catalogs and schemas](#create-catalogs-schemas) and start the migration using [`migrate-tables` command](#migrate-tables). During multiple runs of +the table migration workflow, you can use the [`revert-migrated-tables` command](#revert-migrated-tables) to +revert the tables that were migrated in the previous run. You can also skip the tables that you don't want to migrate +using the [`skip` command](#skip). + +Once you're done with the table migration, proceed to the [code migration](#code-migration). + + +### `principal-prefix-access` + +```text +databricks labs ucx principal-prefix-access [--subscription-ids ] [--aws-profile ] +``` + +This command depends on results from the [assessment workflow](docs/reference/workflows/assessment.md) and requires `AWS ClI` +or `Azure CLI` to be installed and authenticated for the given machine. + +This command identifies all the storage accounts used by tables in the workspace and their permissions on each storage account. +Once you're done running this command, proceed to the [`migrate-credentials` command](#migrate-credentials). + +The "prefix" refers to the start - i.e. prefix - of table locations that point to the cloud storage location. + + +{{< tabs items="AWS,Azure" >}} + +{{< tab >}} + +```commandline +databricks labs ucx principal-prefix-access --aws-profile test-profile +``` + +Use to identify all instance profiles in the workspace, and map their access to S3 buckets. +Also captures the IAM roles which has UC arn listed, and map their access to S3 buckets +This requires `aws` CLI to be installed and configured. + +For AWS this command produces a file named `aws_instance_profile_info.csv`. +It has the following format: + +| **role_arn** | **resource_type** | **privilege** | **resource_path** | +|------------------------------------------------------|-------------------|---------------|-----------------------| +| arn:aws:iam::1234:instance-profile/instance-profile1 | s3 | WRITE_FILES | s3://s3_bucket1/path1 | + +{{< /tab >}} + + +{{< tab >}} + +```commandline +databricks labs ucx principal-prefix-access --subscription-ids test-subscription-id +``` + +Use to identify all storage account used by tables, identify the relevant Azure service principals and their permissions +on each storage account. The command is used to identify Azure Service Principals, which have `Storage Blob Data Contributor`, +`Storage Blob Data Reader`, `Storage Blob Data Owner` roles, or custom read/write roles on ADLS Gen2 locations that are being +used in Databricks. +This requires Azure CLI to be installed and configured via `az login`. + +It outputs `azure_storage_account_info.csv` which will be later used by migrate-credentials command to create UC storage credentials. + +{{< callout type="info" >}} +This command only lists azure storage account gen2, storage format wasb:// or adl:// are not supported in UC and those storage info +will be skipped. +{{< /callout >}} + +{{< /tab >}} + +{{< /tabs >}} + + +Once done, proceed to the [`migrate-credentials` command](#migrate-credentials). + +### `create-missing-principals` + +{{< callout type="info" >}} +This command is only available for AWS. +{{< /callout >}} + +```bash +databricks labs ucx create-missing-principals --aws-profile --single-role +``` +This command identifies all the S3 locations that are missing a UC compatible role and creates them. +It takes single-role optional parameter. +If set to True, it will create a single role for all the S3 locations. +Otherwise, it will create a role for each S3 location. + +Two optional parameter are available for this command: +`--role-name` - This parameter is used to set the prefix for the role name. The default value is `UCX-ROLE`. +`--role-policy` - This parameter is used to set the prefix for the role policy name. The default value is `UCX-POLICY`. + + + +### `delete-missing-principals` + +{{< callout type="info" >}} +This command is only available for AWS. +{{< /callout >}} + +```bash +databricks labs ucx delete-missing-principals --aws-profile +``` +This command helps to delete the IAM role created by UCX. It lists all the IAM Roles generated by the principal-prefix-access +command and allows user to select multiple roles to delete. It also checks if selected roles are mapped to any storage credentials +and asks for confirmation from user. Once confirmed, it deletes the role and its associated inline policy. + + + +### `create-uber-principal` + +{{< callout type="warning" >}} +Requires Cloud IAM admin privileges. +{{< /callout >}} + +```text +databricks labs ucx create-uber-principal [--subscription-ids X] +``` + + + +Once the [`assessment` workflow](docs/reference/workflows/assessment.md) complete, you should run this command to create a service principal with the +_**read-only access to all storage**_ used by tables in this workspace. It will also configure the +[UCX Cluster Policy](docs/installation) & SQL Warehouse data access configuration to use this service principal for migration +workflows. Once migration is complete, this service principal should be unprovisioned. + +On Azure, it creates a principal with `Storage Blob Data Contributor` role assignment on every storage account using +Azure Resource Manager APIs. + +This command is one of prerequisites for the [table migration process](docs/process/table_migration.md). + + + +### `migrate-credentials` + +```commandline +databricks labs ucx migrate-credentials +``` + +For Azure, this command prompts to confirm performing the following credential migration steps: +1. [RECOMMENDED] For each storage account, create access connectors with managed identities that have the + `Storage Blob Data Contributor` role on the respective storage account. A storage credential is created for each + access connector. +2. Migrate Azure Service Principals, which have `Storage Blob Data Contributor`, + `Storage Blob Data Reader`, `Storage Blob Data Owner`, or custom roles on ADLS Gen2 locations that are being used in + Databricks, to UC storage credentials. The Azure Service Principals to location mapping are listed + in `/Users/{user_name}/.ucx/azure_storage_account_info.csv` which is generated by + [`principal-prefix-access` command](#principal-prefix-access). Please review the file and delete the Service + Principals you do not want to be migrated. The command will only migrate the Service Principals that have client + secret stored in Databricks Secret. + + **Warning**: Service principals used to access storage accounts behind firewalls might cause connectivity issues. We + recommend to use access connectors instead. + +For AWS, this command migrates AWS Instance Profiles that are being used in Databricks, to UC storage credentials. +The AWS Instance Profiles to location mapping are listed in +{workspace ucx folder}/aws_instance_profile_info.csv which is generated by principal_prefix_access command. +Please review the file and delete the Instance Profiles you do not want to be migrated.
The aws_profile parameter indicates the aws profile to use. + +Once you're done with this command, run [`validate-external-locations` command](#validate-external-locations) after this one. + + + +### `validate-external-locations` + +```text +databricks labs ucx validate-external-locations +``` + +Once the [`assessment` workflow](docs/reference/workflows/assessment.md) finished successfully, [storage credentials](#migrate-credentials) are configured, +run this command to validate and report the missing Unity Catalog external locations to be created. + +This command validates and provides mapping to external tables to external locations, also as Terraform configurations. + +Once you're done with this command, proceed to the [`migrate-locations` command](#migrate-locations). + + + +### `migrate-locations` + +```text +databricks labs ucx migrate-locations +``` + +Once the [`assessment` workflow](docs/reference/workflows/assessment.md) finished successfully, and [storage credentials](#migrate-credentials) are configured, +run this command to have Unity Catalog external locations created. The candidate locations to be created are extracted from guess_external_locations +task in the assessment job. You can run [`validate-external-locations` command](#validate-external-locations) to check the candidate locations. + +**Location ACLs:** +`migrate-locations` command applies any location ACL from existing cluster. +For Azure, it checks if there are any interactive cluster or SQL warehouse +which has service principals configured to access storage. It maps the storage account to the external location created and grants `CREATE_EXTERNAL_TABLE`, +`CREATE_EXTERNAL_VOLUME` and `READ_FILES` permission on the location to all the user who have access to the interactive cluster or SQL warehouse +For AWS, it checks any instance profiles mapped to the interactive cluster or SQL warehouse. It checks the mapping of instance profiles to the bucket. It then +maps the bucket to the external locations created and grants `CREATE_EXTERNAL_TABLE`, `CREATE_EXTERNAL_VOLUME` and `READ_FILES` permission on the location to all the user who have access to the interactive cluster +or SQL warehouse + +Once you're done with this command, proceed to the [`create-table-mapping` command](#create-table-mapping). + + + +### `create-table-mapping` + +```text +databricks labs ucx create-table-mapping +``` + +Once the [`assessment` workflow](docs/reference/workflows/assessment.md) finished successfully and [workspace info is synchronized](#sync-workspace-info), +run this command to create the initial table mapping for review in CSV format in the Databricks Workspace: + +```text +workspace_name,catalog_name,src_schema,dst_schema,src_table,dst_table +labs-azure,labs_azure,default,default,ucx_tybzs,ucx_tybzs +``` + +The format of the mapping file is as follows: + +| **columns:** | **workspace_name** | **catalog_name** | **src_schema** | **dst_schema** | **src_table** | **dst_table** | +|--------------|---------------------|--------------|----------------|----------------|---------------|---------------| +| values: | data_engineering_ws | de_catalog | database1 | database1 | table1 | table1 | + +You are supposed to review this mapping and adjust it if necessary. This file is in CSV format, so that you can edit it +easier in your favorite spreadsheet application. + +Once you're done with this command, [create catalogs and schemas](#create-catalogs-schemas). During +multiple runs of the table migration workflow, you can use the [`revert-migrated-tables` command](#revert-migrated-tables) +to revert the tables that were migrated in the previous run. You can also skip the tables that you don't want to migrate +using the [`skip` command](#skip). + +This command is one of prerequisites for the [table migration process](docs/process/table_migration.md). + +Once you're done with table migration, proceed to the [code migration](#code-migration). + + + +### `skip` + +```text +databricks labs ucx skip --schema X [--table Y] [--view Z] +``` + +Anytime after [`create-table-mapping` command](#create-table-mapping) is executed, you can run this command. + +This command allows users to skip certain schemas, tables or views during the [table migration](docs/process/table_migration.md) process. +The command takes `--schema` and, optionally, `--table` and `--view` flags to specify the schema, table or view to skip. +If no `--table` flag is provided, all tables in the specified HMS database are skipped. The `--table` and `--view` can +only be used exclusively. This command is useful to temporarily disable migration of a particular schema, table or view. + +Once you're done with table migration, proceed to the [code migration](#code-migration-commands). + + + +### `unskip` + +```commandline +databricks labs ucx unskip --schema X [--table Y] [--view Z] +``` + +This command removes the mark set by the [`skip` command](#skip) on the given schema, table or view. + + + +### `create-catalogs-schemas` + +```text +databricks labs ucx create-catalogs-schemas +``` +After [`create-table-mapping` command](#create-table-mapping) is executed, you can run this command to have the required UC catalogs and schemas created. +This command is supposed to be run before migrating tables to UC using [table migration process](docs/process/table_migration.md). +Catalog & Schema ACL: +`create-catalogs-schemas` command also applies any catalog and schema ACL from existing clusters. +For Azure it checks if there are any interactive cluster or sql warehouse which has service principals configured to access storage. +It maps the storage account to the tables which has external location on those storage account created and grants `USAGE` access to +the schema and catalog if at least one such table is migrated to it. +For AWS, it checks any instance profiles mapped to the interactive cluster or sql warehouse. It checks the mapping of instance profiles +to the bucket. It then maps the bucket to the tables which has external location on those bucket created and grants `USAGE` access to +the schema and catalog if at least one such table is migrated to it. + + +### `migrate-tables` + +```text +databricks labs ucx migrate-tables +``` + +Anytime after [`create-table-mapping` command](#create-table-mapping) is executed, you can run this command. + +This command kicks off the [table migration](docs/process/table_migration.md) process. It triggers the `migrate-tables` workflow, +and if there are HiveSerDe tables detected, prompt whether to trigger the `migrate-external-hiveserde-tables-in-place-experimental` workflow. + +Table and View ACL: +`migrate-tables` command also applies any table and view ACL from existing clusters. +For Azure it checks if there are any interactive cluster or sql warehouse which has service principals configured to access storage. +It maps the storage account to the tables which has external location on those storage account created and grants either `SELECT` permission if +the service principal only has read access on the storage account and `ALL_PRIVILEGES` if the service principal has write access on the storage account +For AWS, it checks any instance profiles mapped to the interactive cluster or sql warehouse. It checks the mapping of instance profiles +to the bucket. It then maps the bucket to the tables which has external location on those bucket created and grants either `SELECT` permission if +the instance profile only has read access on the bucket and `ALL_PRIVILEGES` if the instance profile has write access on the bucket. + + + +### `revert-migrated-tables` + +```text +databricks labs ucx revert-migrated-tables --schema X --table Y [--delete-managed] +``` + +Anytime after [`create-table-mapping` command](#create-table-mapping) is executed, you can run this command. + +This command removes the `upgraded_from` property on a migrated table for re-migration in the [table migration](docs/process/table_migration.md) process. +This command is useful for developers and administrators who want to revert the migration of a table. It can also be used +to debug issues related to table migration. + +Go back to the [`create-table-mapping` command](#create-table-mapping) after you're done with this command. + + + +### `move` + +```text +databricks labs ucx move --from-catalog A --from-schema B --from-table C --to-catalog D --to-schema E +``` + +This command moves a UC table/tables from one schema to another schema after +the [table migration](docs/process/table_migration.md) process. This is useful for developers and administrators who want +to adjust their catalog structure after tables upgrade. + +Users will be prompted whether tables/view are dropped after moving to new schema. This only applies to `MANAGED` tables and views. + +This command moves different table types differently: +- `MANAGED` tables are deep-cloned to the new schema. +- `EXTERNAL` tables are dropped from the original schema, then created in the target schema using the same location. +This is due to Unity Catalog not supporting multiple tables with overlapping paths +- `VIEW` are recreated using the same view definition. + +This command supports moving multiple tables at once, by specifying `*` as the table name. + + + +### `alias` + +```text +databricks labs ucx alias --from-catalog A --from-schema B --from-table C --to-catalog D --to-schema E +``` + +This command aliases a UC table/tables from one schema to another schema in the same or different catalog. +It takes a `WorkspaceClient` object and `from` and `to` parameters as parameters and aliases the tables using +the `TableMove` class. This command is useful for developers and administrators who want to create an alias for a table. +It can also be used to debug issues related to table aliasing. + + + +## Utility + +### `logs` + +```text +$ databricks labs ucx logs [--workflow WORKFLOW_NAME] [--debug] +``` + +This command displays the logs of the last run of the specified workflow. If no workflow is specified, it displays +the logs of the workflow that was run the last. This command is useful for developers and administrators who want to +check the logs of the last run of a workflow and ensure that it was executed as expected. It can also be used for +debugging purposes when a workflow is not behaving as expected. By default, only `INFO`, `WARNING`, and `ERROR` logs +are displayed. To display `DEBUG` logs, use the `--debug` flag. + + + +### `ensure-assessment-run` + +```commandline +databricks labs ucx ensure-assessment-run +``` + +This command ensures that the [assessment workflow](docs/reference/workflows/assessment.md) was run on a workspace. +This command will block until job finishes. +Failed workflows can be fixed with the [`repair-run` command](#repair-run). Workflows and their status can be +listed with the [`workflows` command](#workflows). + + + +### `update-migration-progress` + +```commandline +databricks labs ucx update-migration-progress +``` + +This command runs the [(experimental) migration progress workflow](docs/reference/workflows/progress.md) to update +the migration status of workspace resources that need to be migrated. It does this by triggering +the `migration-progress-experimental` workflow to run on a workspace and waiting for +it to complete. + +Workflows and their status can be listed with the [`workflows` command](#workflows), while failed workflows can +be fixed with the [`repair-run` command](#repair-run). + + + +### `repair-run` + +```commandline +databricks labs ucx repair-run --step WORKFLOW_NAME +``` + +This command repairs a failed [UCX Workflow](#workflows). This command is useful for developers and administrators who +want to repair a failed job. It can also be used to debug issues related to job failures. This operation can also be +done via [user interface](https://docs.databricks.com/en/workflows/jobs/repair-job-failures.html). Workflows and their +status can be listed with the [`workflows` command](#workflows). + + + +### `workflows` + +See the [migration process diagram](docs/process/overview.md#diagram) to understand the role of each workflow in the migration process. + +```text +$ databricks labs ucx workflows +Step State Started +assessment RUNNING 1 hour 2 minutes ago +099-destroy-schema UNKNOWN +migrate-groups UNKNOWN +remove-workspace-local-backup-groups UNKNOWN +validate-groups-permissions UNKNOWN +``` + +This command displays the [deployed workflows](#workflows) and their state in the current workspace. It fetches the latest +job status from the workspace and prints it in a table format. This command is useful for developers and administrators +who want to check the status of UCX workflows and ensure that they have been executed as expected. It can also be used +for debugging purposes when a workflow is not behaving as expected. Failed workflows can be fixed with +the [`repair-run` command](#repair-run). + + + +### `open-remote-config` + +```commandline +databricks labs ucx open-remote-config +``` + +This command opens the remote configuration file in the default web browser. It generates a link to the configuration file +and opens it using the `webbrowser.open()` method. This command is useful for developers and administrators who want to view or +edit the remote configuration file without having to manually navigate to it in the workspace. It can also be used to quickly +access the configuration file from the command line. Here's the description of configuration properties: + + * `inventory_database`: A string representing the name of the inventory database. + * `workspace_group_regex`: An optional string representing the regular expression to match workspace group names. + * `workspace_group_replace`: An optional string to replace the matched group names with. + * `account_group_regex`: An optional string representing the regular expression to match account group names. + * `group_match_by_external_id`: A boolean value indicating whether to match groups by their external IDs. + * `include_group_names`: An optional list of strings representing the names of groups to include for migration. + * `renamed_group_prefix`: An optional string representing the prefix to add to renamed group names. + * `instance_pool_id`: An optional string representing the ID of the instance pool. + * `warehouse_id`: An optional string representing the ID of the warehouse. + * `connect`: An optional `Config` object representing the configuration for connecting to the warehouse. + * `num_threads`: An optional integer representing the number of threads to use for migration. + * `database_to_catalog_mapping`: An optional dictionary mapping source database names to target catalog names. + * `default_catalog`: An optional string representing the default catalog name. + * `log_level`: An optional string representing the log level. + * `workspace_start_path`: A string representing the starting path for notebooks and directories crawler in the workspace. + * `instance_profile`: An optional string representing the name of the instance profile. + * `spark_conf`: An optional dictionary of Spark configuration properties. + * `override_clusters`: An optional dictionary mapping job cluster names to existing cluster IDs. + * `policy_id`: An optional string representing the ID of the cluster policy. + * `include_databases`: An optional list of strings representing the names of databases to include for migration. + + + +### `installations` + +```text +$ databricks labs ucx installations +... +13:49:16 INFO [d.labs.ucx] Fetching installations... +13:49:17 INFO [d.l.blueprint.parallel][finding_ucx_installations_5] finding ucx installations 10/88, rps: 22.838/sec +13:49:17 INFO [d.l.blueprint.parallel][finding_ucx_installations_9] finding ucx installations 20/88, rps: 35.002/sec +13:49:17 INFO [d.l.blueprint.parallel][finding_ucx_installations_2] finding ucx installations 30/88, rps: 51.556/sec +13:49:18 INFO [d.l.blueprint.parallel][finding_ucx_installations_9] finding ucx installations 40/88, rps: 56.272/sec +13:49:18 INFO [d.l.blueprint.parallel][finding_ucx_installations_19] finding ucx installations 50/88, rps: 67.382/sec +... +Path Database Warehouse +/Users/serge.smertin@databricks.com/.ucx ucx 675eaf1ff976aa98 +``` + +This command displays the [installations](docs/installation/advanced.md) by different users on the same workspace. It fetches all +the installations where the `ucx` package is installed and prints their details in JSON format. This command is useful +for administrators who want to see which users have installed `ucx` and where. It can also be used to debug issues +related to multiple installations of `ucx` on the same workspace. + + + + +### `report-account-compatibility` + +```text +databricks labs ucx report-account-compatibility --profile labs-azure-account +12:56:09 INFO [databricks.sdk] Using Azure CLI authentication with AAD tokens +12:56:09 INFO [d.l.u.account.aggregate] Generating readiness report +12:56:10 INFO [databricks.sdk] Using Azure CLI authentication with AAD tokens +12:56:10 INFO [databricks.sdk] Using Azure CLI authentication with AAD tokens +12:56:15 INFO [databricks.sdk] Using Azure CLI authentication with AAD tokens +12:56:15 INFO [d.l.u.account.aggregate] Querying Schema ucx +12:56:21 WARN [d.l.u.account.aggregate] Workspace 4045495039142306 does not have UCX installed +12:56:21 INFO [d.l.u.account.aggregate] UC compatibility: 30.303030303030297% (69/99) +12:56:21 INFO [d.l.u.account.aggregate] cluster type not supported : LEGACY_TABLE_ACL: 22 objects +12:56:21 INFO [d.l.u.account.aggregate] cluster type not supported : LEGACY_SINGLE_USER: 24 objects +12:56:21 INFO [d.l.u.account.aggregate] unsupported config: spark.hadoop.javax.jdo.option.ConnectionURL: 10 objects +12:56:21 INFO [d.l.u.account.aggregate] Uses azure service principal credentials config in cluster.: 1 objects +12:56:21 INFO [d.l.u.account.aggregate] No isolation shared clusters not supported in UC: 1 objects +12:56:21 INFO [d.l.u.account.aggregate] Data is in DBFS Root: 23 objects +12:56:21 INFO [d.l.u.account.aggregate] Non-DELTA format: UNKNOWN: 5 objects +``` + + +### `export-assessment` + +```commandline +databricks labs ucx export-assessment +``` +The export-assessment command is used to export UCX assessment results to a specified location. When you run this command, you will be prompted to provide details on the destination path and the type of report you wish to generate. If you do not specify these details, the command will default to exporting the main results to the current directory. The exported file will be named based on the selection made in the format. Eg: export_{query_choice}_results.zip +- **Choose a path to save the UCX Assessment results:** + - **Description:** Specify the path where the results should be saved. If not provided, results will be saved in the current directory. + +- **Choose which assessment results to export:** + - **Description:** Select the type of results to export. Options include: + - `azure` + - `estimates` + - `interactive` + - `main` + - **Default:** `main` + + diff --git a/docs/content/docs/reference/common_challenges.md b/docs/content/docs/reference/common_challenges.md new file mode 100644 index 0000000000..d8be7a8fc2 --- /dev/null +++ b/docs/content/docs/reference/common_challenges.md @@ -0,0 +1,235 @@ +# Common Challenges and the Solutions +Users might encounter some challenges while installing and executing UCX. Please find the listing of some common challenges and the solutions below. + + + +### Network Connectivity Issues + +**From local machine to the Databricks Account and Workspace:** UCX +installation process has to be run from the local laptop using +Databricks CLI and it will deploy the latest version of UCX into the +Databricks workspace. For this reason, the Databricks account and +workspace needs to be accessible from the laptop. Sometimes, the +workspace might have a network isolation, like it can only be reached +from a VPC, or from a specific IP range. + +**Solution:** Please check that your laptop has network connectivity to +the Databricks account and workspace. If not, you might need to be +connected to a VPN or configure an HTTP proxy to access your workspace. + +**From local machine to GitHub:** UCX needs internet access to connect to GitHub (https://api.github.com +and https://raw.githubusercontent.com) for downloading the tool from the machine running the installation. The +installation will fail if there is no internet connectivity to these URLs. + +**Solution:** Ensure that GitHub is reachable from the local machine. If +not, make necessary changes to the network/firewall settings. + +**From Databricks workspace to PyPi:** There are some dependent +libraries which need to be installed from +[pypi.org](https://pypi.org/) to run the UCX workflows from the +Databricks workspace. If the workspace doesn’t have network +connectivity, then the job might fail with +NO_MATCHING_DISTRIBUTION_ERROR. + +**Solution:** Version 0.24.0 of UCX supports workspace with no internet +access. Please upgrade UCX and rerun the installation. Reply *yes* to +the question "Does the given workspace block Internet access?" asked +during installation. It will then upload all necessary dependencies to +the workspace. Also, please note that UCX uses both UC and non-UC +enabled clusters. If you have different proxy settings for each, then +please update the necessary proxies (eg. with init scripts) for each +cluster type. + +**Local machine to Databricks Account and Workspace connection failed due to proxy and self-signed cert:** +When customer uses web proxy and self-singed certification, UCX may not be able to connect to Account and Workspace +with following errors: +``` +File "/Users/userabc/.databricks/labs/ucx/state/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 466, in _make_request + self._validate_conn(conn) + File "/Users/userabc/.databricks/labs/ucx/state/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 1095, in _validate_conn + conn.connect() + File "/Users/userabc/.databricks/labs/ucx/state/venv/lib/python3.10/site-packages/urllib3/connection.py", line 652, in connect + sock_and_verified = _ssl_wrap_socket_and_match_hostname( + File "/Users/userabc/.databricks/labs/ucx/state/venv/lib/python3.10/site-packages/urllib3/connection.py", line 805, in _ssl_wrap_socket_and_match_hostname + ssl_sock = ssl_wrap_socket( + File "/Users/userabc/.databricks/labs/ucx/state/venv/lib/python3.10/site-packages/urllib3/util/ssl_.py", line 465, in ssl_wrap_socket + ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) + File "/Users/userabc/.databricks/labs/ucx/state/venv/lib/python3.10/site-packages/urllib3/util/ssl_.py", line 509, in _ssl_wrap_socket_impl + return ssl_context.wrap_socket(sock, server_hostname=server_hostname) + File "/opt/homebrew/Cellar/python@3.10/3.10.14/Frameworks/Python.framework/Versions/3.10/lib/python3.10/ssl.py", line 513, in wrap_socket + return self.sslsocket_class._create( + File "/opt/homebrew/Cellar/python@3.10/3.10.14/Frameworks/Python.framework/Versions/3.10/lib/python3.10/ssl.py", line 1104, in _create + self.do_handshake() + File "/opt/homebrew/Cellar/python@3.10/3.10.14/Frameworks/Python.framework/Versions/3.10/lib/python3.10/ssl.py", line 1375, in do_handshake + self._sslobj.do_handshake() +ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1007) +``` + +**Solution:** set both `REQUESTS_CA_BUNDLE` and `CURL_CA_BUNDLE` +[to force requests library to set verify=False](https://github.com/psf/requests/blob/8c211a96cdbe9fe320d63d9e1ae15c5c07e179f8/requests/sessions.py#L718) +as well as set `SSL_CERT_DIR` env var pointing to the proxy CA cert for the urllib3 library. + + + +### Insufficient Privileges + +**User is not a Databricks workspace administrator:** User running the +installation needs to be a workspace administrator as the CLI will +deploy the UCX tool into the workspace, create jobs, and dashboards. + +**Solution:** Identify a workspace admin from your team and ask them to +install UCX with their authentication, or request a workspace +administrator to grant you temporary administrator privileges to run the +installation. More details on the issues that you can run into if +you are not an admin (and some possible solutions) can be found +[here](/docs/reference/troubleshooting.md#resolving-common-errors-on-ucx-install). + +**User is not a Cloud IAM Administrator:** Cloud CLI needs to be +installed in the local machine for certain cloud related activities, +like creating an [uber principal](docs/reference/commands.md#create-uber-principal). +For this, the user needs Cloud IAM Administrator privileges. + +**Solution:** Work with a cloud administrator in your organization to +run the commands that need cloud administrator rights. + +Admin privileges required for commands: + +| **CLI command** | **Admin privileges** | +| ------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------- | +| [install](docs/installation/install_ucx.md) | Workspace Admin | +| [account install](docs/installation/collection.md) | Account Admin | +| [create-account-groups](docs/reference/commands.md#create-account-groups) | Account Admin | +| [validate-groups-membership](docs/reference/commands.md#validate-groups-membership) | Account Admin | +| [create-uber-principal](docs/reference/commands.md#create-uber-principal) | Cloud Admin | +| [principal-prefix-access](docs/reference/commands.md#principal-prefix-access) | Cloud Admin | +| [create-missing-principals](docs/reference/commands.md#create-missing-principals) | Cloud Admin | +| [delete-missing-principals](docs/reference/commands.md#delete-missing-principals) | Cloud Admin | +| [migrate-credentials](docs/reference/commands.md#migrate-credentials) | Cloud Admin, Account Admin / Metastore Admin / CREATE STORAGE CREDENTIAL privilege | +| [migrate-location](docs/reference/commands.md#migrate-locations) | Metastore Admin / CREATE EXTERNAL LOCATION privilege | +| [create-catalogs-schemas](docs/reference/commands.md#create-catalogs-schemas) | Metastore Admin / CREATE CATALOG privilege | +| [sync-workspace-info](docs/reference/commands.md#sync-workspace-info) | Account Admin | +| [manual-workspace-info](docs/reference/commands.md#manual-workspace-info) | Workspace Admin | + + + +### Version Issues + +**Python:** UCX needs Python version 3.10 or later. + +**Solution:** Check the current version using `python --version`. If the +version is lower than 3.10, upgrade the local Python version to 3.10 or +higher. + +**Databricks CLI:** Databricks CLI v0.213 or higher is needed. + +**Solution:** Check the current version with `databricks --version`. For +lower versions of CLI, +[update](https://docs.databricks.com/en/dev-tools/cli/install.html#update) +the Databricks CLI on the local machine. + +**UCX:** When you install UCX, you get the latest version. But since UCX +is being actively developed, new versions are released frequently. There +might be issues if you have run the assessment with a much earlier +version, and then trying to run the migration workflows with the latest +UCX version. + +**Solution:** Upgrade UCX, and rerun the assessment job before running +the migration workflows. For some reason, if you want to install a +specific version of UCX, you can do it using the command +`databricks labs install ucx@\`, for example, +`databricks labs install ucx@v0.21.0`. + + + +### Authentication Issues + +**Workspace Level:** If you are facing authentication issues while +setting up Databricks CLI, please refer to the +[Cryptic errors on authentication](docs/reference/troubleshooting.md#cryptic-errors-on-authentication) +section to resolve the common errors related to authentication, +profiles, and tokens. + +**Account Level:** Not only workspace, but account level authentication +is also needed for installing UCX. If you do not have an account +configured in .databrickscfg, you will get an error message +".databrickscfg does not contain account profiles; please create one +first". + +**Solution:** To authenticate with a Databricks account, consider using +one of the following authentication types: +- [OAuth machine-to-machine (M2M) authentication](https://docs.databricks.com/en/dev-tools/cli/authentication.html#m2m-auth) +- [OAuth user-to-machine (U2M) authentication](https://docs.databricks.com/en/dev-tools/cli/authentication.html#u2m-auth) +- [Basic authentication (legacy)](https://docs.databricks.com/en/dev-tools/cli/authentication.html#basic-auth) + + + +### Multiple Profiles in Databricks CLI + +**Workspace Level:** More than one workspace profile can be configured +in the .databrickscfg file. For example, you can have profiles set for +Dev and Prod workspaces. You want to install UCX only for the Prod +workspace. + +**Solution:** The Databricks CLI provides an option to select the +[profile](https://docs.databricks.com/en/dev-tools/cli/profiles.html) +using `--profile \` or `-p \`. You can +test that the correct workspace is getting selected by running any +Databricks CLI command. For example, you can run `databricks clusters +list -p prod` and check that the Prod clusters are being returned. Once +the profile is verified, you can run UCX install for that specific +profile: `databricks labs install ucx -p prod`. + +**Account Level:** Multiple account level profiles are set in the +.databrickscfg file. + +**Solution:** The installation command `databricks labs install ucx` +will provide an option to select one account profile. + + + +### Workspace has an external Hive Metastore (HMS) + +**External HMS connectivity from UCX clusters:** If the workspace has an +external HMS, the clusters running the UCX jobs need to have specific +configurations to connect to the external HMS. Otherwise, UCX assessment +will not be able to assess the tables on HMS. + +**Solution:** Use a cluster policy before installation to set the +required Spark config for connecting to the external HMS, or manually +edit the cluster post-installation to have the correct configurations. +Detailed steps can be found +[here](./external_hms_glue.md). + +**External HMS connectivity from UCX SQL warehouse:** UCX requires a SQL +warehouse to create tables, run queries, create and refresh dashboards. +If you already have a Pro or Serverless warehouse connected to the +external HMS, you can select the same warehouse for UCX. You will also +be given an option (during installation) to create a new warehouse for +UCX. If you have never used a warehouse before, the new warehouse +created might not have proper configuration set to connect to the +external HMS. + +**Solution:** Set Spark configuration for connecting to external HMS in +the Admin Settings of SQL warehouse. This will only be needed if the +admin settings do not have the configurations already set. For example, +add *spark.hadoop.javax.jdo.option.ConnectionURL \* +under Data Access Configuration of SQL Warehouse Admin Settings. + + + +### Verify the Installation + +Once the UCX command `databricks labs install ucx` has completed +successfully, the installation can be verified with the following steps: + +1. Go to the Databricks Catalog Explorer and check if a new schema for + ucx is available in Hive Metastore with all empty tables. + +2. Check that the UCX jobs are visible under Workflows. + +3. Run the assessment. This will start the UCX clusters, crawl through + the workspace, and display results in the UCX dashboards. In case of + external HMS, verify from the results that the assessment has + analyzed the external HMS tables. + + diff --git a/docs/content/docs/reference/dashboards.md b/docs/content/docs/reference/dashboards.md new file mode 100644 index 0000000000..b92e93e95a --- /dev/null +++ b/docs/content/docs/reference/dashboards.md @@ -0,0 +1,16 @@ +# Dashboards + +The dashboards visualize UCX's outcomes. Each dashboard has text widgets detailing the contents in more detail. Below an +overview with a short description is given. + +| Name | Description | +|-----------------------------------------------------------------------------------------------------------------|----------------------------------------------------| +| [Assessment \[Main\]](https://github.com/databrickslabs/ucx/blob/main/src/databricks/labs/ucx/queries/assessment/main/00_0__assessment_overview.md) | Assessment overview | +| [Assessment \[Estimates\]](https://github.com/databrickslabs/ucx/blob/main/src/databricks/labs/ucx/queries/assessment/estimates/00_0_metastore_assignment.md) | Migration effort estimates based on the assessment | +| [Assessment \[Interactive\]](https://github.com/databrickslabs/ucx/blob/main/src/databricks/labs/ucx/queries/assessment/interactive/00_0_interactive.md) | Assessment outcomes on interactive clusters | +| [Assessment \[Azure\]](https://github.com/databrickslabs/ucx/blob/main/src/databricks/labs/ucx/queries/assessment/azure/00_0_azure_service_principals.md) | Assessment outcomes specific to Azure | +| [Migration \[Main\]](https://github.com/databrickslabs/ucx/blob/main/src/databricks/labs/ucx/queries/migration/main/00_0_migration_overview.md) | Migration overview | +| [Migration \[Groups\]](https://github.com/databrickslabs/ucx/blob/main/src/databricks/labs/ucx/queries/migration/groups/00_0_migration_overview.md) | Group migration outcomes | +| [Progress \[Main\]](https://github.com/databrickslabs/ucx/blob/main/src/databricks/labs/ucx/queries/progress/main/00_0_migration_progress.md) | Migration progress | + + diff --git a/docs/external_hms_glue.md b/docs/content/docs/reference/external_hms_glue.md similarity index 86% rename from docs/external_hms_glue.md rename to docs/content/docs/reference/external_hms_glue.md index 69ca592cfc..2a7a4d4b09 100644 --- a/docs/external_hms_glue.md +++ b/docs/content/docs/reference/external_hms_glue.md @@ -1,19 +1,13 @@ -External Hive Metastore Integration -=== - - -* [External Hive Metastore Integration](#external-hive-metastore-integration) -* [Installation](#installation) -* [Manual Override](#manual-override) -* [Assessment Workflow](#assessment-workflow) -* [Table Migration Workflow](#table-migration-workflow) -* [Additional Considerations](#additional-considerations) - +--- +linkTitle: External Hive Metastore Integration +title: External Hive Metastore Integration +--- + UCX works with both the default workspace metastore, or an external Hive metastore. This document outlines the current integration and how to set up UCX to work with your existing external metastore. -# Installation +## Installation The setup process follows the following steps @@ -37,9 +31,9 @@ As UCX uses both job clusters and SQL Warehouses, it is important to ensure that external Hive metastore. If the SQL Warehouses are not configured for external Hive metastore, please manually update the data access configuration. See [Enable data access configuration](https://learn.microsoft.com/en-us/azure/databricks/admin/sql/data-access-configuration) for more details -[[back to top](#external-hive-metastore-integration)] -# Manual Override + +## Manual Override If the workspace does not have a cluster policy or SQL data access configuration for external Hive metastore, there are two options to manually enable this: @@ -66,9 +60,9 @@ two options to manually enable this: If UCX does not prompt you to set up with an external metastore, you can manually override the default behavior by following the post-installation steps above. -[[back to top](#external-hive-metastore-integration)] -# Assessment Workflow + +## Assessment Workflow Once UCX is set up with external Hive metastore the assessment workflow will scan tables & views from the external Hive metastore instead of the default workspace metastore. @@ -79,17 +73,17 @@ database name for each UCX installation. This is to avoid conflicts between the As the inventory database is stored in the external Hive metastore, it can only be queried from a cluster or SQL warehouse with external Hive metastore configuration. The assessment dashboard will also fail if the SQL warehouse is not configured correctly. -[[back to top](#external-hive-metastore-integration)] -# Table Migration Workflow + +## Table Migration Workflow Table migration workflow will upgrade tables & views from the external Hive metastore the Unity Catalog. This workflow only needs to be executed once per external Hive metastore. Running the workflow on other workspaces sharing the same metastore is redundant and will be a no-op. -[[back to top](#external-hive-metastore-integration)] -# Additional Considerations + +## Additional Considerations If a workspace is set up with multiple external Hive metastores, you will need to plan the approach carefully. Below are a few considerations to keep in mind: @@ -101,4 +95,4 @@ the SQL data access configuration, but it is a cleaner approach. - You can manually modify the cluster policy and SQL data access configuration to point to the correct external Hive metastore, after UCX has been installed. This is the most flexible approach, but requires manual intervention. -[[back to top](#external-hive-metastore-integration)] + diff --git a/docs/content/docs/reference/linter_message_codes.md b/docs/content/docs/reference/linter_message_codes.md new file mode 100644 index 0000000000..75f35430aa --- /dev/null +++ b/docs/content/docs/reference/linter_message_codes.md @@ -0,0 +1,265 @@ +# Linter message codes + +![code compatibility problems](/images/code_compatibility_problems.png) + +Here's the detailed explanation of the linter message codes: + +## `cannot-autofix-table-reference` + +This indicates that the linter has found a table reference that cannot be automatically fixed. The user must manually +update the table reference to point to the correct table in Unity Catalog. This mostly occurs, when table name is +computed dynamically, and it's too complex for our static code analysis to detect it. We detect this problem anywhere +where table name could be used: `spark.sql`, `spark.catalog.*`, `spark.table`, `df.write.*` and many more. Code examples +that trigger this problem: + +```python +spark.table(f"foo_{some_table_name}") +# .. +df = spark.range(10) +df.write.saveAsTable(f"foo_{some_table_name}") +# .. or even +df.write.insertInto(f"foo_{some_table_name}") +``` + +Here the `some_table_name` variable is not defined anywhere in the visible scope. Though, the analyser would +successfully detect table name if it is defined: + +```python +some_table_name = 'bar' +spark.table(f"foo_{some_table_name}") +``` + +We even detect string constants when coming either from `dbutils.widgets.get` (via job named parameters) or through +loop variables. If `old.things` table is migrated to `brand.new.stuff` in Unity Catalog, the following code will +trigger two messages: [`table-migrated-to-uc`](#table-migrated-to-uc) for the first query, as the contents are clearly +analysable, and `cannot-autofix-table-reference` for the second query. + +```python +# ucx[table-migrated-to-uc:+4:4:+4:20] Table old.things is migrated to brand.new.stuff in Unity Catalog +# ucx[cannot-autofix-table-reference:+3:4:+3:20] Can't migrate table_name argument in 'spark.sql(query)' because its value cannot be computed +table_name = f"table_{index}" +for query in ["SELECT * FROM old.things", f"SELECT * FROM {table_name}"]: + spark.sql(query).collect() +``` + + + +## `catalog-api-in-shared-clusters` + +`spark.catalog.*` functions require Databricks Runtime 14.3 LTS or above on Unity Catalog clusters in Shared access +mode, so of your code has `spark.catalog.tableExists("table")` or `spark.catalog.listDatabases()`, you need to ensure +that your cluster is running the correct runtime version and data security mode. + + + +## `changed-result-format-in-uc` + +Calls to these functions would return a list of `..
` instead of `.
`. So if +you have code like this: + +```python +for table in spark.catalog.listTables(): + do_stuff_with_table(table) +``` + +you need to make sure that `do_stuff_with_table` can handle the new format. + + + +## `direct-filesystem-access-in-sql-query` + +Direct filesystem access is deprecated in Unity Catalog. +DBFS is no longer supported, so if you have code like this: + +```python +df = spark.sql("SELECT * FROM parquet.`/mnt/foo/path/to/parquet.file`") +``` + +you need to change it to use UC tables. + + + +## `direct-filesystem-access` + +Direct filesystem access is deprecated in Unity Catalog. +DBFS is no longer supported, so if you have code like this: + +```python +display(spark.read.csv('/mnt/things/data.csv')) +``` + +or this: + +```python +display(spark.read.csv('s3://bucket/folder/data.csv')) +``` + +You need to change it to use UC tables or UC volumes. + + + +## `dependency-not-found` + +This message indicates that the linter has found a dependency, like Python source file or a notebook, that is not +available in the workspace. The user must ensure that the dependency is available in the workspace. This usually +means an error in the user code. + + + +## `jvm-access-in-shared-clusters` + +You cannot access Spark Driver JVM on Unity Catalog clusters in Shared Access mode. If you have code like this: + +```python +spark._jspark._jvm.com.my.custom.Name() +``` + +or like this: + +```python +log4jLogger = sc._jvm.org.apache.log4j +LOGGER = log4jLogger.LogManager.getLogger(__name__) +``` + +you need to change it to use Python equivalents. + + + +## `legacy-context-in-shared-clusters` + +SparkContext (`sc`) is not supported on Unity Catalog clusters in Shared access mode. Rewrite it using SparkSession +(`spark`). Example code that triggers this message: + +```python +df = spark.createDataFrame(sc.emptyRDD(), schema) +``` + +or this: + +```python +sc.parallelize([1, 2, 3]) +``` + + + +## `not-supported` + +Installing eggs is no longer supported on Databricks 14.0 or higher. + + + +## `notebook-run-cannot-compute-value` + +Path for `dbutils.notebook.run` cannot be computed and requires adjusting the notebook path. +It is not clear for automated code analysis where the notebook is located, so you need to simplify the code like: + +```python +b = some_function() +dbutils.notebook.run(b) +``` + +to something like this: + +```python +a = "./leaf1.py" +dbutils.notebook.run(a) +``` + + + +## `python-udf-in-shared-clusters` + +`applyInPandas` requires DBR 14.3 LTS or above on Unity Catalog clusters in Shared access mode. Example: + +```python +df.groupby("id").applyInPandas(subtract_mean, schema="id long, v double").show() +``` + +Arrow UDFs require DBR 14.3 LTS or above on Unity Catalog clusters in Shared access mode. + +```python +@udf(returnType='int', useArrow=True) +def arrow_slen(s): + return len(s) +``` + +It is not possible to register Java UDF from Python code on Unity Catalog clusters in Shared access mode. Use a +`%scala` cell to register the Scala UDF using `spark.udf.register`. Example code that triggers this message: + +```python +spark.udf.registerJavaFunction("func", "org.example.func", IntegerType()) +``` + + + +## `rdd-in-shared-clusters` + +RDD APIs are not supported on Unity Catalog clusters in Shared access mode. Use mapInArrow() or Pandas UDFs instead. + +```python +df.rdd.mapPartitions(myUdf) +``` + + + +## `spark-logging-in-shared-clusters` + +Cannot set Spark log level directly from code on Unity Catalog clusters in Shared access mode. Remove the call and set +the cluster spark conf `spark.log.level` instead: + +```python +sc.setLogLevel("INFO") +setLogLevel("WARN") +``` + +Another example could be: + +```python +log4jLogger = sc._jvm.org.apache.log4j +LOGGER = log4jLogger.LogManager.getLogger(__name__) +``` + +or + +```python +sc._jvm.org.apache.log4j.LogManager.getLogger(__name__).info("test") +``` + + + +## `sql-parse-error` + +This is a generic message indicating that the SQL query could not be parsed. The user must manually check the SQL query. + + + +## `sys-path-cannot-compute-value` + +Path for `sys.path.append` cannot be computed and requires adjusting the path. It is not clear for automated code +analysis where the path is located. + + + +## `table-migrated-to-uc` + +This message indicates that the linter has found a table that has been migrated to Unity Catalog. The user must ensure +that the table is available in Unity Catalog. + + + +## `to-json-in-shared-clusters` + +`toJson()` is not available on Unity Catalog clusters in Shared access mode. Use `toSafeJson()` on DBR 13.3 LTS or +above to get a subset of command context information. Example code that triggers this message: + +```python +dbutils.notebook.entry_point.getDbutils().notebook().getContext().toSafeJson() +``` + + + +## `unsupported-magic-line` + +This message indicates the code that could not be analysed by UCX. User must check the code manually. + + diff --git a/docs/migration-progress.md b/docs/content/docs/reference/migration-progress.md similarity index 94% rename from docs/migration-progress.md rename to docs/content/docs/reference/migration-progress.md index 5cce73a2ec..3bec9d4fc7 100644 --- a/docs/migration-progress.md +++ b/docs/content/docs/reference/migration-progress.md @@ -24,8 +24,8 @@ See the [resource index](#resource-index) for more details on the above objects. ## Usage -Use the migration progress through the [migration progress dashboard](../README.md#dashboards) after running the -[(experimental) migration progress workflow](../README.md#experimental-migration-progress-workflow). +Use the migration progress through the [migration progress dashboard](./dashboards.md) after running the +[(experimental) migration progress workflow](./workflows/progress.md). ### Failures @@ -41,7 +41,7 @@ more central in [Unity Catalog](https://docs.databricks.com/en/data-governance/u ## Tracking -The [(experimental) migration progress workflow](../README.md#experimental-migration-progress-workflow) tracks the +The [(experimental) migration progress workflow](./workflows/progress.md) tracks the migration progress and populates [migration progress tables](#persistence). ### Roll-up to business resources @@ -73,12 +73,12 @@ to be _dangling_ objects. For now, these objects are tracked separately, thus no ## Persistence -The progress is persisted in the [UCX UC catalog](../README.md#create-ucx-catalog-command) so that migration progress can be +The progress is persisted in the [UCX UC catalog](docs/reference/table_persistence.md) so that migration progress can be tracked cross-workspace. The catalog contains the tables below. ### Historical -The [`historical` table](../src/databricks/labs/ucx/progress/install.py) contains historical records of inventory +The [`historical` table](https://github.com/databrickslabs/ucx/blob/main/src/databricks/labs/ucx/progress/install.py) contains historical records of inventory objects relevant to the migration progress | Column | Data type | Description | @@ -100,7 +100,7 @@ Example historical record: ### Workflow run -The auxiliary [`workflow_runs` table](../src/databricks/labs/ucx/progress/install.py) tracks UCX workflow runs. +The auxiliary [`workflow_runs` table](https://github.com/databrickslabs/ucx/blob/main/src/databricks/labs/ucx/progress/install.py) tracks UCX workflow runs. | Column | Data type | Description | |----------------------|-------------|--------------------------------------------| diff --git a/docs/table_persistence.md b/docs/content/docs/reference/table_persistence.md similarity index 99% rename from docs/table_persistence.md rename to docs/content/docs/reference/table_persistence.md index 85fa36fcbb..9b3f5a2220 100644 --- a/docs/table_persistence.md +++ b/docs/content/docs/reference/table_persistence.md @@ -1,4 +1,8 @@ -# UCX objects +--- +title: Table Persistence +linkTitle: Table Persistence +--- + List of all UCX objects and their respective metadata. diff --git a/docs/troubleshooting.md b/docs/content/docs/reference/troubleshooting.md similarity index 83% rename from docs/troubleshooting.md rename to docs/content/docs/reference/troubleshooting.md index db85db5b0c..b3a06ff773 100644 --- a/docs/troubleshooting.md +++ b/docs/content/docs/reference/troubleshooting.md @@ -1,30 +1,10 @@ -# UCX Troubleshooting guide -This guide will help you troubleshoot potential issues running the UCX toolkit. +--- +title: Troubleshooting +linkTitle: Troubleshooting +weight: 4 +--- -* [UCX Troubleshooting guide](#ucx-troubleshooting-guide) - * [Common errors](#common-errors) - * [Locating error messages](#locating-error-messages) - * [Databricks workspace](#databricks-workspace) - * [Databricks jobs](#databricks-jobs) - * [Databricks notebooks](#databricks-notebooks) - * [UCX command Line](#ucx-command-line) - * [UCX Log Files](#ucx-log-files) - * [Accessing Databricks UCX Logs through the Databricks UI](#accessing-databricks-ucx-logs-through-the-databricks-ui) - * [Accessing Databricks UCX logs via CLI](#accessing-databricks-ucx-logs-via-cli) - * [Reading log files](#reading-log-files) - * [Getting more help](#getting-more-help) - * [UCX GitHub repository](#ucx-github-repository) - * [Databricks community](#databricks-community) - * [Databricks support](#databricks-support) - * [Databricks partners](#databricks-partners) - * [Resolving common UCX errors](#resolving-common-ucx-errors) - * [Cryptic errors on authentication](#cryptic-errors-on-authentication) - * [Resolving common errors on UCX install](#resolving-common-errors-on-ucx-install) - * [Error on installing the ucx inventory database](#error-on-installing-the-ucx-inventory-database) - * [Error installing a wheel file](#error-installing-a-wheel-file) - * [Error running the assessment job](#error-running-the-assessment-job) - * [Specific assessment job tasks fail.](#specific-assessment-job-tasks-fail) - * [Resolving other common errors](#resolving-other-common-errors) +This guide will help you troubleshoot potential issues running the UCX toolkit. ## Common errors Errors may occur during: @@ -168,5 +148,5 @@ See the gathering log information sections elsewhere in this document. ### Resolving other common errors - If you have an external Hive Metastore (HMS) such as Glue Catalog or a MySQL, Postgres or SQL server database, please consult the [External Hive Metastore Integration guide](external_hms_glue.md) -- If you are running table upgrade commands and workflows. Please consult the [Table Upgrade guide](table_upgrade.md) -- If you are trying to understand the Assessment report, please consult the [Assessment documentation](assessment.md) +- If you are running table upgrade commands and workflows. Please consult the [Table Upgrade guide](docs/process/table_migration.md) +- If you are trying to understand the Assessment report, please consult the [Assessment documentation](./assessment.md) diff --git a/docs/content/docs/reference/workflows/_index.md b/docs/content/docs/reference/workflows/_index.md new file mode 100644 index 0000000000..a46b4484fb --- /dev/null +++ b/docs/content/docs/reference/workflows/_index.md @@ -0,0 +1,9 @@ +--- +title: Workflows +linkTitle: Workflows +weight: 2 +--- + +Part of this application is deployed as [Databricks Workflows](https://docs.databricks.com/en/workflows/index.html). +You can view the status of deployed workflows via the [`workflows` command](docs/reference/commands.md#workflows). +Failed workflows can be fixed with the [`repair-run` command](docs/reference/commands.md##repair-run). diff --git a/docs/content/docs/reference/workflows/assessment.md b/docs/content/docs/reference/workflows/assessment.md new file mode 100644 index 0000000000..d9c3be95fa --- /dev/null +++ b/docs/content/docs/reference/workflows/assessment.md @@ -0,0 +1,85 @@ +## Assessment workflow + +![ucx_assessment_workflow](/images/ucx_assessment_workflow.png) + +The assessment workflow can be triggered using the Databricks UI or via the +[`ensure-assessment-run` command](docs/reference/commands.md#ensure-assessment-run). + +The assessment workflow retrieves - or *crawls* - details of [workspace assets](https://docs.databricks.com/en/workspace/workspace-assets.html) +and [securable objects in the Hive metastore](https://docs.databricks.com/en/data-governance/table-acls/object-privileges.html#securable-objects-in-the-hive-metastore) +relevant for upgrading to UC to assess the compatibility with UC. The `crawl_` tasks retrieve assess and objects. The +`assess_` tasks assess the compatibility with UC. The output of each task is stored in the +[inventory database](docs/installation/resources.md) so that it can be used for further analysis and decision-making through +the [assessment report](docs/reference/assessment.md). + +1. `crawl_tables`: This task retrieves table definitions from the Hive metastore and persists the definitions in + the `tables` table. The definitions include information such as: + - Database/schema name + - Table name + - Table type + - Table location +2. `crawl_udfs`: This task retrieves UDF definitions from the Hive metastore and persists the definitions in + the `udfs` table. +3. `setup_tacl`: (Optimization) This task starts the `tacl` job cluster in parallel to other tasks. +4. `crawl_grants`: This task retrieves [privileges you can grant on Hive objects](https://docs.databricks.com/en/data-governance/table-acls/object-privileges.html#privileges-you-can-grant-on-hive-metastore-objects) + and persists the privilege definitions in the `grants` table + The retrieved permission definitions include information such as: + - Securable object: schema, table, view, (anonymous) function or any file. + - Principal: user, service principal or group + - Action type: grant, revoke or deny +5. `estimate_table_size_for_migration`: This task analyzes the Delta table retrieved by `crawl_tables` to retrieve + an estimate of their size and persists the table sizes in the `table_size` table. The table size support the decision + for using the `SYNC` or `CLONE` [table migration strategy](#step-3-upgrade-the-metastore). +6. `crawl_mounts`: This task retrieves mount point definitions and persists the definitions in the `mounts` table. +7. `guess_external_locations`: This task guesses shared mount path prefixes of [external tables](https://docs.databricks.com/en/sql/language-manual/sql-ref-external-tables.html) + retrieved by `crawl_tables` that use mount points and persists the locations in the `external_locations` table. The + goal is to identify the to-be created UC [external locations](#step-23-create-external-locations). +8. `assess_jobs`: This task retrieves the job definitions and persists the definitions in the `jobs` table. Job + definitions may require updating to become UC compatible. +9. `assess_clusters`: This task retrieves the clusters definitions and persists the definitions in the `clusters` table. + Cluster definitions may require updating to become UC compatible. +10. `assess_pipelines`: This task retrieves the [Delta Live Tables](https://www.databricks.com/product/delta-live-tables) + (DLT) pipelines definitions and persists the definitions in the `pipelines` table. DLT definitions may require + updating to become UC compatible. +11. `assess_incompatible_submit_runs` : This task retrieves + [job runs](https://docs.databricks.com/en/jobs/monitor.html#view-runs-for-a-job), also known as + [job submit runs](https://docs.databricks.com/api/workspace/jobs/submit), and persists the definitions in the + `submit_runs` table. Incompatibility with UC is assessed: + - [Databricks runtime](https://www.databricks.com/glossary/what-is-databricks-runtime) should be version 11.3 or above + - [Access mode](https://docs.databricks.com/en/compute/configure.html#access-modes) should be set. +12. `crawl_cluster_policies` : This tasks retrieves + [cluster policies](https://docs.databricks.com/en/admin/clusters/policies.html) and persists the policies in the + `policies` table. Incompatibility with UC is assessed: + - [Databricks runtime](https://www.databricks.com/glossary/what-is-databricks-runtime) should be version 11.3 or above +13. `assess_azure_service_principals`: This tasks retrieves Azure service principal + [authentications information](https://learn.microsoft.com/en-us/azure/databricks/connect/storage/tutorial-azure-storage#connect-adls) + from Spark configurations to access storage accounts and persists these configuration information + in `azure_service_principals` table. The Spark configurations from the following places are retrieved: + - Clusters configurations + - Cluster policies + - Job cluster configurations + - Pipeline configurations + - Warehouse configuration +14. `assess_global_init_scripts`: This task retrieves + [global init scripts](https://docs.databricks.com/en/init-scripts/global.html) and persists references to the + scripts in the `global_init_scripts` table. Again, Azure service principal authentication information might be given + in those scripts. +15. `workspace_listing` : This tasks lists [workspace files](https://docs.databricks.com/en/workspace/workspace-assets.html#files) + recursively to compile a collection of directories, notebooks, files, repos and libraries. The task uses multi-threading + to parallelize the listing process for speeding up execution on big workspaces. +16. `crawl_permissions` : This tasks retrieves workspace-local groups permissions and persists these permissions in the `permissions` table. +17. `crawl_groups`: This tasks retrieves workspace-local groups and persists these permissions in the `groups` table. +18. `assess_dashboards`: This task retrieves the dashboards to analyze their queries for + [migration problems](#linter-message-codes) +19. `assess_workflows`: This task retrieves the jobs to analyze their notebooks and files for + [migration problems](#linter-message-codes). + +After UCX assessment workflow finished, see the assessment dashboard for findings and recommendations. +See [this guide](docs/reference/assessment.md) for more details. + +![report](/images/assessment-report.png) + +Proceed to the [group migration workflow](#group-migration-workflow) below or go back to the +[migration process diagram](#migration-process). + + diff --git a/docs/content/docs/reference/workflows/group_migration.md b/docs/content/docs/reference/workflows/group_migration.md new file mode 100644 index 0000000000..5b46a55f2c --- /dev/null +++ b/docs/content/docs/reference/workflows/group_migration.md @@ -0,0 +1,107 @@ +--- +title: "Group Migration Workflow" +linkTitle: "Group Migration" +--- + +{{}} +You are required to complete the [assessment workflow](docs/reference/workflows/assessment.md) before starting the table migration workflow. +{{}} + +## Pre-read + +Please check [this document](https://docs.databricks.com/en/admin/users-groups/groups.html#difference-between-account-groups-and-workspace-local-groups) to understand the difference between account-level and workspace-level groups. + +The main outcome from the document above is that workspace-level groups cannot be used to manage permissions on Unity Catalog objects. This is because the Unity Catalog is a shared resource across all workspaces in the account, and workspace-level groups are not shared across workspaces. + +Therefore, to simplify permission management (both on Unity Catalog objects and other workspace objects), we recommend using account-level groups. This document describes the workflow that is used to migrate workspace-level groups to account-level groups. + +## Account group creation + +{{}} +The group migration *workflow* does NOT CREATE account groups. + +We recommend using [identity federation](https://docs.databricks.com/en/admin/users-groups/index.html#enable-identity-federation) to centrally provision groups from your identity provider. + +Alternatively, you can create groups by using the [`create-account-groups` command](docs/reference/commands.md#create-account-groups) before running the group migration workflow. + +{{}} + +## The workflow + +The group migration workflow is designed to migrate workspace-local groups to account-level groups. + + +It verifies if the necessary groups are available to the workspace with the correct permissions, and removes unnecessary groups and permissions. The group migration workflow depends on the output of the **assessment workflow**, thus, should only be +executed after a successful run of the assessment workflow. + +{{}} +The group migration workflow may be executed multiple times. If you accidentially created new workspace-local groups, you can run the group migration workflow again to migrate the new groups. +{{}} + +Workflow consists of the following tasks: + +{{% steps %}} + +### Verify Metastore Attached +Verifies if a metastore is attached. Account level groups are only available when +a metastore is attached. [See `assign-metastore` command.](docs/reference/commands.md#assign-metastore) + +### Rename Workspace Local Groups +This task renames workspace-local groups by adding a `ucx-renamed-` prefix. This +step is taken to avoid conflicts with account groups that may have the same name as workspace-local groups. + +### Reflect Account Groups on Workspace +This task adds matching account groups to this workspace. The matching account groups must exist for this step to be successful. This step is necessary to ensure that the account groups are available in the workspace for assigning permissions. + +### Apply Permissions to Account Groups +This task assigns the full set of permissions of the original group to the account-level one. This step is necessary to ensure that the account-level groups have the necessary permissions to manage the entities in the workspace. + +{{% details title="Click here to see all workspace-level entities" closed="true" %}} +- Legacy Table ACLs +- Entitlements +- AWS instance profiles +- Clusters +- Cluster policies +- Instance Pools +- Databricks SQL warehouses +- Delta Live Tables +- Jobs +- MLflow experiments +- MLflow registry +- SQL Dashboards & Queries +- SQL Alerts +- Token and Password usage permissions +- Secret Scopes +- Notebooks +- Directories +- Repos +- Files +{{% /details %}} + +### Validate Groups Permissions +This task validates that all the crawled permissions are applied correctly to the destination groups. + +{{% /steps %}} + +You can trigger the workflow from the Workspace UI. + +## Post-migration steps + +After successfully running the group migration workflow: +1. Use [`validate-groups-membership` command](docs/reference/commands.md#validate-groups-membership) for extra confidence the newly created + account level groups are considered to be valid. +2. Run the [`remove-workspace-local-backup-grups`](docs/reference/commands.md#validate-groups-membership) to remove workspace-level backup + groups, along with their permissions. This should only be executed after confirming that the workspace-local + migration worked successfully for all the groups involved. This step is necessary to clean up the workspace and + remove any unnecessary groups and permissions. +3. Proceed to the [table migration process](docs/process/table_migration.md). + + +## Additional resources + +For more information, see: +- The [detailed design](docs/dev/implementation/local_group_migration.md) of thie group migration workflow. +- The [migration process diagram](docs/process/overview.md#diagram) showing the group migration workflow in context of the whole + migration process. + + diff --git a/docs/content/docs/reference/workflows/progress.md b/docs/content/docs/reference/workflows/progress.md new file mode 100644 index 0000000000..81ca16920a --- /dev/null +++ b/docs/content/docs/reference/workflows/progress.md @@ -0,0 +1,14 @@ +## [EXPERIMENTAL] Migration Progress Workflow + +The `migration-progress-experimental` workflow updates a subset of the inventory tables to track migration status of +workspace resources that need to be migrated. Besides updating the inventory tables, this workflow tracks the migration +progress by updating the following [UCX catalog](docs/reference/commands.md#create-ucx-catalog) tables: + +- `workflow_runs`: Tracks the status of the workflow runs. + +{{< callout type="info">}} +A subset of the inventory is updated, *not* the complete inventory that is initially gathered by +the [assessment workflow](docs/reference/workflows/assessment.md). +{{< /callout >}} + + diff --git a/docs/content/docs/reference/workflows/reconciliation.md b/docs/content/docs/reference/workflows/reconciliation.md new file mode 100644 index 0000000000..e3379d622f --- /dev/null +++ b/docs/content/docs/reference/workflows/reconciliation.md @@ -0,0 +1,41 @@ +## Post-migration data reconciliation workflow + +The `migrate-data-reconciliation` workflow validates the integrity of the migrated tables and persists its results in +the `recon_results` table. The workflow compares the following between the migrated Hive metastore and its UC +counterpart table: +- `Schema` : See this result in the `schema_matches` column. +- `Column by column` : See this result in the `column_comparison` column. +- `Row counts` : If the row count is within the reconciliation threshold (defaults to 5%), the `data_matches` column is + set to `true`, otherwise it is set to `false`. +- `Rows` : If the `compare_rows` flag is set to `true`, rows are compared using a hash comparison. Number of missing + rows are stored in the `source_missing_count` and `target_missing_count` column, respectively. + +The output is processed and displayed in the migration dashboard using the in `reconciliation_results` view. + +![reconciliation results](/images/recon_results.png) + +### [LEGACY] Scan tables in mounts Workflow + +{{< callout type="warning" >}} +Always run this workflow AFTER the assessment has finished. +{{< /callout >}} + +- This experimental workflow attempts to find all Tables inside mount points that are present on your workspace. +- If you do not run this workflow, then `migrate-tables-in-mounts-experimental` won't do anything. +- It writes all results to `hive_metastore..tables`, you can query those tables found by filtering on database values that starts with `mounted_` +- This command is incremental, meaning that each time you run it, it will overwrite the previous tables in mounts found. +- Current format are supported: + - DELTA - PARQUET - CSV - JSON + - Also detects partitioned DELTA and PARQUET +- You can configure these workflows with the following options available on conf.yml: + - include_mounts : A list of mount points to scans, by default the workflow scans for all mount points + - exclude_paths_in_mount : A list of paths to exclude in all mount points + - include_paths_in_mount : A list of paths to include in all mount points + +### [LEGACY] Migrate tables in mounts Workflow +- An experimental workflow that migrates tables in mount points using a `CREATE TABLE` command, optinally sets a default tables owner if provided in `default_table_owner` conf parameter. +- You must do the following in order to make this work: + - run the Assessment [workflow](docs/reference/workflows/assessment.md) + - run the scan tables in mounts [workflow](#legacy-scan-tables-in-mounts-workflow) + - run the [`create-table-mapping` command](docs/reference/commands.md#create-table-mapping) + - or manually create a `mapping.csv` file in Workspace -> Applications -> ucx diff --git a/docs/content/docs/reference/workflows/table_migration.md b/docs/content/docs/reference/workflows/table_migration.md new file mode 100644 index 0000000000..649e9de626 --- /dev/null +++ b/docs/content/docs/reference/workflows/table_migration.md @@ -0,0 +1,38 @@ +## Table migration workflows + +This section lists the workflows that migrate tables and views. See [this section](#migrate-hive-metastore-data-objects) +for deciding which workflow to run and additional context for migrating tables. + +### Migrate tables + +The general table migration workflow `migrate-tables` migrates all tables and views using default strategies. + +1. `migrate_external_tables_sync` : This step migrates the external tables that are supported by `SYNC` command. +2. `migrate_dbfs_root_delta_tables` : This step migrates delta tables stored in DBFS root using the `DEEP CLONE` + command. +3. `migrate_dbfs_root_non_delta_tables` : This step migrates non-delta tables stored in DBFS root using the `CREATE + TABLE AS SELECT * FROM` command. +4. `migrate_views` : This step migrates views using the `CREATE VIEW` command. +5. `update_migration_status` : Refresh the migration status of all data objects. + +### Migrate external Hive SerDe tables + +The experimental table migration workflow `migrate-external-hiveserde-tables-in-place-experimental` migrates tables that +support the `SYNC AS EXTERNAL` command. + +1. `migrate_hive_serde_in_place` : This step migrates the Hive SerDe tables that are supported by `SYNC AS EXTERNAL` + command. +2. `migrate_views` : This step migrates views using the `CREATE VIEW` command. +3. `update_migration_status` : Refresh the migration status of all data objects. + +### Migrate external tables CTAS + +The table migration workflow `migrate-external-tables-ctas` migrates tables with the `CREATE TABLE AS SELECT * FROM` +command. + +1. `migrate_other_external_ctas` This step migrates the Hive Serde tables using the `CREATE TABLE AS SELECT * FROM` + command. +2. `migrate_hive_serde_ctas` : This step migrates the Hive Serde tables using the `CREATE TABLE AS SELECT * FROM` + command. +4. `migrate_views` : This step migrates views using the `CREATE VIEW` command. +4. `update_migration_status` : Refresh the migration status of all data objects. diff --git a/docs/go.mod b/docs/go.mod new file mode 100644 index 0000000000..91cc1de4a0 --- /dev/null +++ b/docs/go.mod @@ -0,0 +1,5 @@ +module github.com/databrickslabs/ucx + +go 1.23.1 + +require github.com/imfing/hextra v0.8.6 // indirect diff --git a/docs/go.sum b/docs/go.sum new file mode 100644 index 0000000000..8cc7ebb90c --- /dev/null +++ b/docs/go.sum @@ -0,0 +1,2 @@ +github.com/imfing/hextra v0.8.6 h1:fpOqzcUs26Lc/ZzowYSBcnpe00d/aZw4HhiHP7ycSks= +github.com/imfing/hextra v0.8.6/go.mod h1:cEfel3lU/bSx7lTE/+uuR4GJaphyOyiwNR3PTqFTXpI= diff --git a/docs/hugo.yaml b/docs/hugo.yaml new file mode 100644 index 0000000000..cb45b17f90 --- /dev/null +++ b/docs/hugo.yaml @@ -0,0 +1,75 @@ +baseURL: https://databrickslabs.github.io/ucx +languageCode: en-us +title: UCX + +module: + imports: + - path: github.com/imfing/hextra + +# https://gohugo.io/functions/css/tailwindcss/ +build: + cachebusters: + - source: layouts/.* + target: css + +params: + + editURL: + enable: true + base: "https://github.com/renardeinside/ucx/edit/main/docs/content" + + theme: + default: dark + displayToggle: true + + footer: + enable: true + displayCopyright: true + displayPoweredBy: false + + search: + enable: true + type: flexsearch + + flexsearch: + # index page by: content | summary | heading | title + index: content + + navbar: + displayTitle: true + displayLogo: true + logo: + path: logo.svg + dark: logo.svg + link: /ucx + width: 40 + height: 20 + +menu: + main: + - name: Overview + pageRef: /docs + weight: 1 + - name: Installation + pageRef: /docs/installation + weight: 2 + - name: Process + pageRef: /docs/process + weight: 2 + - name: Reference + pageRef: /docs/reference + weight: 3 + - name: Search + weight: 4 + params: + type: search + - name: GitHub + weight: 5 + url: "https://github.com/databrickslabs/ucx" + params: + icon: github + +markup: + goldmark: + renderer: + unsafe: true \ No newline at end of file diff --git a/docs/i18n/en.yaml b/docs/i18n/en.yaml new file mode 100644 index 0000000000..fc7d8a055f --- /dev/null +++ b/docs/i18n/en.yaml @@ -0,0 +1 @@ +copyright: "© 2024 Databricks" \ No newline at end of file diff --git a/docs/layouts/_default/_markup/render-link.html b/docs/layouts/_default/_markup/render-link.html new file mode 100644 index 0000000000..027c061504 --- /dev/null +++ b/docs/layouts/_default/_markup/render-link.html @@ -0,0 +1,51 @@ +{{- $attrs := dict }} +{{- $u := urls.Parse .Destination }} + +{{- if $u.IsAbs }} + {{- /* Remote */}} + {{- $attrs = dict "href" $u.String "rel" "external" }} +{{- else }} + {{- with $u.Path }} + {{- with $.Page.GetPage . }} + {{- /* Page */}} + {{- $href := .RelPermalink }} + {{- with $u.RawQuery }} + {{- $href = printf "%s?%s" $href $u.RawQuery }} + {{- end }} + {{- with $u.Fragment }} + {{- $href = printf "%s#%s" $href $u.Fragment }} + {{- end }} + {{- $attrs = dict "href" $href }} + {{- else }} + {{- with $.Page.Resources.GetMatch $u.Path }} + {{- /* Page resource; drop query and fragment */}} + {{- $attrs = dict "href" .RelPermalink }} + {{- else }} + {{- with resources.Get $u.Path }} + {{- /* Global resource; drop query and fragment */}} + {{- $attrs = dict "href" .RelPermalink }} + {{- else }} + {{- warnf "Unable to resolve reference to %s" $u.Path }} + {{- end }} + {{- end }} + {{- end }} + {{- else }} + {{- with $u.Fragment }} + {{- /* Fragment only; prepend page's relative permalink */}} + {{- $attrs = dict "href" (printf "%s#%s" $.Page.RelPermalink .) }} + {{- else }} + {{- errorf "Unable to resolve reference to %s from %s" $u.Path $.Page.File.Path }} + {{- end }} + {{- end }} +{{- end }} + +{{- with .Title }} + {{- $attrs = merge $attrs (dict "title" .) }} +{{- end -}} + +{{ .Text | safe.HTML }} +{{- /**/ -}} \ No newline at end of file diff --git a/docs/layouts/hextra-home.html b/docs/layouts/hextra-home.html new file mode 100644 index 0000000000..9558cfeb5f --- /dev/null +++ b/docs/layouts/hextra-home.html @@ -0,0 +1,14 @@ +{{ define "main" }} + +
+
+ {{ partial "sidebar.html" (dict "context" . "disableSidebar" true) }} +
+
+ {{ partial "custom/hero.html" . }} +
+
+
+
+{{ end }} \ No newline at end of file diff --git a/docs/layouts/partials/custom/head-end.html b/docs/layouts/partials/custom/head-end.html new file mode 100644 index 0000000000..a9f42a42ac --- /dev/null +++ b/docs/layouts/partials/custom/head-end.html @@ -0,0 +1,22 @@ +{{ $t := debug.Timer "tailwindcss" }} + {{ with resources.Get "css/tailwind.css" }} + {{ $opts := dict + "inlineImports" true + "optimize" (not hugo.IsDevelopment) + }} + {{ with . | css.TailwindCSS $opts }} + {{ if hugo.IsDevelopment }} + + {{ else }} + {{ with . | minify | fingerprint }} + + {{ end }} + {{ end }} + {{ end }} + {{ end }} +{{ $t.Stop }} + diff --git a/docs/layouts/partials/custom/hero.html b/docs/layouts/partials/custom/hero.html new file mode 100644 index 0000000000..c67012602b --- /dev/null +++ b/docs/layouts/partials/custom/hero.html @@ -0,0 +1,29 @@ +
+ +
+

UCX - Unity Catalog + Migration Assistant

+

+ Ready-to-use toolkit to migrate to UC, provided by Databricks Labs +

+
+ + Installation + + + Process + + + Reference + +
+
+ + + +
diff --git a/docs/package.json b/docs/package.json new file mode 100644 index 0000000000..5c3da28b76 --- /dev/null +++ b/docs/package.json @@ -0,0 +1,12 @@ +{ + "name": "ucx", + "version": "1.0.0", + "main": "index.js", + "license": "Databricks", + "dependencies": { + "@tailwindcss/cli": "^4.0.0-beta.5", + "linkinator": "^6.1.2", + "tailwindcss": "^4.0.0-beta.5" + }, + "devDependencies": {} +} diff --git a/docs/migration_config.yml b/docs/static/config/migration_config.yml similarity index 100% rename from docs/migration_config.yml rename to docs/static/config/migration_config.yml diff --git a/docs/static/favicon-dark.svg b/docs/static/favicon-dark.svg new file mode 100644 index 0000000000..8f9a7567ad --- /dev/null +++ b/docs/static/favicon-dark.svg @@ -0,0 +1,12 @@ + + + + + + + \ No newline at end of file diff --git a/docs/static/favicon.ico b/docs/static/favicon.ico new file mode 100644 index 0000000000..db0e294c2a Binary files /dev/null and b/docs/static/favicon.ico differ diff --git a/docs/static/favicon.svg b/docs/static/favicon.svg new file mode 100644 index 0000000000..8f9a7567ad --- /dev/null +++ b/docs/static/favicon.svg @@ -0,0 +1,12 @@ + + + + + + + \ No newline at end of file diff --git a/docs/assessment-report.png b/docs/static/images/assessment-report.png similarity index 100% rename from docs/assessment-report.png rename to docs/static/images/assessment-report.png diff --git a/docs/code_compatibility_problems.png b/docs/static/images/code_compatibility_problems.png similarity index 100% rename from docs/code_compatibility_problems.png rename to docs/static/images/code_compatibility_problems.png diff --git a/docs/debug-logs.png b/docs/static/images/debug-logs.png similarity index 100% rename from docs/debug-logs.png rename to docs/static/images/debug-logs.png diff --git a/docs/debug-notebook.png b/docs/static/images/debug-notebook.png similarity index 100% rename from docs/debug-notebook.png rename to docs/static/images/debug-notebook.png diff --git a/docs/debugging-tests.gif b/docs/static/images/debugging-tests.gif similarity index 100% rename from docs/debugging-tests.gif rename to docs/static/images/debugging-tests.gif diff --git a/docs/hatch-intellij.gif b/docs/static/images/hatch-intellij.gif similarity index 100% rename from docs/hatch-intellij.gif rename to docs/static/images/hatch-intellij.gif diff --git a/docs/lint-local-code-output.png b/docs/static/images/lint-local-code-output.png similarity index 100% rename from docs/lint-local-code-output.png rename to docs/static/images/lint-local-code-output.png diff --git a/docs/logo-no-background.png b/docs/static/images/logo-no-background.png similarity index 100% rename from docs/logo-no-background.png rename to docs/static/images/logo-no-background.png diff --git a/docs/macos_1_databrickslabsmac_installdatabricks.gif b/docs/static/images/macos_1_databrickslabsmac_installdatabricks.gif similarity index 100% rename from docs/macos_1_databrickslabsmac_installdatabricks.gif rename to docs/static/images/macos_1_databrickslabsmac_installdatabricks.gif diff --git a/docs/macos_2_databrickslabsmac_installucx.gif b/docs/static/images/macos_2_databrickslabsmac_installucx.gif similarity index 100% rename from docs/macos_2_databrickslabsmac_installucx.gif rename to docs/static/images/macos_2_databrickslabsmac_installucx.gif diff --git a/docs/macos_3_databrickslabsmac_upgradeucx.gif b/docs/static/images/macos_3_databrickslabsmac_upgradeucx.gif similarity index 100% rename from docs/macos_3_databrickslabsmac_upgradeucx.gif rename to docs/static/images/macos_3_databrickslabsmac_upgradeucx.gif diff --git a/docs/macos_4_databrickslabsmac_uninstallucx.gif b/docs/static/images/macos_4_databrickslabsmac_uninstallucx.gif similarity index 100% rename from docs/macos_4_databrickslabsmac_uninstallucx.gif rename to docs/static/images/macos_4_databrickslabsmac_uninstallucx.gif diff --git a/docs/migration-report.png b/docs/static/images/migration-report.png similarity index 100% rename from docs/migration-report.png rename to docs/static/images/migration-report.png diff --git a/docs/readme-notebook.png b/docs/static/images/readme-notebook.png similarity index 100% rename from docs/readme-notebook.png rename to docs/static/images/readme-notebook.png diff --git a/docs/recon_results.png b/docs/static/images/recon_results.png similarity index 100% rename from docs/recon_results.png rename to docs/static/images/recon_results.png diff --git a/docs/ucx_assessment_workflow.png b/docs/static/images/ucx_assessment_workflow.png similarity index 100% rename from docs/ucx_assessment_workflow.png rename to docs/static/images/ucx_assessment_workflow.png diff --git a/docs/windows_install_databricks.png b/docs/static/images/windows_install_databricks.png similarity index 100% rename from docs/windows_install_databricks.png rename to docs/static/images/windows_install_databricks.png diff --git a/docs/static/logo.svg b/docs/static/logo.svg new file mode 100644 index 0000000000..8f9a7567ad --- /dev/null +++ b/docs/static/logo.svg @@ -0,0 +1,12 @@ + + + + + + + \ No newline at end of file diff --git a/docs/tailwind.config.js b/docs/tailwind.config.js new file mode 100644 index 0000000000..67024cdaaa --- /dev/null +++ b/docs/tailwind.config.js @@ -0,0 +1,8 @@ +/* +This file is present to satisy a requiremenet of the Tailwind CSS IntelliSense +extension for Visual Studio Code. + +https://marketplace.visualstudio.com/items?itemName=bradlc.vscode-tailwindcss + +The rest of this file is intentionally empty. +*/ \ No newline at end of file diff --git a/docs/yarn.lock b/docs/yarn.lock new file mode 100644 index 0000000000..fc9f6e1a33 --- /dev/null +++ b/docs/yarn.lock @@ -0,0 +1,779 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@isaacs/cliui@^8.0.2": + version "8.0.2" + resolved "https://registry.yarnpkg.com/@isaacs/cliui/-/cliui-8.0.2.tgz#b37667b7bc181c168782259bab42474fbf52b550" + integrity sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA== + dependencies: + string-width "^5.1.2" + string-width-cjs "npm:string-width@^4.2.0" + strip-ansi "^7.0.1" + strip-ansi-cjs "npm:strip-ansi@^6.0.1" + wrap-ansi "^8.1.0" + wrap-ansi-cjs "npm:wrap-ansi@^7.0.0" + +"@parcel/watcher-android-arm64@2.5.0": + version "2.5.0" + resolved "https://registry.yarnpkg.com/@parcel/watcher-android-arm64/-/watcher-android-arm64-2.5.0.tgz#e32d3dda6647791ee930556aee206fcd5ea0fb7a" + integrity sha512-qlX4eS28bUcQCdribHkg/herLe+0A9RyYC+mm2PXpncit8z5b3nSqGVzMNR3CmtAOgRutiZ02eIJJgP/b1iEFQ== + +"@parcel/watcher-darwin-arm64@2.5.0": + version "2.5.0" + resolved "https://registry.yarnpkg.com/@parcel/watcher-darwin-arm64/-/watcher-darwin-arm64-2.5.0.tgz#0d9e680b7e9ec1c8f54944f1b945aa8755afb12f" + integrity sha512-hyZ3TANnzGfLpRA2s/4U1kbw2ZI4qGxaRJbBH2DCSREFfubMswheh8TeiC1sGZ3z2jUf3s37P0BBlrD3sjVTUw== + +"@parcel/watcher-darwin-x64@2.5.0": + version "2.5.0" + resolved "https://registry.yarnpkg.com/@parcel/watcher-darwin-x64/-/watcher-darwin-x64-2.5.0.tgz#f9f1d5ce9d5878d344f14ef1856b7a830c59d1bb" + integrity sha512-9rhlwd78saKf18fT869/poydQK8YqlU26TMiNg7AIu7eBp9adqbJZqmdFOsbZ5cnLp5XvRo9wcFmNHgHdWaGYA== + +"@parcel/watcher-freebsd-x64@2.5.0": + version "2.5.0" + resolved "https://registry.yarnpkg.com/@parcel/watcher-freebsd-x64/-/watcher-freebsd-x64-2.5.0.tgz#2b77f0c82d19e84ff4c21de6da7f7d096b1a7e82" + integrity sha512-syvfhZzyM8kErg3VF0xpV8dixJ+RzbUaaGaeb7uDuz0D3FK97/mZ5AJQ3XNnDsXX7KkFNtyQyFrXZzQIcN49Tw== + +"@parcel/watcher-linux-arm-glibc@2.5.0": + version "2.5.0" + resolved "https://registry.yarnpkg.com/@parcel/watcher-linux-arm-glibc/-/watcher-linux-arm-glibc-2.5.0.tgz#92ed322c56dbafa3d2545dcf2803334aee131e42" + integrity sha512-0VQY1K35DQET3dVYWpOaPFecqOT9dbuCfzjxoQyif1Wc574t3kOSkKevULddcR9znz1TcklCE7Ht6NIxjvTqLA== + +"@parcel/watcher-linux-arm-musl@2.5.0": + version "2.5.0" + resolved "https://registry.yarnpkg.com/@parcel/watcher-linux-arm-musl/-/watcher-linux-arm-musl-2.5.0.tgz#cd48e9bfde0cdbbd2ecd9accfc52967e22f849a4" + integrity sha512-6uHywSIzz8+vi2lAzFeltnYbdHsDm3iIB57d4g5oaB9vKwjb6N6dRIgZMujw4nm5r6v9/BQH0noq6DzHrqr2pA== + +"@parcel/watcher-linux-arm64-glibc@2.5.0": + version "2.5.0" + resolved "https://registry.yarnpkg.com/@parcel/watcher-linux-arm64-glibc/-/watcher-linux-arm64-glibc-2.5.0.tgz#7b81f6d5a442bb89fbabaf6c13573e94a46feb03" + integrity sha512-BfNjXwZKxBy4WibDb/LDCriWSKLz+jJRL3cM/DllnHH5QUyoiUNEp3GmL80ZqxeumoADfCCP19+qiYiC8gUBjA== + +"@parcel/watcher-linux-arm64-musl@2.5.0": + version "2.5.0" + resolved "https://registry.yarnpkg.com/@parcel/watcher-linux-arm64-musl/-/watcher-linux-arm64-musl-2.5.0.tgz#dcb8ff01077cdf59a18d9e0a4dff7a0cfe5fd732" + integrity sha512-S1qARKOphxfiBEkwLUbHjCY9BWPdWnW9j7f7Hb2jPplu8UZ3nes7zpPOW9bkLbHRvWM0WDTsjdOTUgW0xLBN1Q== + +"@parcel/watcher-linux-x64-glibc@2.5.0": + version "2.5.0" + resolved "https://registry.yarnpkg.com/@parcel/watcher-linux-x64-glibc/-/watcher-linux-x64-glibc-2.5.0.tgz#2e254600fda4e32d83942384d1106e1eed84494d" + integrity sha512-d9AOkusyXARkFD66S6zlGXyzx5RvY+chTP9Jp0ypSTC9d4lzyRs9ovGf/80VCxjKddcUvnsGwCHWuF2EoPgWjw== + +"@parcel/watcher-linux-x64-musl@2.5.0": + version "2.5.0" + resolved "https://registry.yarnpkg.com/@parcel/watcher-linux-x64-musl/-/watcher-linux-x64-musl-2.5.0.tgz#01fcea60fedbb3225af808d3f0a7b11229792eef" + integrity sha512-iqOC+GoTDoFyk/VYSFHwjHhYrk8bljW6zOhPuhi5t9ulqiYq1togGJB5e3PwYVFFfeVgc6pbz3JdQyDoBszVaA== + +"@parcel/watcher-win32-arm64@2.5.0": + version "2.5.0" + resolved "https://registry.yarnpkg.com/@parcel/watcher-win32-arm64/-/watcher-win32-arm64-2.5.0.tgz#87cdb16e0783e770197e52fb1dc027bb0c847154" + integrity sha512-twtft1d+JRNkM5YbmexfcH/N4znDtjgysFaV9zvZmmJezQsKpkfLYJ+JFV3uygugK6AtIM2oADPkB2AdhBrNig== + +"@parcel/watcher-win32-ia32@2.5.0": + version "2.5.0" + resolved "https://registry.yarnpkg.com/@parcel/watcher-win32-ia32/-/watcher-win32-ia32-2.5.0.tgz#778c39b56da33e045ba21c678c31a9f9d7c6b220" + integrity sha512-+rgpsNRKwo8A53elqbbHXdOMtY/tAtTzManTWShB5Kk54N8Q9mzNWV7tV+IbGueCbcj826MfWGU3mprWtuf1TA== + +"@parcel/watcher-win32-x64@2.5.0": + version "2.5.0" + resolved "https://registry.yarnpkg.com/@parcel/watcher-win32-x64/-/watcher-win32-x64-2.5.0.tgz#33873876d0bbc588aacce38e90d1d7480ce81cb7" + integrity sha512-lPrxve92zEHdgeff3aiu4gDOIt4u7sJYha6wbdEZDCDUhtjTsOMiaJzG5lMY4GkWH8p0fMmO2Ppq5G5XXG+DQw== + +"@parcel/watcher@^2.5.0": + version "2.5.0" + resolved "https://registry.yarnpkg.com/@parcel/watcher/-/watcher-2.5.0.tgz#5c88818b12b8de4307a9d3e6dc3e28eba0dfbd10" + integrity sha512-i0GV1yJnm2n3Yq1qw6QrUrd/LI9bE8WEBOTtOkpCXHHdyN3TAGgqAK/DAT05z4fq2x04cARXt2pDmjWjL92iTQ== + dependencies: + detect-libc "^1.0.3" + is-glob "^4.0.3" + micromatch "^4.0.5" + node-addon-api "^7.0.0" + optionalDependencies: + "@parcel/watcher-android-arm64" "2.5.0" + "@parcel/watcher-darwin-arm64" "2.5.0" + "@parcel/watcher-darwin-x64" "2.5.0" + "@parcel/watcher-freebsd-x64" "2.5.0" + "@parcel/watcher-linux-arm-glibc" "2.5.0" + "@parcel/watcher-linux-arm-musl" "2.5.0" + "@parcel/watcher-linux-arm64-glibc" "2.5.0" + "@parcel/watcher-linux-arm64-musl" "2.5.0" + "@parcel/watcher-linux-x64-glibc" "2.5.0" + "@parcel/watcher-linux-x64-musl" "2.5.0" + "@parcel/watcher-win32-arm64" "2.5.0" + "@parcel/watcher-win32-ia32" "2.5.0" + "@parcel/watcher-win32-x64" "2.5.0" + +"@pkgjs/parseargs@^0.11.0": + version "0.11.0" + resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33" + integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg== + +"@tailwindcss/cli@^4.0.0-beta.5": + version "4.0.0-beta.5" + resolved "https://registry.yarnpkg.com/@tailwindcss/cli/-/cli-4.0.0-beta.5.tgz#512333d9670d8f19a2e124c7b4404dd4ceb38d1f" + integrity sha512-FblUvGA4fvd7cOPLcA+ieZC7jMBZ6NG+QApq3gLrUr24tO9hRZZBVA4GcYWdFu6pgW332mSZKzCoLzI7aN0AqQ== + dependencies: + "@parcel/watcher" "^2.5.0" + "@tailwindcss/node" "4.0.0-beta.5" + "@tailwindcss/oxide" "4.0.0-beta.5" + enhanced-resolve "^5.17.1" + lightningcss "^1.26.0" + mri "^1.2.0" + picocolors "^1.1.1" + tailwindcss "4.0.0-beta.5" + +"@tailwindcss/node@4.0.0-beta.5": + version "4.0.0-beta.5" + resolved "https://registry.yarnpkg.com/@tailwindcss/node/-/node-4.0.0-beta.5.tgz#6dbc47a728caf0a0a73569c405800d262836ec20" + integrity sha512-29Ym+rT27zmWMbqcQUdbA9h1J+6k6EcV9nSfswModSWkeJAhrY8Sqzt7uU7hOtI7ETXjy22bSSHPjt/0+cjBdg== + dependencies: + enhanced-resolve "^5.17.1" + jiti "^2.4.0" + tailwindcss "4.0.0-beta.5" + +"@tailwindcss/oxide-android-arm64@4.0.0-beta.5": + version "4.0.0-beta.5" + resolved "https://registry.yarnpkg.com/@tailwindcss/oxide-android-arm64/-/oxide-android-arm64-4.0.0-beta.5.tgz#fed9db4ca92e7469ae631cadcc05b75c1c57de53" + integrity sha512-1eBq4Jo9R4CMHkrJuuHNrwBEkuSg37D/9OxsxcxvC372r8GdQ9fpy3cwYoVtqX6fG/Wg32P/n0t58QiQIlhkzg== + +"@tailwindcss/oxide-darwin-arm64@4.0.0-beta.5": + version "4.0.0-beta.5" + resolved "https://registry.yarnpkg.com/@tailwindcss/oxide-darwin-arm64/-/oxide-darwin-arm64-4.0.0-beta.5.tgz#5648d7a7faabfecd874b1eabf480076e71a27ce2" + integrity sha512-mUjChnLgul9DeKNsWNVuIykwYhKNOp8Suu4qob3T5d8joVYvt+zYM8fpQulAJDH5OzbKYoekkzquJDA6plNxEA== + +"@tailwindcss/oxide-darwin-x64@4.0.0-beta.5": + version "4.0.0-beta.5" + resolved "https://registry.yarnpkg.com/@tailwindcss/oxide-darwin-x64/-/oxide-darwin-x64-4.0.0-beta.5.tgz#8f42023eb3c082cd51e8a58b232f82d765723e6d" + integrity sha512-zLD9Z6B7olFOQUVn/3wZY5IL+6x294n7N+BA1cwwZtnxZNyp3o/26KGDP8mM1TcdbXHb5DOj4OX08wwbffzJrA== + +"@tailwindcss/oxide-freebsd-x64@4.0.0-beta.5": + version "4.0.0-beta.5" + resolved "https://registry.yarnpkg.com/@tailwindcss/oxide-freebsd-x64/-/oxide-freebsd-x64-4.0.0-beta.5.tgz#15c0e8d2d2fa31dd2e9818dde9eadf4b4e7b92fc" + integrity sha512-FsjvHqsf/AXuEZVNA4TQlLZmanP9qjaqedjCyXv19JwfdxlEuFcGHwx6f1zATuiQnQ1DnYeoJLfjv88N59UvZQ== + +"@tailwindcss/oxide-linux-arm-gnueabihf@4.0.0-beta.5": + version "4.0.0-beta.5" + resolved "https://registry.yarnpkg.com/@tailwindcss/oxide-linux-arm-gnueabihf/-/oxide-linux-arm-gnueabihf-4.0.0-beta.5.tgz#41f9b5d24ce8ae91f144d24df584a06dd2ec42e1" + integrity sha512-qvtbk+AdBjd2TvwJ+nuYRHeNDnlP20QIsfnVIENiU1oRTjiWvejPwRlN7bPWn0IR41GG8NEkdhM9L6vHSzaqbw== + +"@tailwindcss/oxide-linux-arm64-gnu@4.0.0-beta.5": + version "4.0.0-beta.5" + resolved "https://registry.yarnpkg.com/@tailwindcss/oxide-linux-arm64-gnu/-/oxide-linux-arm64-gnu-4.0.0-beta.5.tgz#ecd7810fe4c0e14f51b47772e8c49752982f4ed2" + integrity sha512-kcz1OP9Bocq/1wPCldBYDCsll0VLUzUTWZfmg3vh7yKGbtBqL3NrJqmq8jT7FdctgcMnrJ3x3pBchKt/n6OkeQ== + +"@tailwindcss/oxide-linux-arm64-musl@4.0.0-beta.5": + version "4.0.0-beta.5" + resolved "https://registry.yarnpkg.com/@tailwindcss/oxide-linux-arm64-musl/-/oxide-linux-arm64-musl-4.0.0-beta.5.tgz#8b534755f7739e9b0876c23ba3cf78b1561070ac" + integrity sha512-wzA1w/aJBfMFplEg6vVtMwuC4AOi8wyJPwhToxZuk3uq6sgm74prMbnWPhocJK8AwySJHOWDEj4ncXrRkMPPLg== + +"@tailwindcss/oxide-linux-x64-gnu@4.0.0-beta.5": + version "4.0.0-beta.5" + resolved "https://registry.yarnpkg.com/@tailwindcss/oxide-linux-x64-gnu/-/oxide-linux-x64-gnu-4.0.0-beta.5.tgz#8cf61bc45366d824d011a0e595d33af082d4f133" + integrity sha512-nrk9HDW31TDpsr8Iiu0DRtMyr9Rniq/8IWqLV2NeS0EhsmpEi+Ll6PbbuZdPRLA+TCK7pWstqcTC/trxp7LGOQ== + +"@tailwindcss/oxide-linux-x64-musl@4.0.0-beta.5": + version "4.0.0-beta.5" + resolved "https://registry.yarnpkg.com/@tailwindcss/oxide-linux-x64-musl/-/oxide-linux-x64-musl-4.0.0-beta.5.tgz#d9c9f3e588420eaa8b7dee3d4d3dada61749e1c5" + integrity sha512-ETzuMsqiPwY1kt2i1ROTkL9zSJXCvT2Y4onIWKwKPVnEpezVouxXN9guIKoet6KkvDJgZoVtpbO6izgUqpNVwg== + +"@tailwindcss/oxide-win32-arm64-msvc@4.0.0-beta.5": + version "4.0.0-beta.5" + resolved "https://registry.yarnpkg.com/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.0.0-beta.5.tgz#9c8ab30ef6c967834650840bec3aa2f95dd3e5f4" + integrity sha512-Mstca4eNIMwMr5JvjYaE7C/MmGyQeYVxYRMGCNGbW9UDzxRNNVrg9Z4YgXZ2NdnjyhQF7N6BysPMhfQjv0Czng== + +"@tailwindcss/oxide-win32-x64-msvc@4.0.0-beta.5": + version "4.0.0-beta.5" + resolved "https://registry.yarnpkg.com/@tailwindcss/oxide-win32-x64-msvc/-/oxide-win32-x64-msvc-4.0.0-beta.5.tgz#929b50bb46276d14ff04b70c5f1ffd805159367d" + integrity sha512-q3B9DmTMsHpaxg1w0ZxcIL7F3Em8s42SlwDBgnWWPnc6bZMG/gpN6EEL6mWfnvNyyD2LuYLjwG/D0VDuccQdIA== + +"@tailwindcss/oxide@4.0.0-beta.5": + version "4.0.0-beta.5" + resolved "https://registry.yarnpkg.com/@tailwindcss/oxide/-/oxide-4.0.0-beta.5.tgz#b68136376243fccc4f3911a9796078bb28d09bb8" + integrity sha512-m5Vc6UdnFWh1lca5XfcJhkD7peMYE5bHeqFo3Jse+q5bd/ahJqGdIdRp72XtIjhpMztNS/hu0xRzabW/Jo+U9w== + optionalDependencies: + "@tailwindcss/oxide-android-arm64" "4.0.0-beta.5" + "@tailwindcss/oxide-darwin-arm64" "4.0.0-beta.5" + "@tailwindcss/oxide-darwin-x64" "4.0.0-beta.5" + "@tailwindcss/oxide-freebsd-x64" "4.0.0-beta.5" + "@tailwindcss/oxide-linux-arm-gnueabihf" "4.0.0-beta.5" + "@tailwindcss/oxide-linux-arm64-gnu" "4.0.0-beta.5" + "@tailwindcss/oxide-linux-arm64-musl" "4.0.0-beta.5" + "@tailwindcss/oxide-linux-x64-gnu" "4.0.0-beta.5" + "@tailwindcss/oxide-linux-x64-musl" "4.0.0-beta.5" + "@tailwindcss/oxide-win32-arm64-msvc" "4.0.0-beta.5" + "@tailwindcss/oxide-win32-x64-msvc" "4.0.0-beta.5" + +agent-base@^7.0.2: + version "7.1.1" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-7.1.1.tgz#bdbded7dfb096b751a2a087eeeb9664725b2e317" + integrity sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA== + dependencies: + debug "^4.3.4" + +ansi-regex@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" + integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== + +ansi-regex@^6.0.1: + version "6.1.0" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-6.1.0.tgz#95ec409c69619d6cb1b8b34f14b660ef28ebd654" + integrity sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA== + +ansi-styles@^4.0.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937" + integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg== + dependencies: + color-convert "^2.0.1" + +ansi-styles@^6.1.0: + version "6.2.1" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-6.2.1.tgz#0e62320cf99c21afff3b3012192546aacbfb05c5" + integrity sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug== + +balanced-match@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" + integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== + +brace-expansion@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae" + integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA== + dependencies: + balanced-match "^1.0.0" + +braces@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789" + integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA== + dependencies: + fill-range "^7.1.1" + +chalk@^5.0.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-5.3.0.tgz#67c20a7ebef70e7f3970a01f90fa210cb6860385" + integrity sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w== + +color-convert@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3" + integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ== + dependencies: + color-name "~1.1.4" + +color-name@~1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" + integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== + +cross-spawn@^7.0.0: + version "7.0.6" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f" + integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA== + dependencies: + path-key "^3.1.0" + shebang-command "^2.0.0" + which "^2.0.1" + +debug@4, debug@^4.3.4: + version "4.3.7" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.7.tgz#87945b4151a011d76d95a198d7111c865c360a52" + integrity sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ== + dependencies: + ms "^2.1.3" + +detect-libc@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b" + integrity sha512-pGjwhsmsp4kL2RTz08wcOlGN83otlqHeD/Z5T8GXZB+/YcpQ/dgo+lbU8ZsGxV0HIvqqxo9l7mqYwyYMD9bKDg== + +dom-serializer@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-2.0.0.tgz#e41b802e1eedf9f6cae183ce5e622d789d7d8e53" + integrity sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg== + dependencies: + domelementtype "^2.3.0" + domhandler "^5.0.2" + entities "^4.2.0" + +domelementtype@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.3.0.tgz#5c45e8e869952626331d7aab326d01daf65d589d" + integrity sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw== + +domhandler@^5.0.2, domhandler@^5.0.3: + version "5.0.3" + resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-5.0.3.tgz#cc385f7f751f1d1fc650c21374804254538c7d31" + integrity sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w== + dependencies: + domelementtype "^2.3.0" + +domutils@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/domutils/-/domutils-3.1.0.tgz#c47f551278d3dc4b0b1ab8cbb42d751a6f0d824e" + integrity sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA== + dependencies: + dom-serializer "^2.0.0" + domelementtype "^2.3.0" + domhandler "^5.0.3" + +eastasianwidth@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/eastasianwidth/-/eastasianwidth-0.2.0.tgz#696ce2ec0aa0e6ea93a397ffcf24aa7840c827cb" + integrity sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA== + +emoji-regex@^8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" + integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== + +emoji-regex@^9.2.2: + version "9.2.2" + resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-9.2.2.tgz#840c8803b0d8047f4ff0cf963176b32d4ef3ed72" + integrity sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg== + +enhanced-resolve@^5.17.1: + version "5.17.1" + resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.17.1.tgz#67bfbbcc2f81d511be77d686a90267ef7f898a15" + integrity sha512-LMHl3dXhTcfv8gM4kEzIUeTQ+7fpdA0l2tUf34BddXPkz2A5xJ5L/Pchd5BL6rdccM9QGvu0sWZzK1Z1t4wwyg== + dependencies: + graceful-fs "^4.2.4" + tapable "^2.2.0" + +entities@^4.2.0, entities@^4.5.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/entities/-/entities-4.5.0.tgz#5d268ea5e7113ec74c4d033b79ea5a35a488fb48" + integrity sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw== + +escape-html@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988" + integrity sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow== + +extend@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" + integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== + +fill-range@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292" + integrity sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg== + dependencies: + to-regex-range "^5.0.1" + +foreground-child@^3.1.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/foreground-child/-/foreground-child-3.3.0.tgz#0ac8644c06e431439f8561db8ecf29a7b5519c77" + integrity sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg== + dependencies: + cross-spawn "^7.0.0" + signal-exit "^4.0.1" + +gaxios@^6.0.0: + version "6.7.1" + resolved "https://registry.yarnpkg.com/gaxios/-/gaxios-6.7.1.tgz#ebd9f7093ede3ba502685e73390248bb5b7f71fb" + integrity sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ== + dependencies: + extend "^3.0.2" + https-proxy-agent "^7.0.1" + is-stream "^2.0.0" + node-fetch "^2.6.9" + uuid "^9.0.1" + +glob@^10.3.10: + version "10.4.5" + resolved "https://registry.yarnpkg.com/glob/-/glob-10.4.5.tgz#f4d9f0b90ffdbab09c9d77f5f29b4262517b0956" + integrity sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg== + dependencies: + foreground-child "^3.1.0" + jackspeak "^3.1.2" + minimatch "^9.0.4" + minipass "^7.1.2" + package-json-from-dist "^1.0.0" + path-scurry "^1.11.1" + +graceful-fs@^4.2.4: + version "4.2.11" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" + integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== + +htmlparser2@^9.0.0: + version "9.1.0" + resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-9.1.0.tgz#cdb498d8a75a51f739b61d3f718136c369bc8c23" + integrity sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ== + dependencies: + domelementtype "^2.3.0" + domhandler "^5.0.3" + domutils "^3.1.0" + entities "^4.5.0" + +https-proxy-agent@^7.0.1: + version "7.0.5" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz#9e8b5013873299e11fab6fd548405da2d6c602b2" + integrity sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw== + dependencies: + agent-base "^7.0.2" + debug "4" + +is-extglob@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2" + integrity sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ== + +is-fullwidth-code-point@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d" + integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg== + +is-glob@^4.0.3: + version "4.0.3" + resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.3.tgz#64f61e42cbbb2eec2071a9dac0b28ba1e65d5084" + integrity sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg== + dependencies: + is-extglob "^2.1.1" + +is-number@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b" + integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng== + +is-stream@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.1.tgz#fac1e3d53b97ad5a9d0ae9cef2389f5810a5c077" + integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg== + +isexe@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" + integrity sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw== + +jackspeak@^3.1.2: + version "3.4.3" + resolved "https://registry.yarnpkg.com/jackspeak/-/jackspeak-3.4.3.tgz#8833a9d89ab4acde6188942bd1c53b6390ed5a8a" + integrity sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw== + dependencies: + "@isaacs/cliui" "^8.0.2" + optionalDependencies: + "@pkgjs/parseargs" "^0.11.0" + +jiti@^2.4.0: + version "2.4.1" + resolved "https://registry.yarnpkg.com/jiti/-/jiti-2.4.1.tgz#4de9766ccbfa941d9b6390d2b159a4b295a52e6b" + integrity sha512-yPBThwecp1wS9DmoA4x4KR2h3QoslacnDR8ypuFM962kI4/456Iy1oHx2RAgh4jfZNdn0bctsdadceiBUgpU1g== + +lightningcss-darwin-arm64@1.28.2: + version "1.28.2" + resolved "https://registry.yarnpkg.com/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.28.2.tgz#a906fd84cb43d753cb5db9c367f8f38482e8fb03" + integrity sha512-/8cPSqZiusHSS+WQz0W4NuaqFjquys1x+NsdN/XOHb+idGHJSoJ7SoQTVl3DZuAgtPZwFZgRfb/vd1oi8uX6+g== + +lightningcss-darwin-x64@1.28.2: + version "1.28.2" + resolved "https://registry.yarnpkg.com/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.28.2.tgz#6c43249d4ae821416d0d78403eae56111d0c6a94" + integrity sha512-R7sFrXlgKjvoEG8umpVt/yutjxOL0z8KWf0bfPT3cYMOW4470xu5qSHpFdIOpRWwl3FKNMUdbKtMUjYt0h2j4g== + +lightningcss-freebsd-x64@1.28.2: + version "1.28.2" + resolved "https://registry.yarnpkg.com/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.28.2.tgz#804bc6652c6721e94a92e7bbb5e65165376cf108" + integrity sha512-l2qrCT+x7crAY+lMIxtgvV10R8VurzHAoUZJaVFSlHrN8kRLTvEg9ObojIDIexqWJQvJcVVV3vfzsEynpiuvgA== + +lightningcss-linux-arm-gnueabihf@1.28.2: + version "1.28.2" + resolved "https://registry.yarnpkg.com/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.28.2.tgz#c32595127b565690d854c9ff641831e4ad739ee1" + integrity sha512-DKMzpICBEKnL53X14rF7hFDu8KKALUJtcKdFUCW5YOlGSiwRSgVoRjM97wUm/E0NMPkzrTi/rxfvt7ruNK8meg== + +lightningcss-linux-arm64-gnu@1.28.2: + version "1.28.2" + resolved "https://registry.yarnpkg.com/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.28.2.tgz#85646f08c5efbfd7c94f8e5ed6392d5cf95fa42c" + integrity sha512-nhfjYkfymWZSxdtTNMWyhFk2ImUm0X7NAgJWFwnsYPOfmtWQEapzG/DXZTfEfMjSzERNUNJoQjPAbdqgB+sjiw== + +lightningcss-linux-arm64-musl@1.28.2: + version "1.28.2" + resolved "https://registry.yarnpkg.com/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.28.2.tgz#4d9bc20cf6de28c4d0c586d81c577891555ad831" + integrity sha512-1SPG1ZTNnphWvAv8RVOymlZ8BDtAg69Hbo7n4QxARvkFVCJAt0cgjAw1Fox0WEhf4PwnyoOBaVH0Z5YNgzt4dA== + +lightningcss-linux-x64-gnu@1.28.2: + version "1.28.2" + resolved "https://registry.yarnpkg.com/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.28.2.tgz#74bd797d7157817c4e42ec45f1844a69636a9d82" + integrity sha512-ZhQy0FcO//INWUdo/iEdbefntTdpPVQ0XJwwtdbBuMQe+uxqZoytm9M+iqR9O5noWFaxK+nbS2iR/I80Q2Ofpg== + +lightningcss-linux-x64-musl@1.28.2: + version "1.28.2" + resolved "https://registry.yarnpkg.com/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.28.2.tgz#13ce6db4c491ebbb93099d6427746ab7bff3774f" + integrity sha512-alb/j1NMrgQmSFyzTbN1/pvMPM+gdDw7YBuQ5VSgcFDypN3Ah0BzC2dTZbzwzaMdUVDszX6zH5MzjfVN1oGuww== + +lightningcss-win32-arm64-msvc@1.28.2: + version "1.28.2" + resolved "https://registry.yarnpkg.com/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.28.2.tgz#eaae12c4a58a545a3adf40b22ba9625e5c0ebd29" + integrity sha512-WnwcjcBeAt0jGdjlgbT9ANf30pF0C/QMb1XnLnH272DQU8QXh+kmpi24R55wmWBwaTtNAETZ+m35ohyeMiNt+g== + +lightningcss-win32-x64-msvc@1.28.2: + version "1.28.2" + resolved "https://registry.yarnpkg.com/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.28.2.tgz#1f7c4474b2dc3dd1c12e22de32e4de23bdfa41e7" + integrity sha512-3piBifyT3avz22o6mDKywQC/OisH2yDK+caHWkiMsF82i3m5wDBadyCjlCQ5VNgzYkxrWZgiaxHDdd5uxsi0/A== + +lightningcss@^1.26.0: + version "1.28.2" + resolved "https://registry.yarnpkg.com/lightningcss/-/lightningcss-1.28.2.tgz#cc26fad9ad64a621bd39ac6248095891cf584cce" + integrity sha512-ePLRrbt3fgjXI5VFZOLbvkLD5ZRuxGKm+wJ3ujCqBtL3NanDHPo/5zicR5uEKAPiIjBYF99BM4K4okvMznjkVA== + dependencies: + detect-libc "^1.0.3" + optionalDependencies: + lightningcss-darwin-arm64 "1.28.2" + lightningcss-darwin-x64 "1.28.2" + lightningcss-freebsd-x64 "1.28.2" + lightningcss-linux-arm-gnueabihf "1.28.2" + lightningcss-linux-arm64-gnu "1.28.2" + lightningcss-linux-arm64-musl "1.28.2" + lightningcss-linux-x64-gnu "1.28.2" + lightningcss-linux-x64-musl "1.28.2" + lightningcss-win32-arm64-msvc "1.28.2" + lightningcss-win32-x64-msvc "1.28.2" + +linkinator@^6.1.2: + version "6.1.2" + resolved "https://registry.yarnpkg.com/linkinator/-/linkinator-6.1.2.tgz#0cc25d934685d740f8cd3ccb95127ec1345a7605" + integrity sha512-PndSrQe21Hf4sn2vZldEzJmD0EUJbIsEy4jcZLcHd6IZfQ6rC6iv+Fwo666TWM9DcXjbCrHpxnVX6xaGrcJ/eA== + dependencies: + chalk "^5.0.0" + escape-html "^1.0.3" + gaxios "^6.0.0" + glob "^10.3.10" + htmlparser2 "^9.0.0" + marked "^13.0.0" + meow "^13.0.0" + mime "^4.0.0" + server-destroy "^1.0.1" + srcset "^5.0.0" + +lru-cache@^10.2.0: + version "10.4.3" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.4.3.tgz#410fc8a17b70e598013df257c2446b7f3383f119" + integrity sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ== + +marked@^13.0.0: + version "13.0.3" + resolved "https://registry.yarnpkg.com/marked/-/marked-13.0.3.tgz#5c5b4a5d0198060c7c9bc6ef9420a7fed30f822d" + integrity sha512-rqRix3/TWzE9rIoFGIn8JmsVfhiuC8VIQ8IdX5TfzmeBucdY05/0UlzKaw0eVtpcN/OdVFpBk7CjKGo9iHJ/zA== + +meow@^13.0.0: + version "13.2.0" + resolved "https://registry.yarnpkg.com/meow/-/meow-13.2.0.tgz#6b7d63f913f984063b3cc261b6e8800c4cd3474f" + integrity sha512-pxQJQzB6djGPXh08dacEloMFopsOqGVRKFPYvPOt9XDZ1HasbgDZA74CJGreSU4G3Ak7EFJGoiH2auq+yXISgA== + +micromatch@^4.0.5: + version "4.0.8" + resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.8.tgz#d66fa18f3a47076789320b9b1af32bd86d9fa202" + integrity sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA== + dependencies: + braces "^3.0.3" + picomatch "^2.3.1" + +mime@^4.0.0: + version "4.0.4" + resolved "https://registry.yarnpkg.com/mime/-/mime-4.0.4.tgz#9f851b0fc3c289d063b20a7a8055b3014b25664b" + integrity sha512-v8yqInVjhXyqP6+Kw4fV3ZzeMRqEW6FotRsKXjRS5VMTNIuXsdRoAvklpoRgSqXm6o9VNH4/C0mgedko9DdLsQ== + +minimatch@^9.0.4: + version "9.0.5" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.5.tgz#d74f9dd6b57d83d8e98cfb82133b03978bc929e5" + integrity sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow== + dependencies: + brace-expansion "^2.0.1" + +"minipass@^5.0.0 || ^6.0.2 || ^7.0.0", minipass@^7.1.2: + version "7.1.2" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.1.2.tgz#93a9626ce5e5e66bd4db86849e7515e92340a707" + integrity sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw== + +mri@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/mri/-/mri-1.2.0.tgz#6721480fec2a11a4889861115a48b6cbe7cc8f0b" + integrity sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA== + +ms@^2.1.3: + version "2.1.3" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" + integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== + +node-addon-api@^7.0.0: + version "7.1.1" + resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-7.1.1.tgz#1aba6693b0f255258a049d621329329322aad558" + integrity sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ== + +node-fetch@^2.6.9: + version "2.7.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" + integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== + dependencies: + whatwg-url "^5.0.0" + +package-json-from-dist@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz#4f1471a010827a86f94cfd9b0727e36d267de505" + integrity sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw== + +path-key@^3.1.0: + version "3.1.1" + resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" + integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== + +path-scurry@^1.11.1: + version "1.11.1" + resolved "https://registry.yarnpkg.com/path-scurry/-/path-scurry-1.11.1.tgz#7960a668888594a0720b12a911d1a742ab9f11d2" + integrity sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA== + dependencies: + lru-cache "^10.2.0" + minipass "^5.0.0 || ^6.0.2 || ^7.0.0" + +picocolors@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.1.1.tgz#3d321af3eab939b083c8f929a1d12cda81c26b6b" + integrity sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA== + +picomatch@^2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" + integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== + +server-destroy@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/server-destroy/-/server-destroy-1.0.1.tgz#f13bf928e42b9c3e79383e61cc3998b5d14e6cdd" + integrity sha512-rb+9B5YBIEzYcD6x2VKidaa+cqYBJQKnU4oe4E3ANwRRN56yk/ua1YCJT1n21NTS8w6CcOclAKNP3PhdCXKYtQ== + +shebang-command@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea" + integrity sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA== + dependencies: + shebang-regex "^3.0.0" + +shebang-regex@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" + integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== + +signal-exit@^4.0.1: + version "4.1.0" + resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-4.1.0.tgz#952188c1cbd546070e2dd20d0f41c0ae0530cb04" + integrity sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw== + +srcset@^5.0.0: + version "5.0.1" + resolved "https://registry.yarnpkg.com/srcset/-/srcset-5.0.1.tgz#e660a728f195419e4afa95121099bc9efb7a1e36" + integrity sha512-/P1UYbGfJVlxZag7aABNRrulEXAwCSDo7fklafOQrantuPTDmYgijJMks2zusPCVzgW9+4P69mq7w6pYuZpgxw== + +"string-width-cjs@npm:string-width@^4.2.0": + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +string-width@^4.1.0: + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +string-width@^5.0.1, string-width@^5.1.2: + version "5.1.2" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-5.1.2.tgz#14f8daec6d81e7221d2a357e668cab73bdbca794" + integrity sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA== + dependencies: + eastasianwidth "^0.2.0" + emoji-regex "^9.2.2" + strip-ansi "^7.0.1" + +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + +strip-ansi@^6.0.0, strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + +strip-ansi@^7.0.1: + version "7.1.0" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" + integrity sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ== + dependencies: + ansi-regex "^6.0.1" + +tailwindcss@4.0.0-beta.5, tailwindcss@^4.0.0-beta.5: + version "4.0.0-beta.5" + resolved "https://registry.yarnpkg.com/tailwindcss/-/tailwindcss-4.0.0-beta.5.tgz#af52b6972fbfc60c46ced13a7c81201795c055ff" + integrity sha512-59zeDyaVE5z1iQnhk5cGeBJvb6Z/Iym4qSFzPddiotWbnqWcIqj0kRgZH8OyR4VZU+solTRQaWRgfDZ8PQ+p8A== + +tapable@^2.2.0: + version "2.2.1" + resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.1.tgz#1967a73ef4060a82f12ab96af86d52fdb76eeca0" + integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ== + +to-regex-range@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-5.0.1.tgz#1648c44aae7c8d988a326018ed72f5b4dd0392e4" + integrity sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ== + dependencies: + is-number "^7.0.0" + +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== + +uuid@^9.0.1: + version "9.0.1" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.1.tgz#e188d4c8853cc722220392c424cd637f32293f30" + integrity sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA== + +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== + +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" + +which@^2.0.1: + version "2.0.2" + resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1" + integrity sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA== + dependencies: + isexe "^2.0.0" + +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + +wrap-ansi@^8.1.0: + version "8.1.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" + integrity sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ== + dependencies: + ansi-styles "^6.1.0" + string-width "^5.0.1" + strip-ansi "^7.0.1"