diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 142035af42..b306663d8d 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -15,6 +15,7 @@ To disable this, set the environment variable DATABRICKS_CACHE_ENABLED to false. * Fix false positive folder permission warnings and make them more actionable ([#4216](https://github.com/databricks/cli/pull/4216)) * Pass additional Azure DevOps system variables ([#4236](https://github.com/databricks/cli/pull/4236)) * Replace Black formatter with Ruff in Python bundle templates for faster, all-in-one linting and formatting ([#4196](https://github.com/databricks/cli/pull/4196)) +* engine/direct: support quality monitors ([#4278](https://github.com/databricks/cli/pull/4278)) ### Dependency updates diff --git a/acceptance/bin/create_table.py b/acceptance/bin/create_table.py new file mode 100755 index 0000000000..1fb028dda1 --- /dev/null +++ b/acceptance/bin/create_table.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +import json +import os +import subprocess +import sys +import time + +table_name = sys.argv[1] +# Extract catalog.schema from table_name +parts = table_name.split(".") +if len(parts) != 3: + print(f"Invalid table name: {table_name}. Expected format: catalog.schema.table", file=sys.stderr) + sys.exit(1) + +catalog_name = parts[0] +schema_name = parts[1] +full_schema_name = f"{catalog_name}.{schema_name}" + +cli = os.environ.get("CLI", "databricks") + + +def run_cli(*args): + result = subprocess.run([cli, *args], capture_output=True, text=True) + return result + + +def execute_sql(warehouse_id, sql): + """Execute SQL using the API endpoint.""" + payload = json.dumps({"warehouse_id": warehouse_id, "statement": sql, "wait_timeout": "30s"}) + return run_cli("api", "post", "/api/2.0/sql/statements/", "--json", payload) + + +# Get warehouse ID from environment variable +warehouse_id = os.environ["TEST_DEFAULT_WAREHOUSE_ID"] + +# Create a simple table +sql = f"CREATE TABLE IF NOT EXISTS {table_name} (id INT, value STRING, timestamp TIMESTAMP) USING DELTA" + +result = execute_sql(warehouse_id, sql) +if result.returncode != 0: + print(f"Failed to create table: {result.stderr}", file=sys.stderr) + sys.exit(1) + +print(f"Created table {table_name}") + +# Insert some sample data so the monitor has something to analyze +insert_sql = f"""INSERT INTO {table_name} VALUES +(1, 'test1', current_timestamp()), +(2, 'test2', current_timestamp()), +(3, 'test3', current_timestamp())""" + +result = execute_sql(warehouse_id, insert_sql) +if result.returncode != 0: + print(f"Failed to insert data: {result.stderr}", file=sys.stderr) + sys.exit(1) + +print(f"Inserted sample data into {table_name}") + +# Wait for table to be visible in Unity Catalog +for attempt in range(10): + result = run_cli("tables", "get", table_name) + if result.returncode == 0: + table_info = json.loads(result.stdout) + print(f"Table {table_name} is now visible (catalog_name={table_info.get('catalog_name')})") + break + if attempt < 9: + time.sleep(1) + else: + print(f"Warning: Table may not be immediately visible: {result.stderr}", file=sys.stderr) diff --git a/acceptance/bundle/deployment/bind/quality-monitor/output.txt b/acceptance/bundle/deployment/bind/quality-monitor/output.txt index e45b0350b5..1cd120d91d 100644 --- a/acceptance/bundle/deployment/bind/quality-monitor/output.txt +++ b/acceptance/bundle/deployment/bind/quality-monitor/output.txt @@ -2,10 +2,11 @@ >>> [CLI] quality-monitors create catalog.schema.table --json @input.json { "assets_dir":"/Users/user/databricks_lakehouse_monitoring", - "drift_metrics_table_name":"", + "dashboard_id":"(redacted)", + "drift_metrics_table_name":"catalog.schema.table_drift_metrics", "monitor_version":0, "output_schema_name":"catalog.schema", - "profile_metrics_table_name":"", + "profile_metrics_table_name":"catalog.schema.table_profile_metrics", "snapshot": {}, "status":"MONITOR_STATUS_ACTIVE", "table_name":"catalog.schema.table" diff --git a/acceptance/bundle/deployment/bind/quality-monitor/test.toml b/acceptance/bundle/deployment/bind/quality-monitor/test.toml index 7d36fb9dc1..bc3149360b 100644 --- a/acceptance/bundle/deployment/bind/quality-monitor/test.toml +++ b/acceptance/bundle/deployment/bind/quality-monitor/test.toml @@ -1,2 +1,6 @@ Local = true Cloud = false + +[[Repls]] +Old = '"dashboard_id":"[0-9a-f]+",' +New = '"dashboard_id":"(redacted)",' diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index f302bec0fb..e15c517971 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -3457,6 +3457,60 @@ resources.pipelines.*.permissions.permissions[*].group_name string ALL resources.pipelines.*.permissions.permissions[*].permission_level iam.PermissionLevel ALL resources.pipelines.*.permissions.permissions[*].service_principal_name string ALL resources.pipelines.*.permissions.permissions[*].user_name string ALL +resources.quality_monitors.*.assets_dir string ALL +resources.quality_monitors.*.baseline_table_name string ALL +resources.quality_monitors.*.custom_metrics []catalog.MonitorMetric ALL +resources.quality_monitors.*.custom_metrics[*] catalog.MonitorMetric ALL +resources.quality_monitors.*.custom_metrics[*].definition string ALL +resources.quality_monitors.*.custom_metrics[*].input_columns []string ALL +resources.quality_monitors.*.custom_metrics[*].input_columns[*] string ALL +resources.quality_monitors.*.custom_metrics[*].name string ALL +resources.quality_monitors.*.custom_metrics[*].output_data_type string ALL +resources.quality_monitors.*.custom_metrics[*].type catalog.MonitorMetricType ALL +resources.quality_monitors.*.dashboard_id string REMOTE +resources.quality_monitors.*.data_classification_config *catalog.MonitorDataClassificationConfig ALL +resources.quality_monitors.*.data_classification_config.enabled bool ALL +resources.quality_monitors.*.drift_metrics_table_name string REMOTE +resources.quality_monitors.*.id string INPUT +resources.quality_monitors.*.inference_log *catalog.MonitorInferenceLog ALL +resources.quality_monitors.*.inference_log.granularities []string ALL +resources.quality_monitors.*.inference_log.granularities[*] string ALL +resources.quality_monitors.*.inference_log.label_col string ALL +resources.quality_monitors.*.inference_log.model_id_col string ALL +resources.quality_monitors.*.inference_log.prediction_col string ALL +resources.quality_monitors.*.inference_log.prediction_proba_col string ALL +resources.quality_monitors.*.inference_log.problem_type catalog.MonitorInferenceLogProblemType ALL +resources.quality_monitors.*.inference_log.timestamp_col string ALL +resources.quality_monitors.*.latest_monitor_failure_msg string ALL +resources.quality_monitors.*.lifecycle resources.Lifecycle INPUT +resources.quality_monitors.*.lifecycle.prevent_destroy bool INPUT +resources.quality_monitors.*.modified_status string INPUT +resources.quality_monitors.*.monitor_version int64 REMOTE +resources.quality_monitors.*.notifications *catalog.MonitorNotifications ALL +resources.quality_monitors.*.notifications.on_failure *catalog.MonitorDestination ALL +resources.quality_monitors.*.notifications.on_failure.email_addresses []string ALL +resources.quality_monitors.*.notifications.on_failure.email_addresses[*] string ALL +resources.quality_monitors.*.notifications.on_new_classification_tag_detected *catalog.MonitorDestination ALL +resources.quality_monitors.*.notifications.on_new_classification_tag_detected.email_addresses []string ALL +resources.quality_monitors.*.notifications.on_new_classification_tag_detected.email_addresses[*] string ALL +resources.quality_monitors.*.output_schema_name string ALL +resources.quality_monitors.*.profile_metrics_table_name string REMOTE +resources.quality_monitors.*.schedule *catalog.MonitorCronSchedule ALL +resources.quality_monitors.*.schedule.pause_status catalog.MonitorCronSchedulePauseStatus ALL +resources.quality_monitors.*.schedule.quartz_cron_expression string ALL +resources.quality_monitors.*.schedule.timezone_id string ALL +resources.quality_monitors.*.skip_builtin_dashboard bool INPUT STATE +resources.quality_monitors.*.slicing_exprs []string ALL +resources.quality_monitors.*.slicing_exprs[*] string ALL +resources.quality_monitors.*.snapshot *catalog.MonitorSnapshot ALL +resources.quality_monitors.*.status catalog.MonitorInfoStatus REMOTE +resources.quality_monitors.*.table_name string ALL +resources.quality_monitors.*.time_series *catalog.MonitorTimeSeries ALL +resources.quality_monitors.*.time_series.granularities []string ALL +resources.quality_monitors.*.time_series.granularities[*] string ALL +resources.quality_monitors.*.time_series.timestamp_col string ALL +resources.quality_monitors.*.url string INPUT +resources.quality_monitors.*.warehouse_id string INPUT STATE resources.registered_models.*.aliases []catalog.RegisteredModelAlias ALL resources.registered_models.*.aliases[*] catalog.RegisteredModelAlias ALL resources.registered_models.*.aliases[*].alias_name string ALL diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/databricks.yml.tmpl b/acceptance/bundle/resources/quality_monitors/change_assets_dir/databricks.yml.tmpl new file mode 100644 index 0000000000..3fe5194f1c --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: quality-monitor-update-$UNIQUE_NAME + +resources: + quality_monitors: + monitor1: + table_name: main.qm_test_$UNIQUE_NAME.test_table + assets_dir: /Workspace/Users/$CURRENT_USER_NAME/monitor_assets_$UNIQUE_NAME + output_schema_name: main.qm_test_$UNIQUE_NAME + snapshot: {} + warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.deploy.direct.txt b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.deploy.direct.txt new file mode 100644 index 0000000000..1268e736e1 --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.deploy.direct.txt @@ -0,0 +1,6 @@ + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/quality-monitor-update-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.deploy.requests.direct.json b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.deploy.requests.direct.json new file mode 100644 index 0000000000..64a4352ff6 --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.deploy.requests.direct.json @@ -0,0 +1,14 @@ +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/tables/main.qm_test_[UNIQUE_NAME].test_table/monitor" +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/tables/main.qm_test_[UNIQUE_NAME].test_table/monitor", + "body": { + "assets_dir": "/Workspace/Users/[USERNAME]/monitor_assets2_[UNIQUE_NAME]", + "output_schema_name": "main.qm_test_[UNIQUE_NAME]", + "snapshot": {}, + "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" + } +} diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.deploy.requests.terraform.json b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.deploy.requests.terraform.json new file mode 100644 index 0000000000..db37d531e3 --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.deploy.requests.terraform.json @@ -0,0 +1,9 @@ +{ + "method": "PUT", + "path": "/api/2.1/unity-catalog/tables/main.qm_test_[UNIQUE_NAME].test_table/monitor", + "body": { + "dashboard_id": "(redacted)", + "output_schema_name": "main.qm_test_[UNIQUE_NAME]", + "snapshot": {} + } +} diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.deploy.terraform.txt b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.deploy.terraform.txt new file mode 100644 index 0000000000..99c0d7d2d8 --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.deploy.terraform.txt @@ -0,0 +1,22 @@ + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/quality-monitor-update-[UNIQUE_NAME]/default/files... +Deploying resources... +Error: terraform apply: exit status 1 + +Error: Provider produced inconsistent result after apply + +When applying changes to databricks_quality_monitor.monitor1, provider +"provider[\"registry.terraform.io/databricks/databricks\"]" produced an +unexpected new value: .assets_dir: was +cty.StringVal("/Workspace/Users/[USERNAME]/monitor_assets2_[UNIQUE_NAME]"), +but now +cty.StringVal("/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]"). + +This is a bug in the provider, which should be reported in the provider's own +issue tracker. + + +Updating deployment state... + +Exit code: 1 diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.json b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.json new file mode 100644 index 0000000000..0eacfaf389 --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.json @@ -0,0 +1,48 @@ + +>>> errcode [CLI] bundle plan -o json +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "plan": { + "resources.quality_monitors.monitor1": { + "action": "recreate", + "new_state": { + "value": { + "assets_dir": "/Workspace/Users/[USERNAME]/monitor_assets2_[UNIQUE_NAME]", + "output_schema_name": "main.qm_test_[UNIQUE_NAME]", + "snapshot": {}, + "table_name": "main.qm_test_[UNIQUE_NAME].test_table", + "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" + } + }, + "remote_state": { + "assets_dir": "/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]", + "dashboard_id": "(redacted)", + "drift_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_drift_metrics", + "monitor_version": 0, + "output_schema_name": "main.qm_test_[UNIQUE_NAME]", + "profile_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_profile_metrics", + "snapshot": {}, + "status": "MONITOR_STATUS_ACTIVE", + "table_name": "main.qm_test_[UNIQUE_NAME].test_table" + }, + "changes": { + "assets_dir": { + "action": "recreate", + "reason": "field_triggers", + "old": "/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]", + "new": "/Workspace/Users/[USERNAME]/monitor_assets2_[UNIQUE_NAME]", + "remote": "/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]" + }, + "warehouse_id": { + "action": "skip", + "reason": "config_only", + "old": "[TEST_DEFAULT_WAREHOUSE_ID]", + "new": "[TEST_DEFAULT_WAREHOUSE_ID]" + } + } + } + } +} diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.txt b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.txt new file mode 100644 index 0000000000..c812b16bfa --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.txt @@ -0,0 +1,5 @@ + +>>> errcode [CLI] bundle plan +recreate quality_monitors.monitor1 + +Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.terraform.json b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.terraform.json new file mode 100644 index 0000000000..a209463848 --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.terraform.json @@ -0,0 +1,11 @@ + +>>> errcode [CLI] bundle plan -o json +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.quality_monitors.monitor1": { + "action": "update" + } + } +} diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.terraform.txt b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.terraform.txt new file mode 100644 index 0000000000..f84d2e70af --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.terraform.txt @@ -0,0 +1,5 @@ + +>>> errcode [CLI] bundle plan +update quality_monitors.monitor1 + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan_after_deploy.direct.txt b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan_after_deploy.direct.txt new file mode 100644 index 0000000000..67533bda2c --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan_after_deploy.direct.txt @@ -0,0 +1,3 @@ + +>>> errcode [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan_after_deploy.terraform.txt b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan_after_deploy.terraform.txt new file mode 100644 index 0000000000..f84d2e70af --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan_after_deploy.terraform.txt @@ -0,0 +1,5 @@ + +>>> errcode [CLI] bundle plan +update quality_monitors.monitor1 + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml new file mode 100644 index 0000000000..d61c11e25c --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt new file mode 100644 index 0000000000..08f6c53ae1 --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt @@ -0,0 +1,35 @@ + +>>> [CLI] schemas create qm_test_[UNIQUE_NAME] main -o json +{ + "full_name": "main.qm_test_[UNIQUE_NAME]" +} + +>>> [CLI] schemas create qm_test_[UNIQUE_NAME]_2 main -o json +{ + "full_name": "main.qm_test_[UNIQUE_NAME]_2" +} + +>>> create_table.py main.qm_test_[UNIQUE_NAME].test_table +Created table main.qm_test_[UNIQUE_NAME].test_table +Inserted sample data into main.qm_test_[UNIQUE_NAME].test_table +Table main.qm_test_[UNIQUE_NAME].test_table is now visible (catalog_name=main) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/quality-monitor-update-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.quality_monitors.monitor1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/quality-monitor-update-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script new file mode 100644 index 0000000000..6caf49a7f4 --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script @@ -0,0 +1,30 @@ +SCHEMA_NAME="main.qm_test_${UNIQUE_NAME}" +TABLE_NAME="${SCHEMA_NAME}.test_table" + +trace $CLI schemas create "qm_test_${UNIQUE_NAME}" main -o json | jq '{full_name}' +trace $CLI schemas create "qm_test_${UNIQUE_NAME}_2" main -o json | jq '{full_name}' +trace create_table.py "$TABLE_NAME" + +cleanup() { + trace $CLI bundle destroy --auto-approve + $CLI schemas delete "$SCHEMA_NAME" --force 2>/dev/null || true + $CLI schemas delete "qm_test_${UNIQUE_NAME}_2" --force 2>/dev/null || true + rm -f out.requests.txt +} +trap cleanup EXIT + +envsubst < databricks.yml.tmpl > databricks.yml + +trace $CLI bundle deploy + +trace $CLI bundle plan | contains.py "1 unchanged" + +update_file.py databricks.yml "assets_dir: /Workspace/Users/$CURRENT_USER_NAME/monitor_assets_$UNIQUE_NAME" "assets_dir: /Workspace/Users/$CURRENT_USER_NAME/monitor_assets2_$UNIQUE_NAME" + +trace errcode $CLI bundle plan &> out.plan.$DATABRICKS_BUNDLE_ENGINE.txt +trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json + +rm out.requests.txt +trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt +trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace errcode $CLI bundle plan &> out.plan_after_deploy.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/databricks.yml.tmpl b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/databricks.yml.tmpl new file mode 100644 index 0000000000..3fe5194f1c --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: quality-monitor-update-$UNIQUE_NAME + +resources: + quality_monitors: + monitor1: + table_name: main.qm_test_$UNIQUE_NAME.test_table + assets_dir: /Workspace/Users/$CURRENT_USER_NAME/monitor_assets_$UNIQUE_NAME + output_schema_name: main.qm_test_$UNIQUE_NAME + snapshot: {} + warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.deploy.requests.json b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.deploy.requests.json new file mode 100644 index 0000000000..4e9bab574e --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.deploy.requests.json @@ -0,0 +1,8 @@ +{ + "method": "PUT", + "path": "/api/2.1/unity-catalog/tables/main.qm_test_[UNIQUE_NAME].test_table/monitor", + "body": { + "output_schema_name": "main.qm_test_[UNIQUE_NAME]_2", + "snapshot": {} + } +} diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.direct.json b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.direct.json new file mode 100644 index 0000000000..5fa74725b4 --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.direct.json @@ -0,0 +1,45 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "plan": { + "resources.quality_monitors.monitor1": { + "action": "update", + "new_state": { + "value": { + "assets_dir": "/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]", + "output_schema_name": "main.qm_test_[UNIQUE_NAME]_2", + "snapshot": {}, + "table_name": "main.qm_test_[UNIQUE_NAME].test_table", + "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" + } + }, + "remote_state": { + "assets_dir": "/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]", + "dashboard_id": "(redacted)", + "drift_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_drift_metrics", + "monitor_version": 0, + "output_schema_name": "main.qm_test_[UNIQUE_NAME]", + "profile_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_profile_metrics", + "snapshot": {}, + "status": "MONITOR_STATUS_ACTIVE", + "table_name": "main.qm_test_[UNIQUE_NAME].test_table" + }, + "changes": { + "output_schema_name": { + "action": "update", + "old": "main.qm_test_[UNIQUE_NAME]", + "new": "main.qm_test_[UNIQUE_NAME]_2", + "remote": "main.qm_test_[UNIQUE_NAME]" + }, + "warehouse_id": { + "action": "skip", + "reason": "config_only", + "old": "[TEST_DEFAULT_WAREHOUSE_ID]", + "new": "[TEST_DEFAULT_WAREHOUSE_ID]" + } + } + } + } +} diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.terraform.json b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.terraform.json new file mode 100644 index 0000000000..79770400d2 --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.terraform.json @@ -0,0 +1,9 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.quality_monitors.monitor1": { + "action": "update" + } + } +} diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.test.toml new file mode 100644 index 0000000000..d61c11e25c --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt new file mode 100644 index 0000000000..d67ee41975 --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt @@ -0,0 +1,51 @@ + +>>> [CLI] schemas create qm_test_[UNIQUE_NAME] main -o json +{ + "full_name": "main.qm_test_[UNIQUE_NAME]" +} + +>>> [CLI] schemas create qm_test_[UNIQUE_NAME]_2 main -o json +{ + "full_name": "main.qm_test_[UNIQUE_NAME]_2" +} + +>>> create_table.py main.qm_test_[UNIQUE_NAME].test_table +Created table main.qm_test_[UNIQUE_NAME].test_table +Inserted sample data into main.qm_test_[UNIQUE_NAME].test_table +Table main.qm_test_[UNIQUE_NAME].test_table is now visible (catalog_name=main) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/quality-monitor-update-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle plan +update quality_monitors.monitor1 + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/quality-monitor-update-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.quality_monitors.monitor1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/quality-monitor-update-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script new file mode 100644 index 0000000000..72c57e0840 --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script @@ -0,0 +1,32 @@ +SCHEMA_NAME="main.qm_test_${UNIQUE_NAME}" +TABLE_NAME="${SCHEMA_NAME}.test_table" + +trace $CLI schemas create "qm_test_${UNIQUE_NAME}" main -o json | jq '{full_name}' +trace $CLI schemas create "qm_test_${UNIQUE_NAME}_2" main -o json | jq '{full_name}' +trace create_table.py "$TABLE_NAME" + +cleanup() { + trace $CLI bundle destroy --auto-approve + $CLI schemas delete "$SCHEMA_NAME" --force 2>/dev/null || true + $CLI schemas delete "qm_test_${UNIQUE_NAME}_2" --force 2>/dev/null || true + rm -f out.requests.txt +} +trap cleanup EXIT + +envsubst < databricks.yml.tmpl > databricks.yml + +trace $CLI bundle deploy + +trace $CLI bundle plan | contains.py "1 unchanged" + +update_file.py databricks.yml "output_schema_name: main.qm_test_${UNIQUE_NAME}" "output_schema_name: main.qm_test_${UNIQUE_NAME}_2" + +trace $CLI bundle plan | contains.py "1 to change" +trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json + +rm out.requests.txt +trace $CLI bundle deploy +# dashboard_id is output only field that terraform adds +trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' | grep -v '"dashboard_id":' > out.deploy.requests.json + +trace $CLI bundle plan | contains.py "1 unchanged" diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/databricks.yml.tmpl b/acceptance/bundle/resources/quality_monitors/change_table_name/databricks.yml.tmpl new file mode 100644 index 0000000000..3fe5194f1c --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: quality-monitor-update-$UNIQUE_NAME + +resources: + quality_monitors: + monitor1: + table_name: main.qm_test_$UNIQUE_NAME.test_table + assets_dir: /Workspace/Users/$CURRENT_USER_NAME/monitor_assets_$UNIQUE_NAME + output_schema_name: main.qm_test_$UNIQUE_NAME + snapshot: {} + warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/out.deploy.direct.txt b/acceptance/bundle/resources/quality_monitors/change_table_name/out.deploy.direct.txt new file mode 100644 index 0000000000..1268e736e1 --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/out.deploy.direct.txt @@ -0,0 +1,6 @@ + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/quality-monitor-update-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/out.deploy.requests.direct.json b/acceptance/bundle/resources/quality_monitors/change_table_name/out.deploy.requests.direct.json new file mode 100644 index 0000000000..5561c70392 --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/out.deploy.requests.direct.json @@ -0,0 +1,14 @@ +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/tables/main.qm_test_[UNIQUE_NAME].test_table/monitor" +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/tables/main.qm_test_[UNIQUE_NAME].test_table_2/monitor", + "body": { + "assets_dir": "/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]", + "output_schema_name": "main.qm_test_[UNIQUE_NAME]", + "snapshot": {}, + "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" + } +} diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/out.get.direct.json b/acceptance/bundle/resources/quality_monitors/change_table_name/out.get.direct.json new file mode 100644 index 0000000000..feb478a878 --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/out.get.direct.json @@ -0,0 +1,11 @@ +{ + "assets_dir":"/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]", + "dashboard_id": "(redacted)", + "drift_metrics_table_name":"main.qm_test_[UNIQUE_NAME].test_table_2_drift_metrics", + "monitor_version":0, + "output_schema_name":"main.qm_test_[UNIQUE_NAME]", + "profile_metrics_table_name":"main.qm_test_[UNIQUE_NAME].test_table_2_profile_metrics", + "snapshot": {}, + "status":"MONITOR_STATUS_ACTIVE", + "table_name":"main.qm_test_[UNIQUE_NAME].test_table_2" +} diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json b/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json new file mode 100644 index 0000000000..785e60072b --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json @@ -0,0 +1,48 @@ + +>>> errcode [CLI] bundle plan -o json +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "plan": { + "resources.quality_monitors.monitor1": { + "action": "recreate", + "new_state": { + "value": { + "assets_dir": "/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]", + "output_schema_name": "main.qm_test_[UNIQUE_NAME]", + "snapshot": {}, + "table_name": "main.qm_test_[UNIQUE_NAME].test_table_2", + "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" + } + }, + "remote_state": { + "assets_dir": "/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]", + "dashboard_id": "(redacted)", + "drift_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_drift_metrics", + "monitor_version": 0, + "output_schema_name": "main.qm_test_[UNIQUE_NAME]", + "profile_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_profile_metrics", + "snapshot": {}, + "status": "MONITOR_STATUS_ACTIVE", + "table_name": "main.qm_test_[UNIQUE_NAME].test_table" + }, + "changes": { + "table_name": { + "action": "recreate", + "reason": "field_triggers", + "old": "main.qm_test_[UNIQUE_NAME].test_table", + "new": "main.qm_test_[UNIQUE_NAME].test_table_2", + "remote": "main.qm_test_[UNIQUE_NAME].test_table" + }, + "warehouse_id": { + "action": "skip", + "reason": "config_only", + "old": "[TEST_DEFAULT_WAREHOUSE_ID]", + "new": "[TEST_DEFAULT_WAREHOUSE_ID]" + } + } + } + } +} diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.txt b/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.txt new file mode 100644 index 0000000000..c812b16bfa --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.txt @@ -0,0 +1,5 @@ + +>>> errcode [CLI] bundle plan +recreate quality_monitors.monitor1 + +Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan_after_deploy.direct.txt b/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan_after_deploy.direct.txt new file mode 100644 index 0000000000..67533bda2c --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan_after_deploy.direct.txt @@ -0,0 +1,3 @@ + +>>> errcode [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml new file mode 100644 index 0000000000..f1d40380d0 --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt b/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt new file mode 100644 index 0000000000..879222aa8a --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt @@ -0,0 +1,35 @@ + +>>> [CLI] schemas create qm_test_[UNIQUE_NAME] main -o json +{ + "full_name": "main.qm_test_[UNIQUE_NAME]" +} + +>>> create_table.py main.qm_test_[UNIQUE_NAME].test_table +Created table main.qm_test_[UNIQUE_NAME].test_table +Inserted sample data into main.qm_test_[UNIQUE_NAME].test_table +Table main.qm_test_[UNIQUE_NAME].test_table is now visible (catalog_name=main) + +>>> create_table.py main.qm_test_[UNIQUE_NAME].test_table_2 +Created table main.qm_test_[UNIQUE_NAME].test_table_2 +Inserted sample data into main.qm_test_[UNIQUE_NAME].test_table_2 +Table main.qm_test_[UNIQUE_NAME].test_table_2 is now visible (catalog_name=main) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/quality-monitor-update-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.quality_monitors.monitor1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/quality-monitor-update-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/script b/acceptance/bundle/resources/quality_monitors/change_table_name/script new file mode 100644 index 0000000000..891aece1c1 --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/script @@ -0,0 +1,32 @@ +SCHEMA_NAME="main.qm_test_${UNIQUE_NAME}" +TABLE_NAME="${SCHEMA_NAME}.test_table" + +trace $CLI schemas create "qm_test_${UNIQUE_NAME}" main -o json | jq '{full_name}' +trace create_table.py "$TABLE_NAME" +trace create_table.py "${TABLE_NAME}_2" + +cleanup() { + trace $CLI bundle destroy --auto-approve + $CLI schemas delete "$SCHEMA_NAME" --force 2>/dev/null || true + $CLI schemas delete "qm_test_${UNIQUE_NAME}_2" --force 2>/dev/null || true + rm -f out.requests.txt +} +trap cleanup EXIT + +envsubst < databricks.yml.tmpl > databricks.yml + +trace $CLI bundle deploy + +trace $CLI bundle plan | contains.py "1 unchanged" + +update_file.py databricks.yml "table_name: $TABLE_NAME" "table_name: ${TABLE_NAME}_2" + +trace errcode $CLI bundle plan &> out.plan.$DATABRICKS_BUNDLE_ENGINE.txt +trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.json + +rm out.requests.txt +trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt +trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace errcode $CLI bundle plan &> out.plan_after_deploy.$DATABRICKS_BUNDLE_ENGINE.txt + +trace errcode $CLI quality-monitors get ${TABLE_NAME}_2 2> /dev/null > out.get.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/test.toml b/acceptance/bundle/resources/quality_monitors/change_table_name/test.toml new file mode 100644 index 0000000000..7e03ffedd9 --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/test.toml @@ -0,0 +1,2 @@ +# this does not work on terraform +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/quality_monitors/create/databricks.yml.tmpl b/acceptance/bundle/resources/quality_monitors/create/databricks.yml.tmpl new file mode 100644 index 0000000000..3fe5194f1c --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/create/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: quality-monitor-update-$UNIQUE_NAME + +resources: + quality_monitors: + monitor1: + table_name: main.qm_test_$UNIQUE_NAME.test_table + assets_dir: /Workspace/Users/$CURRENT_USER_NAME/monitor_assets_$UNIQUE_NAME + output_schema_name: main.qm_test_$UNIQUE_NAME + snapshot: {} + warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID diff --git a/acceptance/bundle/resources/quality_monitors/create/out.deploy.direct.txt b/acceptance/bundle/resources/quality_monitors/create/out.deploy.direct.txt new file mode 100644 index 0000000000..af5c31de3b --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/create/out.deploy.direct.txt @@ -0,0 +1,6 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/quality-monitor-update-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/bundle/resources/quality_monitors/create/out.deploy.requests.json b/acceptance/bundle/resources/quality_monitors/create/out.deploy.requests.json new file mode 100644 index 0000000000..9a7f0f6a2a --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/create/out.deploy.requests.json @@ -0,0 +1,10 @@ +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/tables/main.qm_test_[UNIQUE_NAME].test_table/monitor", + "body": { + "assets_dir": "/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]", + "output_schema_name": "main.qm_test_[UNIQUE_NAME]", + "snapshot": {}, + "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" + } +} diff --git a/acceptance/bundle/resources/quality_monitors/create/out.deploy.terraform.txt b/acceptance/bundle/resources/quality_monitors/create/out.deploy.terraform.txt new file mode 100644 index 0000000000..af5c31de3b --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/create/out.deploy.terraform.txt @@ -0,0 +1,6 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/quality-monitor-update-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/bundle/resources/quality_monitors/create/out.plan_create.direct.json b/acceptance/bundle/resources/quality_monitors/create/out.plan_create.direct.json new file mode 100644 index 0000000000..2c81e96864 --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/create/out.plan_create.direct.json @@ -0,0 +1,18 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.quality_monitors.monitor1": { + "action": "create", + "new_state": { + "value": { + "assets_dir": "/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]", + "output_schema_name": "main.qm_test_[UNIQUE_NAME]", + "snapshot": {}, + "table_name": "main.qm_test_[UNIQUE_NAME].test_table", + "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" + } + } + } + } +} diff --git a/acceptance/bundle/resources/quality_monitors/create/out.plan_create.terraform.json b/acceptance/bundle/resources/quality_monitors/create/out.plan_create.terraform.json new file mode 100644 index 0000000000..eac4e46b7b --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/create/out.plan_create.terraform.json @@ -0,0 +1,9 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.quality_monitors.monitor1": { + "action": "create" + } + } +} diff --git a/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json b/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json new file mode 100644 index 0000000000..4c84c7679b --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json @@ -0,0 +1,30 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "plan": { + "resources.quality_monitors.monitor1": { + "action": "skip", + "remote_state": { + "assets_dir": "/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]", + "dashboard_id": "(redacted)", + "drift_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_drift_metrics", + "monitor_version": 0, + "output_schema_name": "main.qm_test_[UNIQUE_NAME]", + "profile_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_profile_metrics", + "snapshot": {}, + "status": "MONITOR_STATUS_ACTIVE", + "table_name": "main.qm_test_[UNIQUE_NAME].test_table" + }, + "changes": { + "warehouse_id": { + "action": "skip", + "reason": "config_only", + "old": "[TEST_DEFAULT_WAREHOUSE_ID]", + "new": "[TEST_DEFAULT_WAREHOUSE_ID]" + } + } + } + } +} diff --git a/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.terraform.json b/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.terraform.json new file mode 100644 index 0000000000..80bbba8d9f --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.terraform.json @@ -0,0 +1,9 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.quality_monitors.monitor1": { + "action": "skip" + } + } +} diff --git a/acceptance/bundle/resources/quality_monitors/create/out.state.direct.txt b/acceptance/bundle/resources/quality_monitors/create/out.state.direct.txt new file mode 100644 index 0000000000..a09fecc62e --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/create/out.state.direct.txt @@ -0,0 +1,2 @@ + "output_schema_name": "main.qm_test_[UNIQUE_NAME]", + "table_name": "main.qm_test_[UNIQUE_NAME].test_table", diff --git a/acceptance/bundle/resources/quality_monitors/create/out.state.terraform.txt b/acceptance/bundle/resources/quality_monitors/create/out.state.terraform.txt new file mode 100644 index 0000000000..dbdd758797 --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/create/out.state.terraform.txt @@ -0,0 +1,6 @@ + "name": "monitor1", + "baseline_table_name": null, + "drift_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_drift_metrics", + "output_schema_name": "main.qm_test_[UNIQUE_NAME]", + "profile_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_profile_metrics", + "table_name": "main.qm_test_[UNIQUE_NAME].test_table", diff --git a/acceptance/bundle/resources/quality_monitors/create/out.test.toml b/acceptance/bundle/resources/quality_monitors/create/out.test.toml new file mode 100644 index 0000000000..d61c11e25c --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/create/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/quality_monitors/create/output.txt b/acceptance/bundle/resources/quality_monitors/create/output.txt new file mode 100644 index 0000000000..8037d5ec9c --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/create/output.txt @@ -0,0 +1,30 @@ + +>>> [CLI] schemas create qm_test_[UNIQUE_NAME] main -o json +{ + "full_name": "main.qm_test_[UNIQUE_NAME]" +} + +>>> [CLI] schemas create qm_test_[UNIQUE_NAME]_2 main -o json +{ + "full_name": "main.qm_test_[UNIQUE_NAME]_2" +} + +>>> create_table.py main.qm_test_[UNIQUE_NAME].test_table +Created table main.qm_test_[UNIQUE_NAME].test_table +Inserted sample data into main.qm_test_[UNIQUE_NAME].test_table +Table main.qm_test_[UNIQUE_NAME].test_table is now visible (catalog_name=main) + +>>> [CLI] bundle plan -o json + +>>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.quality_monitors.monitor1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/quality-monitor-update-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/quality_monitors/create/script b/acceptance/bundle/resources/quality_monitors/create/script new file mode 100644 index 0000000000..78c7853b26 --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/create/script @@ -0,0 +1,29 @@ +SCHEMA_NAME="main.qm_test_${UNIQUE_NAME}" +TABLE_NAME="${SCHEMA_NAME}.test_table" +OUTPUT_SCHEMA1="${SCHEMA_NAME}" + +trace $CLI schemas create "qm_test_${UNIQUE_NAME}" main -o json | jq '{full_name}' +trace $CLI schemas create "qm_test_${UNIQUE_NAME}_2" main -o json | jq '{full_name}' +trace create_table.py "$TABLE_NAME" + +cleanup() { + trace $CLI bundle destroy --auto-approve + # Clean up schemas and table (suppress errors since they may already be gone) + $CLI schemas delete "$SCHEMA_NAME" --force 2>/dev/null || true + $CLI schemas delete "qm_test_${UNIQUE_NAME}_2" --force 2>/dev/null || true + rm -f out.requests.txt +} +trap cleanup EXIT + +envsubst < databricks.yml.tmpl > databricks.yml + +trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json + +rm out.requests.txt +trace $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt +trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json + +# store state to ensure we have table_name there +print_state.py | grep name > out.state.$DATABRICKS_BUNDLE_ENGINE.txt + +trace $CLI bundle plan -o json > out.plan_noop.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/quality_monitors/test.toml b/acceptance/bundle/resources/quality_monitors/test.toml new file mode 100644 index 0000000000..e0d535ddc2 --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/test.toml @@ -0,0 +1,17 @@ +# Test quality monitor output_schema_name updates +Cloud = true +Local = true +RequiresUnityCatalog = true +RecordRequests = true + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + +# Fake SQL endpoint for local tests +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' + + +[[Repls]] +Old = '"dashboard_id":\s*"[0-9a-f]+",' +New = '"dashboard_id": "(redacted)",' diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index 6bf7184526..d72274b16f 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -92,6 +92,7 @@ const ( ReasonAlias = "alias" ReasonRemoteAlreadySet = "remote_already_set" ReasonFieldTriggers = "field_triggers" + ReasonConfigOnly = "config_only" ) // HasChange checks if there are any changes for fields with the given prefix. diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index 6570fcdeea..eb37628961 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -24,6 +24,7 @@ var SupportedResources = map[string]any{ "dashboards": (*ResourceDashboard)(nil), "secret_scopes": (*ResourceSecretScope)(nil), "model_serving_endpoints": (*ResourceModelServingEndpoint)(nil), + "quality_monitors": (*ResourceQualityMonitor)(nil), // Permissions "jobs.permissions": (*ResourcePermissions)(nil), diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 001dd7af46..d6f48c812b 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -183,6 +183,14 @@ var testConfig map[string]any = map[string]any{ }, }, }, + + "quality_monitors": &resources.QualityMonitor{ + TableName: "main.myschema.mytable", + CreateMonitor: catalog.CreateMonitor{ + AssetsDir: "/Workspace/Users/user@example.com/assets", + OutputSchemaName: "main.myschema", + }, + }, } type prepareWorkspace func(client *databricks.WorkspaceClient) (any, error) diff --git a/bundle/direct/dresources/quality_monitor.go b/bundle/direct/dresources/quality_monitor.go new file mode 100644 index 0000000000..28d57085b5 --- /dev/null +++ b/bundle/direct/dresources/quality_monitor.go @@ -0,0 +1,132 @@ +package dresources + +import ( + "context" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/utils" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/catalog" +) + +type QualityMonitorState struct { + catalog.CreateMonitor + + // The table name is a required field but not included as a JSON field in [catalog.CreateMonitor]. + TableName string `json:"table_name"` +} + +// We need to provide these custom marshaller from CreateMonitor takes over and ignores TableName field +func (s *QualityMonitorState) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s QualityMonitorState) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +type ResourceQualityMonitor struct { + client *databricks.WorkspaceClient +} + +func (*ResourceQualityMonitor) New(client *databricks.WorkspaceClient) *ResourceQualityMonitor { + return &ResourceQualityMonitor{client: client} +} + +func (*ResourceQualityMonitor) PrepareState(input *resources.QualityMonitor) *QualityMonitorState { + return &QualityMonitorState{ + CreateMonitor: input.CreateMonitor, + TableName: input.TableName, + } +} + +func (*ResourceQualityMonitor) RemapState(info *catalog.MonitorInfo) *QualityMonitorState { + return &QualityMonitorState{ + CreateMonitor: catalog.CreateMonitor{ + AssetsDir: info.AssetsDir, + BaselineTableName: info.BaselineTableName, + CustomMetrics: info.CustomMetrics, + DataClassificationConfig: info.DataClassificationConfig, + InferenceLog: info.InferenceLog, + LatestMonitorFailureMsg: info.LatestMonitorFailureMsg, + Notifications: info.Notifications, + OutputSchemaName: info.OutputSchemaName, + Schedule: info.Schedule, + SkipBuiltinDashboard: false, + SlicingExprs: info.SlicingExprs, + Snapshot: info.Snapshot, + TableName: info.TableName, + TimeSeries: info.TimeSeries, + WarehouseId: "", + ForceSendFields: utils.FilterFields[catalog.CreateMonitor](info.ForceSendFields), + }, + TableName: info.TableName, + } +} + +func (r *ResourceQualityMonitor) DoRead(ctx context.Context, id string) (*catalog.MonitorInfo, error) { + return r.client.QualityMonitors.Get(ctx, catalog.GetQualityMonitorRequest{ + TableName: id, + }) +} + +func (r *ResourceQualityMonitor) DoCreate(ctx context.Context, config *QualityMonitorState) (string, *catalog.MonitorInfo, error) { + req := config.CreateMonitor + req.TableName = config.TableName + response, err := r.client.QualityMonitors.Create(ctx, req) + if err != nil || response == nil { + return "", nil, err + } + return response.TableName, response, nil +} + +func (r *ResourceQualityMonitor) DoUpdate(ctx context.Context, id string, config *QualityMonitorState, _ Changes) (*catalog.MonitorInfo, error) { + updateRequest := catalog.UpdateMonitor{ + TableName: id, + BaselineTableName: config.BaselineTableName, + CustomMetrics: config.CustomMetrics, + DashboardId: "", + DataClassificationConfig: config.DataClassificationConfig, + InferenceLog: config.InferenceLog, + LatestMonitorFailureMsg: "", + Notifications: config.Notifications, + OutputSchemaName: config.OutputSchemaName, + Schedule: config.Schedule, + SlicingExprs: config.SlicingExprs, + Snapshot: config.Snapshot, + TimeSeries: config.TimeSeries, + ForceSendFields: utils.FilterFields[catalog.UpdateMonitor](config.ForceSendFields), + } + + response, err := r.client.QualityMonitors.Update(ctx, updateRequest) + if err != nil { + return nil, err + } + + return response, nil +} + +func (r *ResourceQualityMonitor) DoDelete(ctx context.Context, id string) error { + _, err := r.client.QualityMonitors.Delete(ctx, catalog.DeleteQualityMonitorRequest{ + TableName: id, + }) + return err +} + +func (*ResourceQualityMonitor) FieldTriggers() map[string]deployplan.ActionType { + return map[string]deployplan.ActionType{ + "assets_dir": deployplan.Recreate, + "table_name": deployplan.Recreate, + } +} + +func (r *ResourceQualityMonitor) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, _ *catalog.MonitorInfo) error { + if path.String() == "warehouse_id" && change.Old == change.New { + change.Action = deployplan.Skip + change.Reason = deployplan.ReasonConfigOnly + } + return nil +} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 6c8d1bbf58..17f9ab4ac0 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -95,6 +95,13 @@ func nextUUID() string { return u.String() } +func nextDashboardID() string { + var b [16]byte + binary.BigEndian.PutUint64(b[0:8], uint64(nextID())) + binary.BigEndian.PutUint64(b[8:16], uint64(nextID())) + return fmt.Sprintf("%032x", b) +} + type FileEntry struct { Info workspace.ObjectInfo Data []byte diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index adbfc3e5ca..616bd96ef5 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "net/http" + "strings" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/compute" @@ -283,6 +284,22 @@ func AddDefaultHandlers(server *Server) { // Quality monitors: + // Simple handler for table existence check - returns a basic table info + server.Handle("GET", "/api/2.1/unity-catalog/tables/{full_name}", func(req Request) any { + parts := strings.Split(req.Vars["full_name"], ".") + if len(parts) != 3 { + return Response{StatusCode: 400, Body: "Invalid table name"} + } + return Response{ + Body: catalog.TableInfo{ + CatalogName: parts[0], + SchemaName: parts[1], + Name: parts[2], + FullName: req.Vars["full_name"], + }, + } + }) + server.Handle("GET", "/api/2.1/unity-catalog/tables/{table_name}/monitor", func(req Request) any { return MapGet(req.Workspace, req.Workspace.Monitors, req.Vars["table_name"]) }) diff --git a/libs/testserver/quality_monitors.go b/libs/testserver/quality_monitors.go index 3580d534e3..f9d7a0e013 100644 --- a/libs/testserver/quality_monitors.go +++ b/libs/testserver/quality_monitors.go @@ -30,12 +30,14 @@ func (s *FakeWorkspace) QualityMonitorUpsert(req Request, tableName string, chec defer s.LockUnlock()() if checkExists { - _, ok := s.Monitors[tableName] + existing, ok := s.Monitors[tableName] if !ok { return Response{ StatusCode: 404, } } + // For updates, preserve fields that cannot be changed after creation + info.AssetsDir = existing.AssetsDir } if info.Status == "" { @@ -46,6 +48,18 @@ func (s *FakeWorkspace) QualityMonitorUpsert(req Request, tableName string, chec info.TableName = tableName } + if info.DriftMetricsTableName == "" { + info.DriftMetricsTableName = tableName + "_drift_metrics" + } + + if info.ProfileMetricsTableName == "" { + info.ProfileMetricsTableName = tableName + "_profile_metrics" + } + + if info.DashboardId == "" { + info.DashboardId = nextDashboardID() + } + s.Monitors[tableName] = info return Response{ Body: info,