From 95b5a0a2ac457ff5b0258845eb3e2c790cfaea41 Mon Sep 17 00:00:00 2001 From: VladaZakharova <80038284+VladaZakharova@users.noreply.github.com> Date: Wed, 24 Jul 2024 16:01:54 +0200 Subject: [PATCH] Refactor dataproc system tests (#40720) --- .../providers/google/cloud/hooks/dataproc.py | 2 +- .../google/cloud/operators/dataproc.py | 2 +- .../cloud/operators/dataproc_metastore.py | 10 +- scripts/ci/pre_commit/check_system_tests.py | 3 +- .../cloud/dataproc/example_dataproc_batch.py | 35 +- .../example_dataproc_batch_deferrable.py | 2 +- .../example_dataproc_batch_persistent.py | 23 +- ...cluster_create_existing_stopped_cluster.py | 14 +- .../example_dataproc_cluster_deferrable.py | 8 +- .../example_dataproc_cluster_diagnose.py | 11 +- .../example_dataproc_cluster_generator.py | 46 +- .../example_dataproc_cluster_start_stop.py | 11 +- .../example_dataproc_cluster_update.py | 8 +- .../cloud/dataproc/example_dataproc_gke.py | 16 +- .../cloud/dataproc/example_dataproc_hadoop.py | 6 +- .../cloud/dataproc/example_dataproc_hive.py | 6 +- .../cloud/dataproc/example_dataproc_pig.py | 6 +- .../cloud/dataproc/example_dataproc_presto.py | 6 +- .../dataproc/example_dataproc_pyspark.py | 42 +- .../cloud/dataproc/example_dataproc_spark.py | 7 +- .../dataproc/example_dataproc_spark_async.py | 7 +- .../example_dataproc_spark_deferrable.py | 7 +- .../dataproc/example_dataproc_spark_sql.py | 7 +- .../cloud/dataproc/example_dataproc_sparkr.py | 40 +- .../cloud/dataproc/example_dataproc_trino.py | 6 +- .../dataproc/example_dataproc_workflow.py | 7 +- .../example_dataproc_workflow_deferrable.py | 6 +- .../cloud/dataproc/resources/__init__.py | 16 - .../cloud/dataproc/resources/hello_world.R | 9 - .../cloud/dataproc/resources/hello_world.py | 25 - .../google/cloud/dataproc/resources/hive.sql | 2188 ----------------- .../cloud/dataproc/resources/pip-install.sh | 69 - .../example_dataproc_metastore.py | 34 +- .../example_dataproc_metastore_backup.py | 5 +- ...ataproc_metastore_hive_partition_sensor.py | 5 +- 35 files changed, 264 insertions(+), 2431 deletions(-) delete mode 100644 tests/system/providers/google/cloud/dataproc/resources/__init__.py delete mode 100644 tests/system/providers/google/cloud/dataproc/resources/hello_world.R delete mode 100644 tests/system/providers/google/cloud/dataproc/resources/hello_world.py delete mode 100644 tests/system/providers/google/cloud/dataproc/resources/hive.sql delete mode 100644 tests/system/providers/google/cloud/dataproc/resources/pip-install.sh diff --git a/airflow/providers/google/cloud/hooks/dataproc.py b/airflow/providers/google/cloud/hooks/dataproc.py index fc7f92c906076..2eb8d8952c56e 100644 --- a/airflow/providers/google/cloud/hooks/dataproc.py +++ b/airflow/providers/google/cloud/hooks/dataproc.py @@ -274,7 +274,7 @@ def wait_for_operation( self, operation: Operation, timeout: float | None = None, - result_retry: AsyncRetry | _MethodDefault = DEFAULT, + result_retry: AsyncRetry | _MethodDefault | Retry = DEFAULT, ) -> Any: """Wait for a long-lasting operation to complete.""" try: diff --git a/airflow/providers/google/cloud/operators/dataproc.py b/airflow/providers/google/cloud/operators/dataproc.py index 889f8b3e596ab..efaf0d6326d08 100644 --- a/airflow/providers/google/cloud/operators/dataproc.py +++ b/airflow/providers/google/cloud/operators/dataproc.py @@ -2995,7 +2995,7 @@ def __init__( metadata: Sequence[tuple[str, str]] = (), gcp_conn_id: str = "google_cloud_default", impersonation_chain: str | Sequence[str] | None = None, - result_retry: AsyncRetry | _MethodDefault = DEFAULT, + result_retry: AsyncRetry | _MethodDefault | Retry = DEFAULT, asynchronous: bool = False, deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), polling_interval_seconds: int = 5, diff --git a/airflow/providers/google/cloud/operators/dataproc_metastore.py b/airflow/providers/google/cloud/operators/dataproc_metastore.py index 2da7ce9269059..aa7dbafd22617 100644 --- a/airflow/providers/google/cloud/operators/dataproc_metastore.py +++ b/airflow/providers/google/cloud/operators/dataproc_metastore.py @@ -22,12 +22,12 @@ import time from typing import TYPE_CHECKING, Sequence +from google.api_core.exceptions import AlreadyExists from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault from google.api_core.retry import Retry, exponential_sleep_generator from google.cloud.metastore_v1 import MetadataExport, MetadataManagementActivity from google.cloud.metastore_v1.types import Backup, MetadataImport, Service from google.cloud.metastore_v1.types.metastore import DatabaseDumpSpec, Restore -from googleapiclient.errors import HttpError from airflow.exceptions import AirflowException from airflow.models import BaseOperator, BaseOperatorLink @@ -242,9 +242,7 @@ def execute(self, context: Context) -> dict: ) backup = hook.wait_for_operation(self.timeout, operation) self.log.info("Backup %s created successfully", self.backup_id) - except HttpError as err: - if err.resp.status not in (409, "409"): - raise + except AlreadyExists: self.log.info("Backup %s already exists", self.backup_id) backup = hook.get_backup( project_id=self.project_id, @@ -448,9 +446,7 @@ def execute(self, context: Context) -> dict: ) service = hook.wait_for_operation(self.timeout, operation) self.log.info("Service %s created successfully", self.service_id) - except HttpError as err: - if err.resp.status not in (409, "409"): - raise + except AlreadyExists: self.log.info("Instance %s already exists", self.service_id) service = hook.get_service( region=self.region, diff --git a/scripts/ci/pre_commit/check_system_tests.py b/scripts/ci/pre_commit/check_system_tests.py index 4c82272ad7875..89e2a9f24ae5c 100755 --- a/scripts/ci/pre_commit/check_system_tests.py +++ b/scripts/ci/pre_commit/check_system_tests.py @@ -35,6 +35,7 @@ errors: list[str] = [] WATCHER_APPEND_INSTRUCTION = "list(dag.tasks) >> watcher()" +WATCHER_APPEND_INSTRUCTION_SHORT = " >> watcher()" PYTEST_FUNCTION = """ from tests.system.utils import get_test_run # noqa: E402 @@ -52,7 +53,7 @@ def _check_file(file: Path): content = file.read_text() if "from tests.system.utils.watcher import watcher" in content: - index = content.find(WATCHER_APPEND_INSTRUCTION) + index = content.find(WATCHER_APPEND_INSTRUCTION_SHORT) if index == -1: errors.append( f"[red]The example {file} imports tests.system.utils.watcher " diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_batch.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_batch.py index a6001c43ffd95..852fa0914d089 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_batch.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_batch.py @@ -23,7 +23,7 @@ import os from datetime import datetime -from google.api_core.retry_async import AsyncRetry +from google.api_core.retry import Retry from airflow.models.dag import DAG from airflow.providers.google.cloud.operators.dataproc import ( @@ -37,10 +37,10 @@ from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") -DAG_ID = "dataproc_batch" +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -REGION = "europe-west1" +DAG_ID = "dataproc_batch" +REGION = "europe-west3" BATCH_ID = f"batch-{ENV_ID}-{DAG_ID}".replace("_", "-") BATCH_ID_2 = f"batch-{ENV_ID}-{DAG_ID}-2".replace("_", "-") @@ -77,7 +77,7 @@ region=REGION, batch=BATCH_CONFIG, batch_id=BATCH_ID_2, - result_retry=AsyncRetry(maximum=10.0, initial=10.0, multiplier=1.0), + result_retry=Retry(maximum=100.0, initial=10.0, multiplier=1.0), ) create_batch_3 = DataprocCreateBatchOperator( @@ -104,10 +104,6 @@ get_batch = DataprocGetBatchOperator( task_id="get_batch", project_id=PROJECT_ID, region=REGION, batch_id=BATCH_ID ) - - get_batch_2 = DataprocGetBatchOperator( - task_id="get_batch_2", project_id=PROJECT_ID, region=REGION, batch_id=BATCH_ID_2 - ) # [END how_to_cloud_dataproc_get_batch_operator] # [START how_to_cloud_dataproc_list_batches_operator] @@ -136,6 +132,14 @@ ) # [END how_to_cloud_dataproc_cancel_operation_operator] + batch_cancelled_sensor = DataprocBatchSensor( + task_id="batch_cancelled_sensor", + region=REGION, + project_id=PROJECT_ID, + batch_id=BATCH_ID_4, + poke_interval=10, + ) + # [START how_to_cloud_dataproc_delete_batch_operator] delete_batch = DataprocDeleteBatchOperator( task_id="delete_batch", project_id=PROJECT_ID, region=REGION, batch_id=BATCH_ID @@ -153,25 +157,30 @@ delete_batch.trigger_rule = TriggerRule.ALL_DONE delete_batch_2.trigger_rule = TriggerRule.ALL_DONE delete_batch_3.trigger_rule = TriggerRule.ALL_DONE - delete_batch_4.trigger_rule = TriggerRule.ALL_DONE + delete_batch_4.trigger_rule = TriggerRule.ALL_FAILED ( # TEST SETUP [create_batch, create_batch_2, create_batch_3] # TEST BODY >> batch_async_sensor - >> [get_batch, get_batch_2, list_batches] + >> get_batch + >> list_batches >> create_batch_4 >> cancel_operation # TEST TEARDOWN - >> [delete_batch, delete_batch_2, delete_batch_3, delete_batch_4] + >> [delete_batch, delete_batch_2, delete_batch_3] + >> batch_cancelled_sensor + >> delete_batch_4 ) from tests.system.utils.watcher import watcher # This test needs watcher in order to properly mark success/failure # when "teardown" task with trigger rule is part of the DAG - list(dag.tasks) >> watcher() + + # Excluding sensor because we expect it to fail due to cancelled operation + [task for task in dag.tasks if task.task_id != "batch_cancelled_sensor"] >> watcher() from tests.system.utils import get_test_run # noqa: E402 diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_batch_deferrable.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_batch_deferrable.py index ad0d513a7a520..1d2fec951e18f 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_batch_deferrable.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_batch_deferrable.py @@ -37,7 +37,7 @@ ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") DAG_ID = "dataproc_batch_deferrable" PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -REGION = "europe-west1" +REGION = "europe-north1" BATCH_ID = f"batch-{ENV_ID}-{DAG_ID}".replace("_", "-") BATCH_CONFIG = { "spark_batch": { diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_batch_persistent.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_batch_persistent.py index 1cf300afe7a20..cb4e731c6785b 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_batch_persistent.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_batch_persistent.py @@ -28,17 +28,21 @@ ClusterGenerator, DataprocCreateBatchOperator, DataprocCreateClusterOperator, + DataprocDeleteBatchOperator, DataprocDeleteClusterOperator, ) from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.utils.trigger_rule import TriggerRule +from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_batch_ps" -PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default") -BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}" -REGION = "europe-west1" -CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "-") +PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID +BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}".replace("-", "_") +REGION = "europe-north1" +CLUSTER_NAME_BASE = f"cluster-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-") +CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL BATCH_ID = f"batch-{ENV_ID}-{DAG_ID}".replace("_", "-") CLUSTER_GENERATOR_CONFIG_FOR_PHS = ClusterGenerator( @@ -98,6 +102,14 @@ ) # [END how_to_cloud_dataproc_create_batch_operator_with_persistent_history_server] + delete_batch = DataprocDeleteBatchOperator( + task_id="delete_batch", + project_id=PROJECT_ID, + region=REGION, + batch_id=BATCH_ID, + trigger_rule=TriggerRule.ALL_DONE, + ) + delete_cluster = DataprocDeleteClusterOperator( task_id="delete_cluster", project_id=PROJECT_ID, @@ -117,6 +129,7 @@ # TEST BODY >> create_batch # TEST TEARDOWN + >> delete_batch >> delete_cluster >> delete_bucket ) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_create_existing_stopped_cluster.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_create_existing_stopped_cluster.py index 31d1c51962703..10403cea3b065 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_create_existing_stopped_cluster.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_create_existing_stopped_cluster.py @@ -32,14 +32,18 @@ DataprocStopClusterOperator, ) from airflow.utils.trigger_rule import TriggerRule +from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -DAG_ID = "example_dataproc_cluster_create_existing_stopped_cluster" +DAG_ID = "dataproc_create_existing_stopped_cluster" -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") -PROJECT_ID = os.environ.get("SYSTEMS_TESTS_GCP_PROJECTS") or "" +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") +PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "-") -REGION = "europe-west1" + +CLUSTER_NAME_BASE = f"{DAG_ID}".replace("_", "-") +CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-") +CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL +REGION = "europe-north1" # Cluster definition CLUSTER_CONFIG = { diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_deferrable.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_deferrable.py index cd97ff0eb48b7..35adca660d9a0 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_deferrable.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_deferrable.py @@ -33,12 +33,14 @@ from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_cluster_def" PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "-") -REGION = "europe-west1" +CLUSTER_NAME_BASE = f"cluster-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-") +CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL +REGION = "europe-north1" # Cluster definition diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_diagnose.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_diagnose.py index 11acdd903b438..a610049bf99e3 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_diagnose.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_diagnose.py @@ -33,12 +33,15 @@ from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_diagnose_cluster" PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "-") -REGION = "europe-west1" +CLUSTER_NAME_BASE = f"cluster-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-") +CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL + +REGION = "europe-north1" # Cluster definition @@ -102,7 +105,7 @@ # TEST SETUP create_cluster # TEST BODY - >> diagnose_cluster + >> [diagnose_cluster, diagnose_cluster_deferrable] # TEST TEARDOWN >> delete_cluster ) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_generator.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_generator.py index 769af2d27ab56..d64fda25e4568 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_generator.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_generator.py @@ -24,7 +24,6 @@ import os from datetime import datetime -from pathlib import Path from airflow.models.dag import DAG from airflow.providers.google.cloud.operators.dataproc import ( @@ -32,25 +31,32 @@ DataprocCreateClusterOperator, DataprocDeleteClusterOperator, ) -from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator -from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator +from airflow.providers.google.cloud.operators.gcs import ( + GCSCreateBucketOperator, + GCSDeleteBucketOperator, + GCSSynchronizeBucketsOperator, +) from airflow.utils.trigger_rule import TriggerRule +from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_cluster_generation" -PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default") +PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}" -CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "-") -REGION = "europe-west1" -ZONE = "europe-west1-b" -INIT_FILE_SRC = str(Path(__file__).parent / "resources" / "pip-install.sh") +RESOURCE_DATA_BUCKET = "airflow-system-tests-resources" +INIT_FILE = "pip-install.sh" +GCS_INIT_FILE = f"gs://{RESOURCE_DATA_BUCKET}/dataproc/{INIT_FILE}" -# Cluster definition: Generating Cluster Config for DataprocCreateClusterOperator -# [START how_to_cloud_dataproc_create_cluster_generate_cluster_config] +CLUSTER_NAME_BASE = f"cluster-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-") +CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL -INIT_FILE = "pip-install.sh" +REGION = "us-east4" +ZONE = "us-east4-a" +# Cluster definition: Generating Cluster Config for DataprocCreateClusterOperator +# [START how_to_cloud_dataproc_create_cluster_generate_cluster_config] CLUSTER_GENERATOR_CONFIG = ClusterGenerator( project_id=PROJECT_ID, zone=ZONE, @@ -60,7 +66,7 @@ worker_disk_size=32, num_workers=2, storage_bucket=BUCKET_NAME, - init_actions_uris=[f"gs://{BUCKET_NAME}/{INIT_FILE}"], + init_actions_uris=[GCS_INIT_FILE], metadata={"PIP_PACKAGES": "pyyaml requests pandas openpyxl"}, num_preemptible_workers=1, preemptibility="PREEMPTIBLE", @@ -80,11 +86,13 @@ task_id="create_bucket", bucket_name=BUCKET_NAME, project_id=PROJECT_ID ) - upload_file = LocalFilesystemToGCSOperator( - task_id="upload_file", - src=INIT_FILE_SRC, - dst=INIT_FILE, - bucket=BUCKET_NAME, + move_init_file = GCSSynchronizeBucketsOperator( + task_id="move_init_file", + source_bucket=RESOURCE_DATA_BUCKET, + source_object="dataproc", + destination_bucket=BUCKET_NAME, + destination_object="dataproc", + recursive=True, ) # [START how_to_cloud_dataproc_create_cluster_generate_cluster_config_operator] @@ -114,7 +122,7 @@ ( # TEST SETUP create_bucket - >> upload_file + >> move_init_file # TEST BODY >> create_dataproc_cluster # TEST TEARDOWN diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_start_stop.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_start_stop.py index 1393183ba4046..2af1352fdc1de 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_start_stop.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_start_stop.py @@ -32,14 +32,17 @@ DataprocStopClusterOperator, ) from airflow.utils.trigger_rule import TriggerRule +from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID DAG_ID = "dataproc_cluster_start_stop" -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") -PROJECT_ID = os.environ.get("SYSTEMS_TESTS_GCP_PROJECTS") or "" +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") +PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "-") -REGION = "europe-west1" +CLUSTER_NAME_BASE = f"cluster-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-") +CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL +REGION = "europe-north1" # Cluster definition CLUSTER_CONFIG = { diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_update.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_update.py index 6706e22cbee03..610a19e17f84f 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_update.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_update.py @@ -33,12 +33,14 @@ from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_update" PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "-") -REGION = "europe-west1" +CLUSTER_NAME_BASE = f"cluster-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-") +CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL +REGION = "europe-north1" # Cluster definition diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_gke.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_gke.py index a9953621bdcb9..550dec97c4187 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_gke.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_gke.py @@ -41,13 +41,16 @@ GKEDeleteClusterOperator, ) from airflow.utils.trigger_rule import TriggerRule +from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") -DAG_ID = "dataproc-gke" -PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default") +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") +DAG_ID = "dataproc_gke" +PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID REGION = "us-central1" -CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_BASE = f"cluster-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-") +CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL GKE_CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}-gke".replace("_", "-") WORKLOAD_POOL = f"{PROJECT_ID}.svc.id.goog" GKE_CLUSTER_CONFIG = { @@ -57,6 +60,7 @@ }, "initial_node_count": 1, } + GKE_NAMESPACE = os.environ.get("GKE_NAMESPACE", f"{CLUSTER_NAME}") # [START how_to_cloud_dataproc_create_cluster_in_gke_config] @@ -70,7 +74,8 @@ "roles": ["DEFAULT"], "node_pool_config": { "config": { - "preemptible": True, + "preemptible": False, + "machine_type": "e2-standard-4", } }, } @@ -130,7 +135,6 @@ # TEST BODY >> create_cluster_in_gke # TEST TEARDOWN - # >> delete_gke_cluster >> delete_dataproc_cluster >> delete_gke_cluster ) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_hadoop.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_hadoop.py index 5cc6274c0404e..f8f5cc3063736 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_hadoop.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_hadoop.py @@ -34,12 +34,14 @@ from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_hadoop" PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}" -CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_BASE = f"cluster-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-") +CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL REGION = "europe-west1" OUTPUT_FOLDER = "wordcount" diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_hive.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_hive.py index 2529ae232d480..5e1189c7f9125 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_hive.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_hive.py @@ -33,11 +33,13 @@ from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_hive" PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_BASE = f"cluster-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-") +CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL REGION = "europe-west1" # Cluster definition diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_pig.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_pig.py index cad53c5d76a2b..9b76c10cbe346 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_pig.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_pig.py @@ -34,10 +34,12 @@ from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID DAG_ID = "dataproc_pig" -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_BASE = f"cluster-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-") +CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL REGION = "europe-west1" diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_presto.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_presto.py index c1217b067057a..1c3cdf208252e 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_presto.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_presto.py @@ -33,11 +33,13 @@ from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_presto" PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_BASE = f"cluster-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-") +CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL REGION = "europe-west1" # Cluster definition diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_pyspark.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_pyspark.py index 2a8fb8fce71e8..4c6a64783a425 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_pyspark.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_pyspark.py @@ -30,15 +30,27 @@ DataprocDeleteClusterOperator, DataprocSubmitJobOperator, ) +from airflow.providers.google.cloud.operators.gcs import ( + GCSCreateBucketOperator, + GCSDeleteBucketOperator, + GCSSynchronizeBucketsOperator, +) from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_pyspark" PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -JOB_FILE_URI = "gs://airflow-system-tests-resources/dataproc/pyspark/dataproc-pyspark-job-pi.py" -CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "-") + +CLUSTER_NAME_BASE = f"cluster-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-") +CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL + +BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}" +RESOURCE_DATA_BUCKET = "airflow-system-tests-resources" +JOB_FILE = "dataproc-pyspark-job-pi.py" +GCS_JOB_FILE = f"gs://{BUCKET_NAME}/dataproc/{JOB_FILE}" REGION = "europe-west1" # Cluster definition @@ -60,7 +72,7 @@ PYSPARK_JOB = { "reference": {"project_id": PROJECT_ID}, "placement": {"cluster_name": CLUSTER_NAME}, - "pyspark_job": {"main_python_file_uri": JOB_FILE_URI}, + "pyspark_job": {"main_python_file_uri": GCS_JOB_FILE}, } # [END how_to_cloud_dataproc_pyspark_config] @@ -72,6 +84,19 @@ catchup=False, tags=["example", "dataproc", "pyspark"], ) as dag: + create_bucket = GCSCreateBucketOperator( + task_id="create_bucket", bucket_name=BUCKET_NAME, project_id=PROJECT_ID + ) + + move_job_file = GCSSynchronizeBucketsOperator( + task_id="move_job_file", + source_bucket=RESOURCE_DATA_BUCKET, + source_object="dataproc/pyspark", + destination_bucket=BUCKET_NAME, + destination_object="dataproc", + recursive=True, + ) + create_cluster = DataprocCreateClusterOperator( task_id="create_cluster", project_id=PROJECT_ID, @@ -94,13 +119,20 @@ trigger_rule=TriggerRule.ALL_DONE, ) + delete_bucket = GCSDeleteBucketOperator( + task_id="delete_bucket", bucket_name=BUCKET_NAME, trigger_rule=TriggerRule.ALL_DONE + ) + ( # TEST SETUP - create_cluster + create_bucket + >> move_job_file + >> create_cluster # TEST BODY >> pyspark_task # TEST TEARDOWN >> delete_cluster + >> delete_bucket ) from tests.system.utils.watcher import watcher diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark.py index 75d888b2a45bf..f939347baac4b 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark.py @@ -33,11 +33,14 @@ from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_spark" PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_BASE = f"cluster-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-") +CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL + REGION = "europe-west1" # Cluster definition diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_async.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_async.py index 0f50e332a8bea..cd1060b5fea57 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_async.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_async.py @@ -34,11 +34,14 @@ from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_spark_async" PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_BASE = f"cluster-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-") +CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL + REGION = "europe-west1" # Cluster definition diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_deferrable.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_deferrable.py index 35345625d9e25..d20aa0aa0ed51 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_deferrable.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_deferrable.py @@ -34,11 +34,14 @@ from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_spark_deferrable" PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_BASE = f"cluster-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-") +CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL + REGION = "europe-west1" # Cluster definition diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_sql.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_sql.py index 158a5ef917dd6..0ca0d062d2989 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_sql.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_sql.py @@ -33,11 +33,14 @@ from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_spark_sql" PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -CLUSTER_NAME = f"dataproc-spark-sql-{ENV_ID}" +CLUSTER_NAME_BASE = f"cluster-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-") +CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL + REGION = "europe-west1" # Cluster definition diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_sparkr.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_sparkr.py index 0927a5bdb7d66..50454904759a6 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_sparkr.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_sparkr.py @@ -30,15 +30,25 @@ DataprocDeleteClusterOperator, DataprocSubmitJobOperator, ) +from airflow.providers.google.cloud.operators.gcs import ( + GCSCreateBucketOperator, + GCSDeleteBucketOperator, + GCSSynchronizeBucketsOperator, +) from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_sparkr" PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -JOB_FILE_URI = "gs://airflow-system-tests-resources/dataproc/sparkr/dataproc-sparkr-job.r" -CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "-") +BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}" +RESOURCE_DATA_BUCKET = "airflow-system-tests-resources" +JOB_FILE = "dataproc-sparkr-job.r" +GCS_JOB_FILE = f"gs://{BUCKET_NAME}/dataproc/{JOB_FILE}" +CLUSTER_NAME_BASE = f"cluster-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-") +CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL REGION = "europe-west1" # Cluster definition @@ -60,7 +70,7 @@ SPARKR_JOB = { "reference": {"project_id": PROJECT_ID}, "placement": {"cluster_name": CLUSTER_NAME}, - "spark_r_job": {"main_r_file_uri": JOB_FILE_URI}, + "spark_r_job": {"main_r_file_uri": GCS_JOB_FILE}, } # [END how_to_cloud_dataproc_sparkr_config] @@ -72,6 +82,19 @@ catchup=False, tags=["example", "dataproc", "sparkr"], ) as dag: + create_bucket = GCSCreateBucketOperator( + task_id="create_bucket", bucket_name=BUCKET_NAME, project_id=PROJECT_ID + ) + + move_job_file = GCSSynchronizeBucketsOperator( + task_id="move_job_file", + source_bucket=RESOURCE_DATA_BUCKET, + source_object="dataproc/sparkr", + destination_bucket=BUCKET_NAME, + destination_object="dataproc", + recursive=True, + ) + create_cluster = DataprocCreateClusterOperator( task_id="create_cluster", project_id=PROJECT_ID, @@ -92,13 +115,20 @@ trigger_rule=TriggerRule.ALL_DONE, ) + delete_bucket = GCSDeleteBucketOperator( + task_id="delete_bucket", bucket_name=BUCKET_NAME, trigger_rule=TriggerRule.ALL_DONE + ) + ( # TEST SETUP - create_cluster + create_bucket + >> move_job_file + >> create_cluster # TEST BODY >> sparkr_task # TEST TEARDOWN >> delete_cluster + >> delete_bucket ) from tests.system.utils.watcher import watcher diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_trino.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_trino.py index efcf9d8bdcbfa..936430ccf0f12 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_trino.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_trino.py @@ -33,11 +33,13 @@ from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_trino" PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_BASE = f"cluster-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-") +CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL REGION = "europe-west1" # Cluster definition diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow.py index 5a18e87084177..334826b56523a 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow.py @@ -31,12 +31,15 @@ ) from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_workflow" PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID REGION = "europe-west1" -CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_BASE = f"cluster-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-") +CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL + CLUSTER_CONFIG = { "master_config": { "num_instances": 1, diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow_deferrable.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow_deferrable.py index 1a724079145f6..ec765250a3326 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow_deferrable.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow_deferrable.py @@ -31,12 +31,14 @@ ) from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "dataproc_workflow_def" PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID REGION = "europe-west1" -CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "") +CLUSTER_NAME_BASE = f"cluster-{DAG_ID}".replace("_", "-") +CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-") +CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL CLUSTER_CONFIG = { "master_config": { "num_instances": 1, diff --git a/tests/system/providers/google/cloud/dataproc/resources/__init__.py b/tests/system/providers/google/cloud/dataproc/resources/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/tests/system/providers/google/cloud/dataproc/resources/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/system/providers/google/cloud/dataproc/resources/hello_world.R b/tests/system/providers/google/cloud/dataproc/resources/hello_world.R deleted file mode 100644 index c6721b7b44f66..0000000000000 --- a/tests/system/providers/google/cloud/dataproc/resources/hello_world.R +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/r -if (nchar(Sys.getenv("SPARK_HOME")) < 1) { -Sys.setenv(SPARK_HOME = "/home/spark") -} -library(SparkR, lib.loc = c(file.path(Sys.getenv("SPARK_HOME"), "R", "lib"))) -sparkR.session() -# Create the SparkDataFrame -df <- as.DataFrame(faithful) -head(summarize(groupBy(df, df$waiting), count = n(df$waiting))) diff --git a/tests/system/providers/google/cloud/dataproc/resources/hello_world.py b/tests/system/providers/google/cloud/dataproc/resources/hello_world.py deleted file mode 100644 index bc4ea3e37e96c..0000000000000 --- a/tests/system/providers/google/cloud/dataproc/resources/hello_world.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/python -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from __future__ import annotations - -import pyspark - -sc = pyspark.SparkContext() -rdd = sc.parallelize(["Hello,", "world!"]) -words = sorted(rdd.collect()) -print(words) diff --git a/tests/system/providers/google/cloud/dataproc/resources/hive.sql b/tests/system/providers/google/cloud/dataproc/resources/hive.sql deleted file mode 100644 index 75e8ec2aa9c03..0000000000000 --- a/tests/system/providers/google/cloud/dataproc/resources/hive.sql +++ /dev/null @@ -1,2188 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - --- MySQL dump 10.13 Distrib 5.7.35, for Linux (x86_64) --- --- Host: localhost Database: hive-7e773be5 --- ------------------------------------------------------ --- Server version 5.7.34-google-log - -/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; -/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; -/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; -/*!40101 SET NAMES utf8 */; -/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; -/*!40103 SET TIME_ZONE='+00:00' */; -/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; -/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; -/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; -/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; - --- --- Current Database: `hive-7e773be5` --- - -CREATE DATABASE /*!32312 IF NOT EXISTS*/ `hive-7e773be5` /*!40100 DEFAULT CHARACTER SET utf8 */; - -USE `hive-7e773be5`; - --- --- Table structure for table `AUX_TABLE` --- - -DROP TABLE IF EXISTS `AUX_TABLE`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `AUX_TABLE` ( - `MT_KEY1` varchar(128) NOT NULL, - `MT_KEY2` bigint(20) NOT NULL, - `MT_COMMENT` varchar(255) DEFAULT NULL, - PRIMARY KEY (`MT_KEY1`,`MT_KEY2`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `AUX_TABLE` --- - -LOCK TABLES `AUX_TABLE` WRITE; -/*!40000 ALTER TABLE `AUX_TABLE` DISABLE KEYS */; -INSERT INTO `AUX_TABLE` VALUES ('Cleaner',0,NULL),('CompactionHistory',0,NULL),('HouseKeeper',0,NULL),('Initiator',0,NULL),('MaterializationRebuild',0,NULL),('WriteSetCleaner',0,NULL); -/*!40000 ALTER TABLE `AUX_TABLE` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `BUCKETING_COLS` --- - -DROP TABLE IF EXISTS `BUCKETING_COLS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `BUCKETING_COLS` ( - `SD_ID` bigint(20) NOT NULL, - `BUCKET_COL_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `INTEGER_IDX` int(11) NOT NULL, - PRIMARY KEY (`SD_ID`,`INTEGER_IDX`), - KEY `BUCKETING_COLS_N49` (`SD_ID`), - CONSTRAINT `BUCKETING_COLS_FK1` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `BUCKETING_COLS` --- - -LOCK TABLES `BUCKETING_COLS` WRITE; -/*!40000 ALTER TABLE `BUCKETING_COLS` DISABLE KEYS */; -/*!40000 ALTER TABLE `BUCKETING_COLS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `CDS` --- - -DROP TABLE IF EXISTS `CDS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `CDS` ( - `CD_ID` bigint(20) NOT NULL, - PRIMARY KEY (`CD_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `CDS` --- - -LOCK TABLES `CDS` WRITE; -/*!40000 ALTER TABLE `CDS` DISABLE KEYS */; -/*!40000 ALTER TABLE `CDS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `COLUMNS_V2` --- - -DROP TABLE IF EXISTS `COLUMNS_V2`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `COLUMNS_V2` ( - `CD_ID` bigint(20) NOT NULL, - `COMMENT` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `COLUMN_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `TYPE_NAME` mediumtext, - `INTEGER_IDX` int(11) NOT NULL, - PRIMARY KEY (`CD_ID`,`COLUMN_NAME`), - KEY `COLUMNS_V2_N49` (`CD_ID`), - CONSTRAINT `COLUMNS_V2_FK1` FOREIGN KEY (`CD_ID`) REFERENCES `CDS` (`CD_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `COLUMNS_V2` --- - -LOCK TABLES `COLUMNS_V2` WRITE; -/*!40000 ALTER TABLE `COLUMNS_V2` DISABLE KEYS */; -/*!40000 ALTER TABLE `COLUMNS_V2` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `COMPACTION_QUEUE` --- - -DROP TABLE IF EXISTS `COMPACTION_QUEUE`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `COMPACTION_QUEUE` ( - `CQ_ID` bigint(20) NOT NULL, - `CQ_DATABASE` varchar(128) NOT NULL, - `CQ_TABLE` varchar(128) NOT NULL, - `CQ_PARTITION` varchar(767) DEFAULT NULL, - `CQ_STATE` char(1) NOT NULL, - `CQ_TYPE` char(1) NOT NULL, - `CQ_TBLPROPERTIES` varchar(2048) DEFAULT NULL, - `CQ_WORKER_ID` varchar(128) DEFAULT NULL, - `CQ_START` bigint(20) DEFAULT NULL, - `CQ_RUN_AS` varchar(128) DEFAULT NULL, - `CQ_HIGHEST_WRITE_ID` bigint(20) DEFAULT NULL, - `CQ_META_INFO` varbinary(2048) DEFAULT NULL, - `CQ_HADOOP_JOB_ID` varchar(32) DEFAULT NULL, - PRIMARY KEY (`CQ_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `COMPACTION_QUEUE` --- - -LOCK TABLES `COMPACTION_QUEUE` WRITE; -/*!40000 ALTER TABLE `COMPACTION_QUEUE` DISABLE KEYS */; -/*!40000 ALTER TABLE `COMPACTION_QUEUE` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `COMPLETED_COMPACTIONS` --- - -DROP TABLE IF EXISTS `COMPLETED_COMPACTIONS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `COMPLETED_COMPACTIONS` ( - `CC_ID` bigint(20) NOT NULL, - `CC_DATABASE` varchar(128) NOT NULL, - `CC_TABLE` varchar(128) NOT NULL, - `CC_PARTITION` varchar(767) DEFAULT NULL, - `CC_STATE` char(1) NOT NULL, - `CC_TYPE` char(1) NOT NULL, - `CC_TBLPROPERTIES` varchar(2048) DEFAULT NULL, - `CC_WORKER_ID` varchar(128) DEFAULT NULL, - `CC_START` bigint(20) DEFAULT NULL, - `CC_END` bigint(20) DEFAULT NULL, - `CC_RUN_AS` varchar(128) DEFAULT NULL, - `CC_HIGHEST_WRITE_ID` bigint(20) DEFAULT NULL, - `CC_META_INFO` varbinary(2048) DEFAULT NULL, - `CC_HADOOP_JOB_ID` varchar(32) DEFAULT NULL, - PRIMARY KEY (`CC_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `COMPLETED_COMPACTIONS` --- - -LOCK TABLES `COMPLETED_COMPACTIONS` WRITE; -/*!40000 ALTER TABLE `COMPLETED_COMPACTIONS` DISABLE KEYS */; -/*!40000 ALTER TABLE `COMPLETED_COMPACTIONS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `COMPLETED_TXN_COMPONENTS` --- - -DROP TABLE IF EXISTS `COMPLETED_TXN_COMPONENTS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `COMPLETED_TXN_COMPONENTS` ( - `CTC_TXNID` bigint(20) NOT NULL, - `CTC_DATABASE` varchar(128) NOT NULL, - `CTC_TABLE` varchar(256) DEFAULT NULL, - `CTC_PARTITION` varchar(767) DEFAULT NULL, - `CTC_TIMESTAMP` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - `CTC_WRITEID` bigint(20) DEFAULT NULL, - `CTC_UPDATE_DELETE` char(1) NOT NULL, - KEY `COMPLETED_TXN_COMPONENTS_IDX` (`CTC_DATABASE`,`CTC_TABLE`,`CTC_PARTITION`) USING BTREE -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `COMPLETED_TXN_COMPONENTS` --- - -LOCK TABLES `COMPLETED_TXN_COMPONENTS` WRITE; -/*!40000 ALTER TABLE `COMPLETED_TXN_COMPONENTS` DISABLE KEYS */; -/*!40000 ALTER TABLE `COMPLETED_TXN_COMPONENTS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `CTLGS` --- - -DROP TABLE IF EXISTS `CTLGS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `CTLGS` ( - `CTLG_ID` bigint(20) NOT NULL, - `NAME` varchar(256) DEFAULT NULL, - `DESC` varchar(4000) DEFAULT NULL, - `LOCATION_URI` varchar(4000) NOT NULL, - PRIMARY KEY (`CTLG_ID`), - UNIQUE KEY `UNIQUE_CATALOG` (`NAME`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `CTLGS` --- - -LOCK TABLES `CTLGS` WRITE; -/*!40000 ALTER TABLE `CTLGS` DISABLE KEYS */; -INSERT INTO `CTLGS` VALUES (1,'hive','Default catalog for Hive','gs://gcs-bucket-wj-dm-service-1-022faf65-f2f7-4f8c-b720-5aab8f43e94d/hive-warehouse'); -/*!40000 ALTER TABLE `CTLGS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `DATABASE_PARAMS` --- - -DROP TABLE IF EXISTS `DATABASE_PARAMS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `DATABASE_PARAMS` ( - `DB_ID` bigint(20) NOT NULL, - `PARAM_KEY` varchar(180) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `PARAM_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - PRIMARY KEY (`DB_ID`,`PARAM_KEY`), - KEY `DATABASE_PARAMS_N49` (`DB_ID`), - CONSTRAINT `DATABASE_PARAMS_FK1` FOREIGN KEY (`DB_ID`) REFERENCES `DBS` (`DB_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `DATABASE_PARAMS` --- - -LOCK TABLES `DATABASE_PARAMS` WRITE; -/*!40000 ALTER TABLE `DATABASE_PARAMS` DISABLE KEYS */; -INSERT INTO `DATABASE_PARAMS` VALUES (1,'transient_lastDdlTime','1634563201'); -/*!40000 ALTER TABLE `DATABASE_PARAMS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `DBS` --- - -DROP TABLE IF EXISTS `DBS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `DBS` ( - `DB_ID` bigint(20) NOT NULL, - `DESC` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `DB_LOCATION_URI` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `OWNER_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `OWNER_TYPE` varchar(10) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `CTLG_NAME` varchar(256) NOT NULL DEFAULT 'hive', - PRIMARY KEY (`DB_ID`), - UNIQUE KEY `UNIQUE_DATABASE` (`NAME`,`CTLG_NAME`), - KEY `CTLG_FK1` (`CTLG_NAME`), - CONSTRAINT `CTLG_FK1` FOREIGN KEY (`CTLG_NAME`) REFERENCES `CTLGS` (`NAME`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `DBS` --- - -LOCK TABLES `DBS` WRITE; -/*!40000 ALTER TABLE `DBS` DISABLE KEYS */; -INSERT INTO `DBS` VALUES (1,'Default Hive database','gs://gcs-bucket-wj-dm-service-1-022faf65-f2f7-4f8c-b720-5aab8f43e94d/hive-warehouse','default','public','ROLE','hive'); -/*!40000 ALTER TABLE `DBS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `DB_PRIVS` --- - -DROP TABLE IF EXISTS `DB_PRIVS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `DB_PRIVS` ( - `DB_GRANT_ID` bigint(20) NOT NULL, - `CREATE_TIME` int(11) NOT NULL, - `DB_ID` bigint(20) DEFAULT NULL, - `GRANT_OPTION` smallint(6) NOT NULL, - `GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `GRANTOR_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `PRINCIPAL_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `PRINCIPAL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `DB_PRIV` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `AUTHORIZER` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - PRIMARY KEY (`DB_GRANT_ID`), - UNIQUE KEY `DBPRIVILEGEINDEX` (`AUTHORIZER`,`DB_ID`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`DB_PRIV`,`GRANTOR`,`GRANTOR_TYPE`), - KEY `DB_PRIVS_N49` (`DB_ID`), - CONSTRAINT `DB_PRIVS_FK1` FOREIGN KEY (`DB_ID`) REFERENCES `DBS` (`DB_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `DB_PRIVS` --- - -LOCK TABLES `DB_PRIVS` WRITE; -/*!40000 ALTER TABLE `DB_PRIVS` DISABLE KEYS */; -/*!40000 ALTER TABLE `DB_PRIVS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `DELEGATION_TOKENS` --- - -DROP TABLE IF EXISTS `DELEGATION_TOKENS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `DELEGATION_TOKENS` ( - `TOKEN_IDENT` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `TOKEN` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - PRIMARY KEY (`TOKEN_IDENT`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `DELEGATION_TOKENS` --- - -LOCK TABLES `DELEGATION_TOKENS` WRITE; -/*!40000 ALTER TABLE `DELEGATION_TOKENS` DISABLE KEYS */; -/*!40000 ALTER TABLE `DELEGATION_TOKENS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `FUNCS` --- - -DROP TABLE IF EXISTS `FUNCS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `FUNCS` ( - `FUNC_ID` bigint(20) NOT NULL, - `CLASS_NAME` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `CREATE_TIME` int(11) NOT NULL, - `DB_ID` bigint(20) DEFAULT NULL, - `FUNC_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `FUNC_TYPE` int(11) NOT NULL, - `OWNER_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `OWNER_TYPE` varchar(10) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - PRIMARY KEY (`FUNC_ID`), - UNIQUE KEY `UNIQUEFUNCTION` (`FUNC_NAME`,`DB_ID`), - KEY `FUNCS_N49` (`DB_ID`), - CONSTRAINT `FUNCS_FK1` FOREIGN KEY (`DB_ID`) REFERENCES `DBS` (`DB_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `FUNCS` --- - -LOCK TABLES `FUNCS` WRITE; -/*!40000 ALTER TABLE `FUNCS` DISABLE KEYS */; -/*!40000 ALTER TABLE `FUNCS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `FUNC_RU` --- - -DROP TABLE IF EXISTS `FUNC_RU`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `FUNC_RU` ( - `FUNC_ID` bigint(20) NOT NULL, - `RESOURCE_TYPE` int(11) NOT NULL, - `RESOURCE_URI` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `INTEGER_IDX` int(11) NOT NULL, - PRIMARY KEY (`FUNC_ID`,`INTEGER_IDX`), - CONSTRAINT `FUNC_RU_FK1` FOREIGN KEY (`FUNC_ID`) REFERENCES `FUNCS` (`FUNC_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `FUNC_RU` --- - -LOCK TABLES `FUNC_RU` WRITE; -/*!40000 ALTER TABLE `FUNC_RU` DISABLE KEYS */; -/*!40000 ALTER TABLE `FUNC_RU` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `GLOBAL_PRIVS` --- - -DROP TABLE IF EXISTS `GLOBAL_PRIVS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `GLOBAL_PRIVS` ( - `USER_GRANT_ID` bigint(20) NOT NULL, - `CREATE_TIME` int(11) NOT NULL, - `GRANT_OPTION` smallint(6) NOT NULL, - `GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `GRANTOR_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `PRINCIPAL_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `PRINCIPAL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `USER_PRIV` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `AUTHORIZER` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - PRIMARY KEY (`USER_GRANT_ID`), - UNIQUE KEY `GLOBALPRIVILEGEINDEX` (`AUTHORIZER`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`USER_PRIV`,`GRANTOR`,`GRANTOR_TYPE`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `GLOBAL_PRIVS` --- - -LOCK TABLES `GLOBAL_PRIVS` WRITE; -/*!40000 ALTER TABLE `GLOBAL_PRIVS` DISABLE KEYS */; -INSERT INTO `GLOBAL_PRIVS` VALUES (1,1634563201,1,'admin','ROLE','admin','ROLE','All','SQL'); -/*!40000 ALTER TABLE `GLOBAL_PRIVS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `HIVE_LOCKS` --- - -DROP TABLE IF EXISTS `HIVE_LOCKS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `HIVE_LOCKS` ( - `HL_LOCK_EXT_ID` bigint(20) NOT NULL, - `HL_LOCK_INT_ID` bigint(20) NOT NULL, - `HL_TXNID` bigint(20) NOT NULL, - `HL_DB` varchar(128) NOT NULL, - `HL_TABLE` varchar(128) DEFAULT NULL, - `HL_PARTITION` varchar(767) DEFAULT NULL, - `HL_LOCK_STATE` char(1) NOT NULL, - `HL_LOCK_TYPE` char(1) NOT NULL, - `HL_LAST_HEARTBEAT` bigint(20) NOT NULL, - `HL_ACQUIRED_AT` bigint(20) DEFAULT NULL, - `HL_USER` varchar(128) NOT NULL, - `HL_HOST` varchar(128) NOT NULL, - `HL_HEARTBEAT_COUNT` int(11) DEFAULT NULL, - `HL_AGENT_INFO` varchar(128) DEFAULT NULL, - `HL_BLOCKEDBY_EXT_ID` bigint(20) DEFAULT NULL, - `HL_BLOCKEDBY_INT_ID` bigint(20) DEFAULT NULL, - PRIMARY KEY (`HL_LOCK_EXT_ID`,`HL_LOCK_INT_ID`), - KEY `HIVE_LOCK_TXNID_INDEX` (`HL_TXNID`), - KEY `HL_TXNID_IDX` (`HL_TXNID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `HIVE_LOCKS` --- - -LOCK TABLES `HIVE_LOCKS` WRITE; -/*!40000 ALTER TABLE `HIVE_LOCKS` DISABLE KEYS */; -/*!40000 ALTER TABLE `HIVE_LOCKS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `IDXS` --- - -DROP TABLE IF EXISTS `IDXS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `IDXS` ( - `INDEX_ID` bigint(20) NOT NULL, - `CREATE_TIME` int(11) NOT NULL, - `DEFERRED_REBUILD` bit(1) NOT NULL, - `INDEX_HANDLER_CLASS` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `INDEX_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `INDEX_TBL_ID` bigint(20) DEFAULT NULL, - `LAST_ACCESS_TIME` int(11) NOT NULL, - `ORIG_TBL_ID` bigint(20) DEFAULT NULL, - `SD_ID` bigint(20) DEFAULT NULL, - PRIMARY KEY (`INDEX_ID`), - UNIQUE KEY `UNIQUEINDEX` (`INDEX_NAME`,`ORIG_TBL_ID`), - KEY `IDXS_N51` (`SD_ID`), - KEY `IDXS_N50` (`INDEX_TBL_ID`), - KEY `IDXS_N49` (`ORIG_TBL_ID`), - CONSTRAINT `IDXS_FK1` FOREIGN KEY (`ORIG_TBL_ID`) REFERENCES `TBLS` (`TBL_ID`), - CONSTRAINT `IDXS_FK2` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`), - CONSTRAINT `IDXS_FK3` FOREIGN KEY (`INDEX_TBL_ID`) REFERENCES `TBLS` (`TBL_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `IDXS` --- - -LOCK TABLES `IDXS` WRITE; -/*!40000 ALTER TABLE `IDXS` DISABLE KEYS */; -/*!40000 ALTER TABLE `IDXS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `INDEX_PARAMS` --- - -DROP TABLE IF EXISTS `INDEX_PARAMS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `INDEX_PARAMS` ( - `INDEX_ID` bigint(20) NOT NULL, - `PARAM_KEY` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `PARAM_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - PRIMARY KEY (`INDEX_ID`,`PARAM_KEY`), - KEY `INDEX_PARAMS_N49` (`INDEX_ID`), - CONSTRAINT `INDEX_PARAMS_FK1` FOREIGN KEY (`INDEX_ID`) REFERENCES `IDXS` (`INDEX_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `INDEX_PARAMS` --- - -LOCK TABLES `INDEX_PARAMS` WRITE; -/*!40000 ALTER TABLE `INDEX_PARAMS` DISABLE KEYS */; -/*!40000 ALTER TABLE `INDEX_PARAMS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `I_SCHEMA` --- - -DROP TABLE IF EXISTS `I_SCHEMA`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `I_SCHEMA` ( - `SCHEMA_ID` bigint(20) NOT NULL, - `SCHEMA_TYPE` int(11) NOT NULL, - `NAME` varchar(256) DEFAULT NULL, - `DB_ID` bigint(20) DEFAULT NULL, - `COMPATIBILITY` int(11) NOT NULL, - `VALIDATION_LEVEL` int(11) NOT NULL, - `CAN_EVOLVE` bit(1) NOT NULL, - `SCHEMA_GROUP` varchar(256) DEFAULT NULL, - `DESCRIPTION` varchar(4000) DEFAULT NULL, - PRIMARY KEY (`SCHEMA_ID`), - KEY `DB_ID` (`DB_ID`), - KEY `UNIQUE_NAME` (`NAME`), - CONSTRAINT `I_SCHEMA_ibfk_1` FOREIGN KEY (`DB_ID`) REFERENCES `DBS` (`DB_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `I_SCHEMA` --- - -LOCK TABLES `I_SCHEMA` WRITE; -/*!40000 ALTER TABLE `I_SCHEMA` DISABLE KEYS */; -/*!40000 ALTER TABLE `I_SCHEMA` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `KEY_CONSTRAINTS` --- - -DROP TABLE IF EXISTS `KEY_CONSTRAINTS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `KEY_CONSTRAINTS` ( - `CHILD_CD_ID` bigint(20) DEFAULT NULL, - `CHILD_INTEGER_IDX` int(11) DEFAULT NULL, - `CHILD_TBL_ID` bigint(20) DEFAULT NULL, - `PARENT_CD_ID` bigint(20) DEFAULT NULL, - `PARENT_INTEGER_IDX` int(11) NOT NULL, - `PARENT_TBL_ID` bigint(20) NOT NULL, - `POSITION` bigint(20) NOT NULL, - `CONSTRAINT_NAME` varchar(400) NOT NULL, - `CONSTRAINT_TYPE` smallint(6) NOT NULL, - `UPDATE_RULE` smallint(6) DEFAULT NULL, - `DELETE_RULE` smallint(6) DEFAULT NULL, - `ENABLE_VALIDATE_RELY` smallint(6) NOT NULL, - `DEFAULT_VALUE` varchar(400) DEFAULT NULL, - PRIMARY KEY (`CONSTRAINT_NAME`,`POSITION`), - KEY `CONSTRAINTS_PARENT_TABLE_ID_INDEX` (`PARENT_TBL_ID`) USING BTREE, - KEY `CONSTRAINTS_CONSTRAINT_TYPE_INDEX` (`CONSTRAINT_TYPE`) USING BTREE -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `KEY_CONSTRAINTS` --- - -LOCK TABLES `KEY_CONSTRAINTS` WRITE; -/*!40000 ALTER TABLE `KEY_CONSTRAINTS` DISABLE KEYS */; -/*!40000 ALTER TABLE `KEY_CONSTRAINTS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `MASTER_KEYS` --- - -DROP TABLE IF EXISTS `MASTER_KEYS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `MASTER_KEYS` ( - `KEY_ID` int(11) NOT NULL AUTO_INCREMENT, - `MASTER_KEY` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - PRIMARY KEY (`KEY_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `MASTER_KEYS` --- - -LOCK TABLES `MASTER_KEYS` WRITE; -/*!40000 ALTER TABLE `MASTER_KEYS` DISABLE KEYS */; -/*!40000 ALTER TABLE `MASTER_KEYS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `MATERIALIZATION_REBUILD_LOCKS` --- - -DROP TABLE IF EXISTS `MATERIALIZATION_REBUILD_LOCKS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `MATERIALIZATION_REBUILD_LOCKS` ( - `MRL_TXN_ID` bigint(20) NOT NULL, - `MRL_DB_NAME` varchar(128) NOT NULL, - `MRL_TBL_NAME` varchar(256) NOT NULL, - `MRL_LAST_HEARTBEAT` bigint(20) NOT NULL, - PRIMARY KEY (`MRL_TXN_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `MATERIALIZATION_REBUILD_LOCKS` --- - -LOCK TABLES `MATERIALIZATION_REBUILD_LOCKS` WRITE; -/*!40000 ALTER TABLE `MATERIALIZATION_REBUILD_LOCKS` DISABLE KEYS */; -/*!40000 ALTER TABLE `MATERIALIZATION_REBUILD_LOCKS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `METASTORE_DB_PROPERTIES` --- - -DROP TABLE IF EXISTS `METASTORE_DB_PROPERTIES`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `METASTORE_DB_PROPERTIES` ( - `PROPERTY_KEY` varchar(255) NOT NULL, - `PROPERTY_VALUE` varchar(1000) NOT NULL, - `DESCRIPTION` varchar(1000) DEFAULT NULL, - PRIMARY KEY (`PROPERTY_KEY`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `METASTORE_DB_PROPERTIES` --- - -LOCK TABLES `METASTORE_DB_PROPERTIES` WRITE; -/*!40000 ALTER TABLE `METASTORE_DB_PROPERTIES` DISABLE KEYS */; -/*!40000 ALTER TABLE `METASTORE_DB_PROPERTIES` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `MIN_HISTORY_LEVEL` --- - -DROP TABLE IF EXISTS `MIN_HISTORY_LEVEL`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `MIN_HISTORY_LEVEL` ( - `MHL_TXNID` bigint(20) NOT NULL, - `MHL_MIN_OPEN_TXNID` bigint(20) NOT NULL, - PRIMARY KEY (`MHL_TXNID`), - KEY `MIN_HISTORY_LEVEL_IDX` (`MHL_MIN_OPEN_TXNID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `MIN_HISTORY_LEVEL` --- - -LOCK TABLES `MIN_HISTORY_LEVEL` WRITE; -/*!40000 ALTER TABLE `MIN_HISTORY_LEVEL` DISABLE KEYS */; -/*!40000 ALTER TABLE `MIN_HISTORY_LEVEL` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `MV_CREATION_METADATA` --- - -DROP TABLE IF EXISTS `MV_CREATION_METADATA`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `MV_CREATION_METADATA` ( - `MV_CREATION_METADATA_ID` bigint(20) NOT NULL, - `CAT_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `DB_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `TBL_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `TXN_LIST` text, - `MATERIALIZATION_TIME` bigint(20) NOT NULL, - PRIMARY KEY (`MV_CREATION_METADATA_ID`), - KEY `MV_UNIQUE_TABLE` (`TBL_NAME`,`DB_NAME`) USING BTREE -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `MV_CREATION_METADATA` --- - -LOCK TABLES `MV_CREATION_METADATA` WRITE; -/*!40000 ALTER TABLE `MV_CREATION_METADATA` DISABLE KEYS */; -/*!40000 ALTER TABLE `MV_CREATION_METADATA` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `MV_TABLES_USED` --- - -DROP TABLE IF EXISTS `MV_TABLES_USED`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `MV_TABLES_USED` ( - `MV_CREATION_METADATA_ID` bigint(20) NOT NULL, - `TBL_ID` bigint(20) NOT NULL, - KEY `MV_TABLES_USED_FK1` (`MV_CREATION_METADATA_ID`), - KEY `MV_TABLES_USED_FK2` (`TBL_ID`), - CONSTRAINT `MV_TABLES_USED_FK1` FOREIGN KEY (`MV_CREATION_METADATA_ID`) REFERENCES `MV_CREATION_METADATA` (`MV_CREATION_METADATA_ID`), - CONSTRAINT `MV_TABLES_USED_FK2` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `MV_TABLES_USED` --- - -LOCK TABLES `MV_TABLES_USED` WRITE; -/*!40000 ALTER TABLE `MV_TABLES_USED` DISABLE KEYS */; -/*!40000 ALTER TABLE `MV_TABLES_USED` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `NEXT_COMPACTION_QUEUE_ID` --- - -DROP TABLE IF EXISTS `NEXT_COMPACTION_QUEUE_ID`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `NEXT_COMPACTION_QUEUE_ID` ( - `NCQ_NEXT` bigint(20) NOT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `NEXT_COMPACTION_QUEUE_ID` --- - -LOCK TABLES `NEXT_COMPACTION_QUEUE_ID` WRITE; -/*!40000 ALTER TABLE `NEXT_COMPACTION_QUEUE_ID` DISABLE KEYS */; -INSERT INTO `NEXT_COMPACTION_QUEUE_ID` VALUES (1); -/*!40000 ALTER TABLE `NEXT_COMPACTION_QUEUE_ID` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `NEXT_LOCK_ID` --- - -DROP TABLE IF EXISTS `NEXT_LOCK_ID`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `NEXT_LOCK_ID` ( - `NL_NEXT` bigint(20) NOT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `NEXT_LOCK_ID` --- - -LOCK TABLES `NEXT_LOCK_ID` WRITE; -/*!40000 ALTER TABLE `NEXT_LOCK_ID` DISABLE KEYS */; -INSERT INTO `NEXT_LOCK_ID` VALUES (1); -/*!40000 ALTER TABLE `NEXT_LOCK_ID` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `NEXT_TXN_ID` --- - -DROP TABLE IF EXISTS `NEXT_TXN_ID`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `NEXT_TXN_ID` ( - `NTXN_NEXT` bigint(20) NOT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `NEXT_TXN_ID` --- - -LOCK TABLES `NEXT_TXN_ID` WRITE; -/*!40000 ALTER TABLE `NEXT_TXN_ID` DISABLE KEYS */; -INSERT INTO `NEXT_TXN_ID` VALUES (1); -/*!40000 ALTER TABLE `NEXT_TXN_ID` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `NEXT_WRITE_ID` --- - -DROP TABLE IF EXISTS `NEXT_WRITE_ID`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `NEXT_WRITE_ID` ( - `NWI_DATABASE` varchar(128) NOT NULL, - `NWI_TABLE` varchar(256) NOT NULL, - `NWI_NEXT` bigint(20) NOT NULL, - UNIQUE KEY `NEXT_WRITE_ID_IDX` (`NWI_DATABASE`,`NWI_TABLE`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `NEXT_WRITE_ID` --- - -LOCK TABLES `NEXT_WRITE_ID` WRITE; -/*!40000 ALTER TABLE `NEXT_WRITE_ID` DISABLE KEYS */; -/*!40000 ALTER TABLE `NEXT_WRITE_ID` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `NOTIFICATION_LOG` --- - -DROP TABLE IF EXISTS `NOTIFICATION_LOG`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `NOTIFICATION_LOG` ( - `NL_ID` bigint(20) NOT NULL, - `EVENT_ID` bigint(20) NOT NULL, - `EVENT_TIME` int(11) NOT NULL, - `EVENT_TYPE` varchar(32) NOT NULL, - `CAT_NAME` varchar(256) DEFAULT NULL, - `DB_NAME` varchar(128) DEFAULT NULL, - `TBL_NAME` varchar(256) DEFAULT NULL, - `MESSAGE` longtext, - `MESSAGE_FORMAT` varchar(16) DEFAULT NULL, - PRIMARY KEY (`NL_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `NOTIFICATION_LOG` --- - -LOCK TABLES `NOTIFICATION_LOG` WRITE; -/*!40000 ALTER TABLE `NOTIFICATION_LOG` DISABLE KEYS */; -/*!40000 ALTER TABLE `NOTIFICATION_LOG` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `NOTIFICATION_SEQUENCE` --- - -DROP TABLE IF EXISTS `NOTIFICATION_SEQUENCE`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `NOTIFICATION_SEQUENCE` ( - `NNI_ID` bigint(20) NOT NULL, - `NEXT_EVENT_ID` bigint(20) NOT NULL, - PRIMARY KEY (`NNI_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `NOTIFICATION_SEQUENCE` --- - -LOCK TABLES `NOTIFICATION_SEQUENCE` WRITE; -/*!40000 ALTER TABLE `NOTIFICATION_SEQUENCE` DISABLE KEYS */; -INSERT INTO `NOTIFICATION_SEQUENCE` VALUES (1,1); -/*!40000 ALTER TABLE `NOTIFICATION_SEQUENCE` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `NUCLEUS_TABLES` --- - -DROP TABLE IF EXISTS `NUCLEUS_TABLES`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `NUCLEUS_TABLES` ( - `CLASS_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `TABLE_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `TYPE` varchar(4) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `OWNER` varchar(2) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `VERSION` varchar(20) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `INTERFACE_NAME` varchar(255) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - PRIMARY KEY (`CLASS_NAME`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `NUCLEUS_TABLES` --- - -LOCK TABLES `NUCLEUS_TABLES` WRITE; -/*!40000 ALTER TABLE `NUCLEUS_TABLES` DISABLE KEYS */; -/*!40000 ALTER TABLE `NUCLEUS_TABLES` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `PARTITIONS` --- - -DROP TABLE IF EXISTS `PARTITIONS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `PARTITIONS` ( - `PART_ID` bigint(20) NOT NULL, - `CREATE_TIME` int(11) NOT NULL, - `LAST_ACCESS_TIME` int(11) NOT NULL, - `PART_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `SD_ID` bigint(20) DEFAULT NULL, - `TBL_ID` bigint(20) DEFAULT NULL, - PRIMARY KEY (`PART_ID`), - UNIQUE KEY `UNIQUEPARTITION` (`PART_NAME`,`TBL_ID`), - KEY `PARTITIONS_N49` (`TBL_ID`), - KEY `PARTITIONS_N50` (`SD_ID`), - CONSTRAINT `PARTITIONS_FK1` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`), - CONSTRAINT `PARTITIONS_FK2` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `PARTITIONS` --- - -LOCK TABLES `PARTITIONS` WRITE; -/*!40000 ALTER TABLE `PARTITIONS` DISABLE KEYS */; -/*!40000 ALTER TABLE `PARTITIONS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `PARTITION_EVENTS` --- - -DROP TABLE IF EXISTS `PARTITION_EVENTS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `PARTITION_EVENTS` ( - `PART_NAME_ID` bigint(20) NOT NULL, - `CAT_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `DB_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `EVENT_TIME` bigint(20) NOT NULL, - `EVENT_TYPE` int(11) NOT NULL, - `PARTITION_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `TBL_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - PRIMARY KEY (`PART_NAME_ID`), - KEY `PARTITIONEVENTINDEX` (`PARTITION_NAME`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `PARTITION_EVENTS` --- - -LOCK TABLES `PARTITION_EVENTS` WRITE; -/*!40000 ALTER TABLE `PARTITION_EVENTS` DISABLE KEYS */; -/*!40000 ALTER TABLE `PARTITION_EVENTS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `PARTITION_KEYS` --- - -DROP TABLE IF EXISTS `PARTITION_KEYS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `PARTITION_KEYS` ( - `TBL_ID` bigint(20) NOT NULL, - `PKEY_COMMENT` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `PKEY_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `PKEY_TYPE` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `INTEGER_IDX` int(11) NOT NULL, - PRIMARY KEY (`TBL_ID`,`PKEY_NAME`), - KEY `PARTITION_KEYS_N49` (`TBL_ID`), - CONSTRAINT `PARTITION_KEYS_FK1` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `PARTITION_KEYS` --- - -LOCK TABLES `PARTITION_KEYS` WRITE; -/*!40000 ALTER TABLE `PARTITION_KEYS` DISABLE KEYS */; -/*!40000 ALTER TABLE `PARTITION_KEYS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `PARTITION_KEY_VALS` --- - -DROP TABLE IF EXISTS `PARTITION_KEY_VALS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `PARTITION_KEY_VALS` ( - `PART_ID` bigint(20) NOT NULL, - `PART_KEY_VAL` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `INTEGER_IDX` int(11) NOT NULL, - PRIMARY KEY (`PART_ID`,`INTEGER_IDX`), - KEY `PARTITION_KEY_VALS_N49` (`PART_ID`), - CONSTRAINT `PARTITION_KEY_VALS_FK1` FOREIGN KEY (`PART_ID`) REFERENCES `PARTITIONS` (`PART_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `PARTITION_KEY_VALS` --- - -LOCK TABLES `PARTITION_KEY_VALS` WRITE; -/*!40000 ALTER TABLE `PARTITION_KEY_VALS` DISABLE KEYS */; -/*!40000 ALTER TABLE `PARTITION_KEY_VALS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `PARTITION_PARAMS` --- - -DROP TABLE IF EXISTS `PARTITION_PARAMS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `PARTITION_PARAMS` ( - `PART_ID` bigint(20) NOT NULL, - `PARAM_KEY` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `PARAM_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - PRIMARY KEY (`PART_ID`,`PARAM_KEY`), - KEY `PARTITION_PARAMS_N49` (`PART_ID`), - CONSTRAINT `PARTITION_PARAMS_FK1` FOREIGN KEY (`PART_ID`) REFERENCES `PARTITIONS` (`PART_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `PARTITION_PARAMS` --- - -LOCK TABLES `PARTITION_PARAMS` WRITE; -/*!40000 ALTER TABLE `PARTITION_PARAMS` DISABLE KEYS */; -/*!40000 ALTER TABLE `PARTITION_PARAMS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `PART_COL_PRIVS` --- - -DROP TABLE IF EXISTS `PART_COL_PRIVS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `PART_COL_PRIVS` ( - `PART_COLUMN_GRANT_ID` bigint(20) NOT NULL, - `COLUMN_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `CREATE_TIME` int(11) NOT NULL, - `GRANT_OPTION` smallint(6) NOT NULL, - `GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `GRANTOR_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `PART_ID` bigint(20) DEFAULT NULL, - `PRINCIPAL_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `PRINCIPAL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `PART_COL_PRIV` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `AUTHORIZER` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - PRIMARY KEY (`PART_COLUMN_GRANT_ID`), - KEY `PART_COL_PRIVS_N49` (`PART_ID`), - KEY `PARTITIONCOLUMNPRIVILEGEINDEX` (`AUTHORIZER`,`PART_ID`,`COLUMN_NAME`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`PART_COL_PRIV`,`GRANTOR`,`GRANTOR_TYPE`), - CONSTRAINT `PART_COL_PRIVS_FK1` FOREIGN KEY (`PART_ID`) REFERENCES `PARTITIONS` (`PART_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `PART_COL_PRIVS` --- - -LOCK TABLES `PART_COL_PRIVS` WRITE; -/*!40000 ALTER TABLE `PART_COL_PRIVS` DISABLE KEYS */; -/*!40000 ALTER TABLE `PART_COL_PRIVS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `PART_COL_STATS` --- - -DROP TABLE IF EXISTS `PART_COL_STATS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `PART_COL_STATS` ( - `CS_ID` bigint(20) NOT NULL, - `CAT_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `DB_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `TABLE_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `PARTITION_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `COLUMN_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `COLUMN_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `PART_ID` bigint(20) NOT NULL, - `LONG_LOW_VALUE` bigint(20) DEFAULT NULL, - `LONG_HIGH_VALUE` bigint(20) DEFAULT NULL, - `DOUBLE_HIGH_VALUE` double(53,4) DEFAULT NULL, - `DOUBLE_LOW_VALUE` double(53,4) DEFAULT NULL, - `BIG_DECIMAL_LOW_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `BIG_DECIMAL_HIGH_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `NUM_NULLS` bigint(20) NOT NULL, - `NUM_DISTINCTS` bigint(20) DEFAULT NULL, - `BIT_VECTOR` blob, - `AVG_COL_LEN` double(53,4) DEFAULT NULL, - `MAX_COL_LEN` bigint(20) DEFAULT NULL, - `NUM_TRUES` bigint(20) DEFAULT NULL, - `NUM_FALSES` bigint(20) DEFAULT NULL, - `LAST_ANALYZED` bigint(20) NOT NULL, - PRIMARY KEY (`CS_ID`), - KEY `PART_COL_STATS_FK` (`PART_ID`), - KEY `PCS_STATS_IDX` (`CAT_NAME`,`DB_NAME`,`TABLE_NAME`,`COLUMN_NAME`,`PARTITION_NAME`) USING BTREE, - CONSTRAINT `PART_COL_STATS_FK` FOREIGN KEY (`PART_ID`) REFERENCES `PARTITIONS` (`PART_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `PART_COL_STATS` --- - -LOCK TABLES `PART_COL_STATS` WRITE; -/*!40000 ALTER TABLE `PART_COL_STATS` DISABLE KEYS */; -/*!40000 ALTER TABLE `PART_COL_STATS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `PART_PRIVS` --- - -DROP TABLE IF EXISTS `PART_PRIVS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `PART_PRIVS` ( - `PART_GRANT_ID` bigint(20) NOT NULL, - `CREATE_TIME` int(11) NOT NULL, - `GRANT_OPTION` smallint(6) NOT NULL, - `GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `GRANTOR_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `PART_ID` bigint(20) DEFAULT NULL, - `PRINCIPAL_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `PRINCIPAL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `PART_PRIV` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `AUTHORIZER` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - PRIMARY KEY (`PART_GRANT_ID`), - KEY `PARTPRIVILEGEINDEX` (`AUTHORIZER`,`PART_ID`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`PART_PRIV`,`GRANTOR`,`GRANTOR_TYPE`), - KEY `PART_PRIVS_N49` (`PART_ID`), - CONSTRAINT `PART_PRIVS_FK1` FOREIGN KEY (`PART_ID`) REFERENCES `PARTITIONS` (`PART_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `PART_PRIVS` --- - -LOCK TABLES `PART_PRIVS` WRITE; -/*!40000 ALTER TABLE `PART_PRIVS` DISABLE KEYS */; -/*!40000 ALTER TABLE `PART_PRIVS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `REPL_TXN_MAP` --- - -DROP TABLE IF EXISTS `REPL_TXN_MAP`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `REPL_TXN_MAP` ( - `RTM_REPL_POLICY` varchar(256) NOT NULL, - `RTM_SRC_TXN_ID` bigint(20) NOT NULL, - `RTM_TARGET_TXN_ID` bigint(20) NOT NULL, - PRIMARY KEY (`RTM_REPL_POLICY`,`RTM_SRC_TXN_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `REPL_TXN_MAP` --- - -LOCK TABLES `REPL_TXN_MAP` WRITE; -/*!40000 ALTER TABLE `REPL_TXN_MAP` DISABLE KEYS */; -/*!40000 ALTER TABLE `REPL_TXN_MAP` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `ROLES` --- - -DROP TABLE IF EXISTS `ROLES`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `ROLES` ( - `ROLE_ID` bigint(20) NOT NULL, - `CREATE_TIME` int(11) NOT NULL, - `OWNER_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `ROLE_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - PRIMARY KEY (`ROLE_ID`), - UNIQUE KEY `ROLEENTITYINDEX` (`ROLE_NAME`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `ROLES` --- - -LOCK TABLES `ROLES` WRITE; -/*!40000 ALTER TABLE `ROLES` DISABLE KEYS */; -INSERT INTO `ROLES` VALUES (1,1634563201,'admin','admin'),(2,1634563201,'public','public'); -/*!40000 ALTER TABLE `ROLES` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `ROLE_MAP` --- - -DROP TABLE IF EXISTS `ROLE_MAP`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `ROLE_MAP` ( - `ROLE_GRANT_ID` bigint(20) NOT NULL, - `ADD_TIME` int(11) NOT NULL, - `GRANT_OPTION` smallint(6) NOT NULL, - `GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `GRANTOR_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `PRINCIPAL_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `PRINCIPAL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `ROLE_ID` bigint(20) DEFAULT NULL, - PRIMARY KEY (`ROLE_GRANT_ID`), - UNIQUE KEY `USERROLEMAPINDEX` (`PRINCIPAL_NAME`,`ROLE_ID`,`GRANTOR`,`GRANTOR_TYPE`), - KEY `ROLE_MAP_N49` (`ROLE_ID`), - CONSTRAINT `ROLE_MAP_FK1` FOREIGN KEY (`ROLE_ID`) REFERENCES `ROLES` (`ROLE_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `ROLE_MAP` --- - -LOCK TABLES `ROLE_MAP` WRITE; -/*!40000 ALTER TABLE `ROLE_MAP` DISABLE KEYS */; -/*!40000 ALTER TABLE `ROLE_MAP` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `RUNTIME_STATS` --- - -DROP TABLE IF EXISTS `RUNTIME_STATS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `RUNTIME_STATS` ( - `RS_ID` bigint(20) NOT NULL, - `CREATE_TIME` bigint(20) NOT NULL, - `WEIGHT` bigint(20) NOT NULL, - `PAYLOAD` blob, - PRIMARY KEY (`RS_ID`), - KEY `IDX_RUNTIME_STATS_CREATE_TIME` (`CREATE_TIME`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `RUNTIME_STATS` --- - -LOCK TABLES `RUNTIME_STATS` WRITE; -/*!40000 ALTER TABLE `RUNTIME_STATS` DISABLE KEYS */; -/*!40000 ALTER TABLE `RUNTIME_STATS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `SCHEMA_VERSION` --- - -DROP TABLE IF EXISTS `SCHEMA_VERSION`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `SCHEMA_VERSION` ( - `SCHEMA_VERSION_ID` bigint(20) NOT NULL, - `SCHEMA_ID` bigint(20) DEFAULT NULL, - `VERSION` int(11) NOT NULL, - `CREATED_AT` bigint(20) NOT NULL, - `CD_ID` bigint(20) DEFAULT NULL, - `STATE` int(11) NOT NULL, - `DESCRIPTION` varchar(4000) DEFAULT NULL, - `SCHEMA_TEXT` mediumtext, - `FINGERPRINT` varchar(256) DEFAULT NULL, - `SCHEMA_VERSION_NAME` varchar(256) DEFAULT NULL, - `SERDE_ID` bigint(20) DEFAULT NULL, - PRIMARY KEY (`SCHEMA_VERSION_ID`), - KEY `CD_ID` (`CD_ID`), - KEY `SERDE_ID` (`SERDE_ID`), - KEY `UNIQUE_VERSION` (`SCHEMA_ID`,`VERSION`), - CONSTRAINT `SCHEMA_VERSION_ibfk_1` FOREIGN KEY (`SCHEMA_ID`) REFERENCES `I_SCHEMA` (`SCHEMA_ID`), - CONSTRAINT `SCHEMA_VERSION_ibfk_2` FOREIGN KEY (`CD_ID`) REFERENCES `CDS` (`CD_ID`), - CONSTRAINT `SCHEMA_VERSION_ibfk_3` FOREIGN KEY (`SERDE_ID`) REFERENCES `SERDES` (`SERDE_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `SCHEMA_VERSION` --- - -LOCK TABLES `SCHEMA_VERSION` WRITE; -/*!40000 ALTER TABLE `SCHEMA_VERSION` DISABLE KEYS */; -/*!40000 ALTER TABLE `SCHEMA_VERSION` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `SDS` --- - -DROP TABLE IF EXISTS `SDS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `SDS` ( - `SD_ID` bigint(20) NOT NULL, - `CD_ID` bigint(20) DEFAULT NULL, - `INPUT_FORMAT` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `IS_COMPRESSED` bit(1) NOT NULL, - `IS_STOREDASSUBDIRECTORIES` bit(1) NOT NULL, - `LOCATION` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `NUM_BUCKETS` int(11) NOT NULL, - `OUTPUT_FORMAT` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `SERDE_ID` bigint(20) DEFAULT NULL, - PRIMARY KEY (`SD_ID`), - KEY `SDS_N49` (`SERDE_ID`), - KEY `SDS_N50` (`CD_ID`), - CONSTRAINT `SDS_FK1` FOREIGN KEY (`SERDE_ID`) REFERENCES `SERDES` (`SERDE_ID`), - CONSTRAINT `SDS_FK2` FOREIGN KEY (`CD_ID`) REFERENCES `CDS` (`CD_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `SDS` --- - -LOCK TABLES `SDS` WRITE; -/*!40000 ALTER TABLE `SDS` DISABLE KEYS */; -/*!40000 ALTER TABLE `SDS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `SD_PARAMS` --- - -DROP TABLE IF EXISTS `SD_PARAMS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `SD_PARAMS` ( - `SD_ID` bigint(20) NOT NULL, - `PARAM_KEY` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `PARAM_VALUE` mediumtext CHARACTER SET latin1 COLLATE latin1_bin, - PRIMARY KEY (`SD_ID`,`PARAM_KEY`), - KEY `SD_PARAMS_N49` (`SD_ID`), - CONSTRAINT `SD_PARAMS_FK1` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `SD_PARAMS` --- - -LOCK TABLES `SD_PARAMS` WRITE; -/*!40000 ALTER TABLE `SD_PARAMS` DISABLE KEYS */; -/*!40000 ALTER TABLE `SD_PARAMS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `SEQUENCE_TABLE` --- - -DROP TABLE IF EXISTS `SEQUENCE_TABLE`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `SEQUENCE_TABLE` ( - `SEQUENCE_NAME` varchar(255) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `NEXT_VAL` bigint(20) NOT NULL, - PRIMARY KEY (`SEQUENCE_NAME`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `SEQUENCE_TABLE` --- - -LOCK TABLES `SEQUENCE_TABLE` WRITE; -/*!40000 ALTER TABLE `SEQUENCE_TABLE` DISABLE KEYS */; -INSERT INTO `SEQUENCE_TABLE` VALUES ('org.apache.hadoop.hive.metastore.model.MDatabase',6),('org.apache.hadoop.hive.metastore.model.MGlobalPrivilege',6),('org.apache.hadoop.hive.metastore.model.MNotificationLog',1),('org.apache.hadoop.hive.metastore.model.MRole',6); -/*!40000 ALTER TABLE `SEQUENCE_TABLE` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `SERDES` --- - -DROP TABLE IF EXISTS `SERDES`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `SERDES` ( - `SERDE_ID` bigint(20) NOT NULL, - `NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `SLIB` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `DESCRIPTION` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `SERIALIZER_CLASS` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `DESERIALIZER_CLASS` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `SERDE_TYPE` int(11) DEFAULT NULL, - PRIMARY KEY (`SERDE_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `SERDES` --- - -LOCK TABLES `SERDES` WRITE; -/*!40000 ALTER TABLE `SERDES` DISABLE KEYS */; -/*!40000 ALTER TABLE `SERDES` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `SERDE_PARAMS` --- - -DROP TABLE IF EXISTS `SERDE_PARAMS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `SERDE_PARAMS` ( - `SERDE_ID` bigint(20) NOT NULL, - `PARAM_KEY` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `PARAM_VALUE` mediumtext CHARACTER SET latin1 COLLATE latin1_bin, - PRIMARY KEY (`SERDE_ID`,`PARAM_KEY`), - KEY `SERDE_PARAMS_N49` (`SERDE_ID`), - CONSTRAINT `SERDE_PARAMS_FK1` FOREIGN KEY (`SERDE_ID`) REFERENCES `SERDES` (`SERDE_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `SERDE_PARAMS` --- - -LOCK TABLES `SERDE_PARAMS` WRITE; -/*!40000 ALTER TABLE `SERDE_PARAMS` DISABLE KEYS */; -/*!40000 ALTER TABLE `SERDE_PARAMS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `SKEWED_COL_NAMES` --- - -DROP TABLE IF EXISTS `SKEWED_COL_NAMES`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `SKEWED_COL_NAMES` ( - `SD_ID` bigint(20) NOT NULL, - `SKEWED_COL_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `INTEGER_IDX` int(11) NOT NULL, - PRIMARY KEY (`SD_ID`,`INTEGER_IDX`), - KEY `SKEWED_COL_NAMES_N49` (`SD_ID`), - CONSTRAINT `SKEWED_COL_NAMES_FK1` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `SKEWED_COL_NAMES` --- - -LOCK TABLES `SKEWED_COL_NAMES` WRITE; -/*!40000 ALTER TABLE `SKEWED_COL_NAMES` DISABLE KEYS */; -/*!40000 ALTER TABLE `SKEWED_COL_NAMES` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `SKEWED_COL_VALUE_LOC_MAP` --- - -DROP TABLE IF EXISTS `SKEWED_COL_VALUE_LOC_MAP`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `SKEWED_COL_VALUE_LOC_MAP` ( - `SD_ID` bigint(20) NOT NULL, - `STRING_LIST_ID_KID` bigint(20) NOT NULL, - `LOCATION` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - PRIMARY KEY (`SD_ID`,`STRING_LIST_ID_KID`), - KEY `SKEWED_COL_VALUE_LOC_MAP_N49` (`STRING_LIST_ID_KID`), - KEY `SKEWED_COL_VALUE_LOC_MAP_N50` (`SD_ID`), - CONSTRAINT `SKEWED_COL_VALUE_LOC_MAP_FK1` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`), - CONSTRAINT `SKEWED_COL_VALUE_LOC_MAP_FK2` FOREIGN KEY (`STRING_LIST_ID_KID`) REFERENCES `SKEWED_STRING_LIST` (`STRING_LIST_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `SKEWED_COL_VALUE_LOC_MAP` --- - -LOCK TABLES `SKEWED_COL_VALUE_LOC_MAP` WRITE; -/*!40000 ALTER TABLE `SKEWED_COL_VALUE_LOC_MAP` DISABLE KEYS */; -/*!40000 ALTER TABLE `SKEWED_COL_VALUE_LOC_MAP` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `SKEWED_STRING_LIST` --- - -DROP TABLE IF EXISTS `SKEWED_STRING_LIST`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `SKEWED_STRING_LIST` ( - `STRING_LIST_ID` bigint(20) NOT NULL, - PRIMARY KEY (`STRING_LIST_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `SKEWED_STRING_LIST` --- - -LOCK TABLES `SKEWED_STRING_LIST` WRITE; -/*!40000 ALTER TABLE `SKEWED_STRING_LIST` DISABLE KEYS */; -/*!40000 ALTER TABLE `SKEWED_STRING_LIST` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `SKEWED_STRING_LIST_VALUES` --- - -DROP TABLE IF EXISTS `SKEWED_STRING_LIST_VALUES`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `SKEWED_STRING_LIST_VALUES` ( - `STRING_LIST_ID` bigint(20) NOT NULL, - `STRING_LIST_VALUE` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `INTEGER_IDX` int(11) NOT NULL, - PRIMARY KEY (`STRING_LIST_ID`,`INTEGER_IDX`), - KEY `SKEWED_STRING_LIST_VALUES_N49` (`STRING_LIST_ID`), - CONSTRAINT `SKEWED_STRING_LIST_VALUES_FK1` FOREIGN KEY (`STRING_LIST_ID`) REFERENCES `SKEWED_STRING_LIST` (`STRING_LIST_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `SKEWED_STRING_LIST_VALUES` --- - -LOCK TABLES `SKEWED_STRING_LIST_VALUES` WRITE; -/*!40000 ALTER TABLE `SKEWED_STRING_LIST_VALUES` DISABLE KEYS */; -/*!40000 ALTER TABLE `SKEWED_STRING_LIST_VALUES` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `SKEWED_VALUES` --- - -DROP TABLE IF EXISTS `SKEWED_VALUES`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `SKEWED_VALUES` ( - `SD_ID_OID` bigint(20) NOT NULL, - `STRING_LIST_ID_EID` bigint(20) NOT NULL, - `INTEGER_IDX` int(11) NOT NULL, - PRIMARY KEY (`SD_ID_OID`,`INTEGER_IDX`), - KEY `SKEWED_VALUES_N50` (`SD_ID_OID`), - KEY `SKEWED_VALUES_N49` (`STRING_LIST_ID_EID`), - CONSTRAINT `SKEWED_VALUES_FK1` FOREIGN KEY (`SD_ID_OID`) REFERENCES `SDS` (`SD_ID`), - CONSTRAINT `SKEWED_VALUES_FK2` FOREIGN KEY (`STRING_LIST_ID_EID`) REFERENCES `SKEWED_STRING_LIST` (`STRING_LIST_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `SKEWED_VALUES` --- - -LOCK TABLES `SKEWED_VALUES` WRITE; -/*!40000 ALTER TABLE `SKEWED_VALUES` DISABLE KEYS */; -/*!40000 ALTER TABLE `SKEWED_VALUES` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `SORT_COLS` --- - -DROP TABLE IF EXISTS `SORT_COLS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `SORT_COLS` ( - `SD_ID` bigint(20) NOT NULL, - `COLUMN_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `ORDER` int(11) NOT NULL, - `INTEGER_IDX` int(11) NOT NULL, - PRIMARY KEY (`SD_ID`,`INTEGER_IDX`), - KEY `SORT_COLS_N49` (`SD_ID`), - CONSTRAINT `SORT_COLS_FK1` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `SORT_COLS` --- - -LOCK TABLES `SORT_COLS` WRITE; -/*!40000 ALTER TABLE `SORT_COLS` DISABLE KEYS */; -/*!40000 ALTER TABLE `SORT_COLS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `TABLE_PARAMS` --- - -DROP TABLE IF EXISTS `TABLE_PARAMS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `TABLE_PARAMS` ( - `TBL_ID` bigint(20) NOT NULL, - `PARAM_KEY` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `PARAM_VALUE` mediumtext CHARACTER SET latin1 COLLATE latin1_bin, - PRIMARY KEY (`TBL_ID`,`PARAM_KEY`), - KEY `TABLE_PARAMS_N49` (`TBL_ID`), - CONSTRAINT `TABLE_PARAMS_FK1` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `TABLE_PARAMS` --- - -LOCK TABLES `TABLE_PARAMS` WRITE; -/*!40000 ALTER TABLE `TABLE_PARAMS` DISABLE KEYS */; -/*!40000 ALTER TABLE `TABLE_PARAMS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `TAB_COL_STATS` --- - -DROP TABLE IF EXISTS `TAB_COL_STATS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `TAB_COL_STATS` ( - `CS_ID` bigint(20) NOT NULL, - `CAT_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `DB_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `TABLE_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `COLUMN_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `COLUMN_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `TBL_ID` bigint(20) NOT NULL, - `LONG_LOW_VALUE` bigint(20) DEFAULT NULL, - `LONG_HIGH_VALUE` bigint(20) DEFAULT NULL, - `DOUBLE_HIGH_VALUE` double(53,4) DEFAULT NULL, - `DOUBLE_LOW_VALUE` double(53,4) DEFAULT NULL, - `BIG_DECIMAL_LOW_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `BIG_DECIMAL_HIGH_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `NUM_NULLS` bigint(20) NOT NULL, - `NUM_DISTINCTS` bigint(20) DEFAULT NULL, - `BIT_VECTOR` blob, - `AVG_COL_LEN` double(53,4) DEFAULT NULL, - `MAX_COL_LEN` bigint(20) DEFAULT NULL, - `NUM_TRUES` bigint(20) DEFAULT NULL, - `NUM_FALSES` bigint(20) DEFAULT NULL, - `LAST_ANALYZED` bigint(20) NOT NULL, - PRIMARY KEY (`CS_ID`), - KEY `TAB_COL_STATS_FK` (`TBL_ID`), - KEY `TAB_COL_STATS_IDX` (`CAT_NAME`,`DB_NAME`,`TABLE_NAME`,`COLUMN_NAME`) USING BTREE, - CONSTRAINT `TAB_COL_STATS_FK` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `TAB_COL_STATS` --- - -LOCK TABLES `TAB_COL_STATS` WRITE; -/*!40000 ALTER TABLE `TAB_COL_STATS` DISABLE KEYS */; -/*!40000 ALTER TABLE `TAB_COL_STATS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `TBLS` --- - -DROP TABLE IF EXISTS `TBLS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `TBLS` ( - `TBL_ID` bigint(20) NOT NULL, - `CREATE_TIME` int(11) NOT NULL, - `DB_ID` bigint(20) DEFAULT NULL, - `LAST_ACCESS_TIME` int(11) NOT NULL, - `OWNER` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `OWNER_TYPE` varchar(10) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `RETENTION` int(11) NOT NULL, - `SD_ID` bigint(20) DEFAULT NULL, - `TBL_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `TBL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `VIEW_EXPANDED_TEXT` mediumtext, - `VIEW_ORIGINAL_TEXT` mediumtext, - `IS_REWRITE_ENABLED` bit(1) NOT NULL DEFAULT b'0', - PRIMARY KEY (`TBL_ID`), - UNIQUE KEY `UNIQUETABLE` (`TBL_NAME`,`DB_ID`), - KEY `TBLS_N50` (`SD_ID`), - KEY `TBLS_N49` (`DB_ID`), - CONSTRAINT `TBLS_FK1` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`), - CONSTRAINT `TBLS_FK2` FOREIGN KEY (`DB_ID`) REFERENCES `DBS` (`DB_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `TBLS` --- - -LOCK TABLES `TBLS` WRITE; -/*!40000 ALTER TABLE `TBLS` DISABLE KEYS */; -/*!40000 ALTER TABLE `TBLS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `TBL_COL_PRIVS` --- - -DROP TABLE IF EXISTS `TBL_COL_PRIVS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `TBL_COL_PRIVS` ( - `TBL_COLUMN_GRANT_ID` bigint(20) NOT NULL, - `COLUMN_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `CREATE_TIME` int(11) NOT NULL, - `GRANT_OPTION` smallint(6) NOT NULL, - `GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `GRANTOR_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `PRINCIPAL_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `PRINCIPAL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `TBL_COL_PRIV` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `TBL_ID` bigint(20) DEFAULT NULL, - `AUTHORIZER` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - PRIMARY KEY (`TBL_COLUMN_GRANT_ID`), - KEY `TABLECOLUMNPRIVILEGEINDEX` (`AUTHORIZER`,`TBL_ID`,`COLUMN_NAME`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`TBL_COL_PRIV`,`GRANTOR`,`GRANTOR_TYPE`), - KEY `TBL_COL_PRIVS_N49` (`TBL_ID`), - CONSTRAINT `TBL_COL_PRIVS_FK1` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `TBL_COL_PRIVS` --- - -LOCK TABLES `TBL_COL_PRIVS` WRITE; -/*!40000 ALTER TABLE `TBL_COL_PRIVS` DISABLE KEYS */; -/*!40000 ALTER TABLE `TBL_COL_PRIVS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `TBL_PRIVS` --- - -DROP TABLE IF EXISTS `TBL_PRIVS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `TBL_PRIVS` ( - `TBL_GRANT_ID` bigint(20) NOT NULL, - `CREATE_TIME` int(11) NOT NULL, - `GRANT_OPTION` smallint(6) NOT NULL, - `GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `GRANTOR_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `PRINCIPAL_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `PRINCIPAL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `TBL_PRIV` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `TBL_ID` bigint(20) DEFAULT NULL, - `AUTHORIZER` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - PRIMARY KEY (`TBL_GRANT_ID`), - KEY `TBL_PRIVS_N49` (`TBL_ID`), - KEY `TABLEPRIVILEGEINDEX` (`AUTHORIZER`,`TBL_ID`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`TBL_PRIV`,`GRANTOR`,`GRANTOR_TYPE`), - CONSTRAINT `TBL_PRIVS_FK1` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `TBL_PRIVS` --- - -LOCK TABLES `TBL_PRIVS` WRITE; -/*!40000 ALTER TABLE `TBL_PRIVS` DISABLE KEYS */; -/*!40000 ALTER TABLE `TBL_PRIVS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `TXNS` --- - -DROP TABLE IF EXISTS `TXNS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `TXNS` ( - `TXN_ID` bigint(20) NOT NULL, - `TXN_STATE` char(1) NOT NULL, - `TXN_STARTED` bigint(20) NOT NULL, - `TXN_LAST_HEARTBEAT` bigint(20) NOT NULL, - `TXN_USER` varchar(128) NOT NULL, - `TXN_HOST` varchar(128) NOT NULL, - `TXN_AGENT_INFO` varchar(128) DEFAULT NULL, - `TXN_META_INFO` varchar(128) DEFAULT NULL, - `TXN_HEARTBEAT_COUNT` int(11) DEFAULT NULL, - `TXN_TYPE` int(11) DEFAULT NULL, - PRIMARY KEY (`TXN_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `TXNS` --- - -LOCK TABLES `TXNS` WRITE; -/*!40000 ALTER TABLE `TXNS` DISABLE KEYS */; -/*!40000 ALTER TABLE `TXNS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `TXN_COMPONENTS` --- - -DROP TABLE IF EXISTS `TXN_COMPONENTS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `TXN_COMPONENTS` ( - `TC_TXNID` bigint(20) NOT NULL, - `TC_DATABASE` varchar(128) NOT NULL, - `TC_TABLE` varchar(128) DEFAULT NULL, - `TC_PARTITION` varchar(767) DEFAULT NULL, - `TC_OPERATION_TYPE` char(1) NOT NULL, - `TC_WRITEID` bigint(20) DEFAULT NULL, - KEY `TC_TXNID_INDEX` (`TC_TXNID`), - CONSTRAINT `TXN_COMPONENTS_ibfk_1` FOREIGN KEY (`TC_TXNID`) REFERENCES `TXNS` (`TXN_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `TXN_COMPONENTS` --- - -LOCK TABLES `TXN_COMPONENTS` WRITE; -/*!40000 ALTER TABLE `TXN_COMPONENTS` DISABLE KEYS */; -/*!40000 ALTER TABLE `TXN_COMPONENTS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `TXN_TO_WRITE_ID` --- - -DROP TABLE IF EXISTS `TXN_TO_WRITE_ID`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `TXN_TO_WRITE_ID` ( - `T2W_TXNID` bigint(20) NOT NULL, - `T2W_DATABASE` varchar(128) NOT NULL, - `T2W_TABLE` varchar(256) NOT NULL, - `T2W_WRITEID` bigint(20) NOT NULL, - UNIQUE KEY `TBL_TO_TXN_ID_IDX` (`T2W_DATABASE`,`T2W_TABLE`,`T2W_TXNID`), - UNIQUE KEY `TBL_TO_WRITE_ID_IDX` (`T2W_DATABASE`,`T2W_TABLE`,`T2W_WRITEID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `TXN_TO_WRITE_ID` --- - -LOCK TABLES `TXN_TO_WRITE_ID` WRITE; -/*!40000 ALTER TABLE `TXN_TO_WRITE_ID` DISABLE KEYS */; -/*!40000 ALTER TABLE `TXN_TO_WRITE_ID` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `TYPES` --- - -DROP TABLE IF EXISTS `TYPES`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `TYPES` ( - `TYPES_ID` bigint(20) NOT NULL, - `TYPE_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `TYPE1` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `TYPE2` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - PRIMARY KEY (`TYPES_ID`), - UNIQUE KEY `UNIQUE_TYPE` (`TYPE_NAME`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `TYPES` --- - -LOCK TABLES `TYPES` WRITE; -/*!40000 ALTER TABLE `TYPES` DISABLE KEYS */; -/*!40000 ALTER TABLE `TYPES` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `TYPE_FIELDS` --- - -DROP TABLE IF EXISTS `TYPE_FIELDS`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `TYPE_FIELDS` ( - `TYPE_NAME` bigint(20) NOT NULL, - `COMMENT` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, - `FIELD_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `FIELD_TYPE` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, - `INTEGER_IDX` int(11) NOT NULL, - PRIMARY KEY (`TYPE_NAME`,`FIELD_NAME`), - KEY `TYPE_FIELDS_N49` (`TYPE_NAME`), - CONSTRAINT `TYPE_FIELDS_FK1` FOREIGN KEY (`TYPE_NAME`) REFERENCES `TYPES` (`TYPES_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `TYPE_FIELDS` --- - -LOCK TABLES `TYPE_FIELDS` WRITE; -/*!40000 ALTER TABLE `TYPE_FIELDS` DISABLE KEYS */; -/*!40000 ALTER TABLE `TYPE_FIELDS` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `VERSION` --- - -DROP TABLE IF EXISTS `VERSION`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `VERSION` ( - `VER_ID` bigint(20) NOT NULL, - `SCHEMA_VERSION` varchar(127) NOT NULL, - `VERSION_COMMENT` varchar(255) DEFAULT NULL, - PRIMARY KEY (`VER_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `VERSION` --- - -LOCK TABLES `VERSION` WRITE; -/*!40000 ALTER TABLE `VERSION` DISABLE KEYS */; -INSERT INTO `VERSION` VALUES (1,'3.1.0','Hive release version 3.1.0'); -/*!40000 ALTER TABLE `VERSION` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `WM_MAPPING` --- - -DROP TABLE IF EXISTS `WM_MAPPING`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `WM_MAPPING` ( - `MAPPING_ID` bigint(20) NOT NULL, - `RP_ID` bigint(20) NOT NULL, - `ENTITY_TYPE` varchar(128) NOT NULL, - `ENTITY_NAME` varchar(128) NOT NULL, - `POOL_ID` bigint(20) DEFAULT NULL, - `ORDERING` int(11) DEFAULT NULL, - PRIMARY KEY (`MAPPING_ID`), - UNIQUE KEY `UNIQUE_WM_MAPPING` (`RP_ID`,`ENTITY_TYPE`,`ENTITY_NAME`), - KEY `WM_MAPPING_FK2` (`POOL_ID`), - CONSTRAINT `WM_MAPPING_FK1` FOREIGN KEY (`RP_ID`) REFERENCES `WM_RESOURCEPLAN` (`RP_ID`), - CONSTRAINT `WM_MAPPING_FK2` FOREIGN KEY (`POOL_ID`) REFERENCES `WM_POOL` (`POOL_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `WM_MAPPING` --- - -LOCK TABLES `WM_MAPPING` WRITE; -/*!40000 ALTER TABLE `WM_MAPPING` DISABLE KEYS */; -/*!40000 ALTER TABLE `WM_MAPPING` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `WM_POOL` --- - -DROP TABLE IF EXISTS `WM_POOL`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `WM_POOL` ( - `POOL_ID` bigint(20) NOT NULL, - `RP_ID` bigint(20) NOT NULL, - `PATH` varchar(767) NOT NULL, - `ALLOC_FRACTION` double DEFAULT NULL, - `QUERY_PARALLELISM` int(11) DEFAULT NULL, - `SCHEDULING_POLICY` varchar(767) DEFAULT NULL, - PRIMARY KEY (`POOL_ID`), - UNIQUE KEY `UNIQUE_WM_POOL` (`RP_ID`,`PATH`), - CONSTRAINT `WM_POOL_FK1` FOREIGN KEY (`RP_ID`) REFERENCES `WM_RESOURCEPLAN` (`RP_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `WM_POOL` --- - -LOCK TABLES `WM_POOL` WRITE; -/*!40000 ALTER TABLE `WM_POOL` DISABLE KEYS */; -/*!40000 ALTER TABLE `WM_POOL` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `WM_POOL_TO_TRIGGER` --- - -DROP TABLE IF EXISTS `WM_POOL_TO_TRIGGER`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `WM_POOL_TO_TRIGGER` ( - `POOL_ID` bigint(20) NOT NULL, - `TRIGGER_ID` bigint(20) NOT NULL, - PRIMARY KEY (`POOL_ID`,`TRIGGER_ID`), - KEY `WM_POOL_TO_TRIGGER_FK2` (`TRIGGER_ID`), - CONSTRAINT `WM_POOL_TO_TRIGGER_FK1` FOREIGN KEY (`POOL_ID`) REFERENCES `WM_POOL` (`POOL_ID`), - CONSTRAINT `WM_POOL_TO_TRIGGER_FK2` FOREIGN KEY (`TRIGGER_ID`) REFERENCES `WM_TRIGGER` (`TRIGGER_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `WM_POOL_TO_TRIGGER` --- - -LOCK TABLES `WM_POOL_TO_TRIGGER` WRITE; -/*!40000 ALTER TABLE `WM_POOL_TO_TRIGGER` DISABLE KEYS */; -/*!40000 ALTER TABLE `WM_POOL_TO_TRIGGER` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `WM_RESOURCEPLAN` --- - -DROP TABLE IF EXISTS `WM_RESOURCEPLAN`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `WM_RESOURCEPLAN` ( - `RP_ID` bigint(20) NOT NULL, - `NAME` varchar(128) NOT NULL, - `QUERY_PARALLELISM` int(11) DEFAULT NULL, - `STATUS` varchar(20) NOT NULL, - `DEFAULT_POOL_ID` bigint(20) DEFAULT NULL, - PRIMARY KEY (`RP_ID`), - UNIQUE KEY `UNIQUE_WM_RESOURCEPLAN` (`NAME`), - KEY `WM_RESOURCEPLAN_FK1` (`DEFAULT_POOL_ID`), - CONSTRAINT `WM_RESOURCEPLAN_FK1` FOREIGN KEY (`DEFAULT_POOL_ID`) REFERENCES `WM_POOL` (`POOL_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `WM_RESOURCEPLAN` --- - -LOCK TABLES `WM_RESOURCEPLAN` WRITE; -/*!40000 ALTER TABLE `WM_RESOURCEPLAN` DISABLE KEYS */; -/*!40000 ALTER TABLE `WM_RESOURCEPLAN` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `WM_TRIGGER` --- - -DROP TABLE IF EXISTS `WM_TRIGGER`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `WM_TRIGGER` ( - `TRIGGER_ID` bigint(20) NOT NULL, - `RP_ID` bigint(20) NOT NULL, - `NAME` varchar(128) NOT NULL, - `TRIGGER_EXPRESSION` varchar(1024) DEFAULT NULL, - `ACTION_EXPRESSION` varchar(1024) DEFAULT NULL, - `IS_IN_UNMANAGED` bit(1) NOT NULL DEFAULT b'0', - PRIMARY KEY (`TRIGGER_ID`), - UNIQUE KEY `UNIQUE_WM_TRIGGER` (`RP_ID`,`NAME`), - CONSTRAINT `WM_TRIGGER_FK1` FOREIGN KEY (`RP_ID`) REFERENCES `WM_RESOURCEPLAN` (`RP_ID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `WM_TRIGGER` --- - -LOCK TABLES `WM_TRIGGER` WRITE; -/*!40000 ALTER TABLE `WM_TRIGGER` DISABLE KEYS */; -/*!40000 ALTER TABLE `WM_TRIGGER` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `WRITE_SET` --- - -DROP TABLE IF EXISTS `WRITE_SET`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `WRITE_SET` ( - `WS_DATABASE` varchar(128) NOT NULL, - `WS_TABLE` varchar(128) NOT NULL, - `WS_PARTITION` varchar(767) DEFAULT NULL, - `WS_TXNID` bigint(20) NOT NULL, - `WS_COMMIT_ID` bigint(20) NOT NULL, - `WS_OPERATION_TYPE` char(1) NOT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `WRITE_SET` --- - -LOCK TABLES `WRITE_SET` WRITE; -/*!40000 ALTER TABLE `WRITE_SET` DISABLE KEYS */; -/*!40000 ALTER TABLE `WRITE_SET` ENABLE KEYS */; -UNLOCK TABLES; -/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; - -/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; -/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; -/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; -/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; -/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; -/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; -/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; - --- Dump completed on 2021-10-18 13:43:15 diff --git a/tests/system/providers/google/cloud/dataproc/resources/pip-install.sh b/tests/system/providers/google/cloud/dataproc/resources/pip-install.sh deleted file mode 100644 index a9839b5b80f5f..0000000000000 --- a/tests/system/providers/google/cloud/dataproc/resources/pip-install.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -exo pipefail - -PACKAGES=$(/usr/share/google/get_metadata_value attributes/PIP_PACKAGES || true) -readonly PACKAGES - -function err() { - echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $*" >&2 - exit 1 -} - -function run_with_retry() { - local -r cmd=("$@") - for ((i = 0; i < 10; i++)); do - if "${cmd[@]}"; then - return 0 - fi - sleep 5 - done - err "Failed to run command: ${cmd[*]}" -} - -function install_pip() { - if command -v pip >/dev/null; then - echo "pip is already installed." - return 0 - fi - - if command -v easy_install >/dev/null; then - echo "Installing pip with easy_install..." - run_with_retry easy_install pip - return 0 - fi - - echo "Installing python-pip..." - run_with_retry apt update - run_with_retry apt install python-pip -y -} - -function main() { - if [[ -z "${PACKAGES}" ]]; then - echo "ERROR: Must specify PIP_PACKAGES metadata key" - exit 1 - fi - - install_pip - # shellcheck disable=SC2086 - run_with_retry pip install --upgrade ${PACKAGES} -} - -main diff --git a/tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py b/tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py index 9c9d3b1d56e1b..0d0c41f52fd9b 100644 --- a/tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py +++ b/tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py @@ -24,7 +24,6 @@ import datetime import os -from pathlib import Path from google.protobuf.field_mask_pb2 import FieldMask @@ -37,26 +36,29 @@ DataprocMetastoreGetServiceOperator, DataprocMetastoreUpdateServiceOperator, ) -from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator -from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator +from airflow.providers.google.cloud.operators.gcs import ( + GCSCreateBucketOperator, + GCSDeleteBucketOperator, + GCSSynchronizeBucketsOperator, +) from airflow.utils.trigger_rule import TriggerRule +from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID DAG_ID = "dataproc_metastore" -PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "") -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") SERVICE_ID = f"{DAG_ID}-service-{ENV_ID}".replace("_", "-") METADATA_IMPORT_ID = f"{DAG_ID}-metadata-{ENV_ID}".replace("_", "-") REGION = "europe-west1" +RESOURCE_DATA_BUCKET = "airflow-system-tests-resources" BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}" TIMEOUT = 2400 DB_TYPE = "MYSQL" DESTINATION_GCS_FOLDER = f"gs://{BUCKET_NAME}/>" - -HIVE_FILE_SRC = str(Path(__file__).parent.parent / "dataproc" / "resources" / "hive.sql") -HIVE_FILE = "data/hive.sql" -GCS_URI = f"gs://{BUCKET_NAME}/data/hive.sql" +HIVE_FILE = "hive.sql" +GCS_URI = f"gs://{BUCKET_NAME}/dataproc/{HIVE_FILE}" # Service definition # Docs: https://cloud.google.com/dataproc-metastore/docs/reference/rest/v1/projects.locations.services#Service @@ -98,11 +100,13 @@ task_id="create_bucket", bucket_name=BUCKET_NAME, project_id=PROJECT_ID ) - upload_file = LocalFilesystemToGCSOperator( - task_id="upload_file", - src=HIVE_FILE_SRC, - dst=HIVE_FILE, - bucket=BUCKET_NAME, + move_file = GCSSynchronizeBucketsOperator( + task_id="move_file", + source_bucket=RESOURCE_DATA_BUCKET, + source_object="dataproc/hive", + destination_bucket=BUCKET_NAME, + destination_object="dataproc", + recursive=True, ) # [START how_to_cloud_dataproc_metastore_create_service_operator] @@ -177,7 +181,7 @@ ( create_bucket - >> upload_file + >> move_file >> create_service >> get_service >> update_service diff --git a/tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py b/tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py index 1dbfbf409040a..6a5a7566b3429 100644 --- a/tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py +++ b/tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py @@ -35,11 +35,12 @@ DataprocMetastoreRestoreServiceOperator, ) from airflow.utils.trigger_rule import TriggerRule +from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID DAG_ID = "dataproc_metastore_backup" -PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "") -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") SERVICE_ID = f"{DAG_ID}-service-{ENV_ID}".replace("_", "-") BACKUP_ID = f"{DAG_ID}-backup-{ENV_ID}".replace("_", "-") diff --git a/tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_hive_partition_sensor.py b/tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_hive_partition_sensor.py index 5c2be86bd8785..07677e11cbf36 100644 --- a/tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_hive_partition_sensor.py +++ b/tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_hive_partition_sensor.py @@ -42,10 +42,11 @@ from airflow.providers.google.cloud.sensors.dataproc_metastore import MetastoreHivePartitionSensor from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator from airflow.utils.trigger_rule import TriggerRule +from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID DAG_ID = "hive_partition_sensor" -PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "demo-project") -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "demo-env") +PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") REGION = "europe-west1" NETWORK = "default"