From d625b646dfc23e9080b6141479b58d94f05b5b61 Mon Sep 17 00:00:00 2001 From: daidai <2017501503@qq.com> Date: Thu, 9 Nov 2023 18:52:17 +0800 Subject: [PATCH] [fix](information_schema)fix bug that metadata_name_ids error tableid and append information_schema case #26238 (#26646) fix bug that #24059 . Added some information_schema scanner tests. files schema_privileges table_privileges partitions rowsets statistics table_constraints Based on infodb_support_ext_catalog=false, it currently includes tests for all tables under the information_schema database. --- be/src/exec/schema_scanner.cpp | 2 - be/src/exec/schema_scanner.h | 6 +- be/src/exec/schema_scanner/schema_helper.cpp | 9 - be/src/exec/schema_scanner/schema_helper.h | 4 - .../schema_metadata_name_ids_scanner.cpp | 4 +- .../query_p0/system/test_query_sys_tables.out | 164 ++++++++++++++++++ .../system/test_query_sys_tables.groovy | 75 +++++++- 7 files changed, 241 insertions(+), 23 deletions(-) diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index 9733558284a8fd..8d640341bfb9a6 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -60,8 +60,6 @@ namespace doris { class ObjectPool; -DorisServer* SchemaScanner::_s_doris_server; - SchemaScanner::SchemaScanner(const std::vector& columns) : _is_init(false), _param(nullptr), diff --git a/be/src/exec/schema_scanner.h b/be/src/exec/schema_scanner.h index e42cdde60f38f6..874b4a34b7014c 100644 --- a/be/src/exec/schema_scanner.h +++ b/be/src/exec/schema_scanner.h @@ -33,7 +33,7 @@ namespace doris { // forehead declare class, because jni function init in DorisServer. -class DorisServer; + class RuntimeState; class ObjectPool; class TUserIdentity; @@ -96,8 +96,6 @@ class SchemaScanner { static std::unique_ptr create(TSchemaTableType::type type); TSchemaTableType::type type() const { return _schema_table_type; } - static void set_doris_server(DorisServer* doris_server) { _s_doris_server = doris_server; } - protected: Status fill_dest_column_for_range(vectorized::Block* block, size_t pos, const std::vector& datas); @@ -108,8 +106,6 @@ class SchemaScanner { // schema table's column desc std::vector _columns; - static DorisServer* _s_doris_server; - TSchemaTableType::type _schema_table_type; RuntimeProfile::Counter* _get_db_timer = nullptr; diff --git a/be/src/exec/schema_scanner/schema_helper.cpp b/be/src/exec/schema_scanner/schema_helper.cpp index 3184aed4d2e4f5..b1d6900e0f5c2f 100644 --- a/be/src/exec/schema_scanner/schema_helper.cpp +++ b/be/src/exec/schema_scanner/schema_helper.cpp @@ -69,15 +69,6 @@ Status SchemaHelper::list_table_metadata_name_ids(const std::string& ip, const i }); } -Status SchemaHelper::describe_table(const std::string& ip, const int32_t port, - const TDescribeTableParams& request, - TDescribeTableResult* result) { - return ThriftRpcHelper::rpc( - ip, port, [&request, &result](FrontendServiceConnection& client) { - client->describeTable(*result, request); - }); -} - Status SchemaHelper::describe_tables(const std::string& ip, const int32_t port, const TDescribeTablesParams& request, TDescribeTablesResult* result) { diff --git a/be/src/exec/schema_scanner/schema_helper.h b/be/src/exec/schema_scanner/schema_helper.h index 900f963f7893cc..c0143136115bf1 100644 --- a/be/src/exec/schema_scanner/schema_helper.h +++ b/be/src/exec/schema_scanner/schema_helper.h @@ -55,10 +55,6 @@ class SchemaHelper { const doris::TGetTablesParams& request, TListTableMetadataNameIdsResult* result); - static Status describe_table(const std::string& ip, const int32_t port, - const TDescribeTableParams& desc_params, - TDescribeTableResult* desc_result); - static Status describe_tables(const std::string& ip, const int32_t port, const TDescribeTablesParams& desc_params, TDescribeTablesResult* desc_result); diff --git a/be/src/exec/schema_scanner/schema_metadata_name_ids_scanner.cpp b/be/src/exec/schema_scanner/schema_metadata_name_ids_scanner.cpp index f99d05dc276bc3..70fb11ec4f77eb 100644 --- a/be/src/exec/schema_scanner/schema_metadata_name_ids_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_metadata_name_ids_scanner.cpp @@ -192,13 +192,13 @@ Status SchemaMetadataNameIdsScanner::_fill_block_impl(vectorized::Block* block) fill_dest_column_for_range(block, 3, null_datas); } } - // table_id + // table_id { int64_t srcs[table_num]; for (int i = 0; i < table_num; ++i) { if (_table_result.tables[i].__isset.id) { srcs[i] = _table_result.tables[i].id; - datas[i] = &srcs; + datas[i] = srcs + i; } else { datas[i] = nullptr; } diff --git a/regression-test/data/query_p0/system/test_query_sys_tables.out b/regression-test/data/query_p0/system/test_query_sys_tables.out index 9d87b432b4a022..91f54556ae8a35 100644 --- a/regression-test/data/query_p0/system/test_query_sys_tables.out +++ b/regression-test/data/query_p0/system/test_query_sys_tables.out @@ -19,12 +19,175 @@ internal bbb 2 varchar varchar(20) 20 internal ccc 3 int int(11) 10 internal ddd 4 smallint smallint(6) 5 +-- !desc_files -- +FILE_ID BIGINT Yes false \N +FILE_NAME TEXT Yes false \N +FILE_TYPE VARCHAR(256) Yes false \N +TABLESPACE_NAME VARCHAR(256) Yes false \N +TABLE_CATALOG CHAR(16) Yes false \N +TABLE_SCHEMA TEXT Yes false \N +TABLE_NAME TEXT Yes false \N +LOGFILE_GROUP_NAME VARCHAR(256) Yes false \N +LOGFILE_GROUP_NUMBER BIGINT Yes false \N +ENGINE VARCHAR(64) Yes false \N +FULLTEXT_KEYS TEXT Yes false \N +DELETED_ROWS TEXT Yes false \N +UPDATE_COUNT TEXT Yes false \N +FREE_EXTENTS BIGINT Yes false \N +TOTAL_EXTENTS BIGINT Yes false \N +EXTENT_SIZE BIGINT Yes false \N +INITIAL_SIZE BIGINT Yes false \N +MAXIMUM_SIZE BIGINT Yes false \N +AUTOEXTEND_SIZE BIGINT Yes false \N +CREATION_TIME TEXT Yes false \N +LAST_UPDATE_TIME TEXT Yes false \N +LAST_ACCESS_TIME TEXT Yes false \N +RECOVER_TIME TEXT Yes false \N +TRANSACTION_COUNTER TEXT Yes false \N +VERSION BIGINT Yes false \N +ROW_FORMAT VARCHAR(256) Yes false \N +TABLE_ROWS TEXT Yes false \N +AVG_ROW_LENGTH TEXT Yes false \N +DATA_LENGTH TEXT Yes false \N +MAX_DATA_LENGTH TEXT Yes false \N +INDEX_LENGTH TEXT Yes false \N +DATA_FREE BIGINT Yes false \N +CREATE_TIME TEXT Yes false \N +UPDATE_TIME TEXT Yes false \N +CHECK_TIME TEXT Yes false \N +CHECKSUM TEXT Yes false \N +STATUS VARCHAR(256) Yes false \N +EXTRA VARCHAR(256) Yes false \N + +-- !query_files -- + +-- !desc_statistics -- +TABLE_CATALOG VARCHAR(512) Yes false \N +TABLE_SCHEMA VARCHAR(64) Yes false \N +TABLE_NAME VARCHAR(64) Yes false \N +NON_UNIQUE BIGINT Yes false \N +INDEX_SCHEMA VARCHAR(64) Yes false \N +INDEX_NAME VARCHAR(64) Yes false \N +SEQ_IN_INDEX BIGINT Yes false \N +COLUMN_NAME VARCHAR(64) Yes false \N +COLLATION VARCHAR(1) Yes false \N +CARDINALITY BIGINT Yes false \N +SUB_PART BIGINT Yes false \N +PACKED VARCHAR(10) Yes false \N +NULLABLE VARCHAR(3) Yes false \N +INDEX_TYPE VARCHAR(16) Yes false \N +COMMENT VARCHAR(16) Yes false \N +INDEX_COMMENT VARCHAR(1024) Yes false \N + +-- !query_statistics -- + +-- !desc_statistics -- +CONSTRAINT_CATALOG VARCHAR(512) Yes false \N +CONSTRAINT_SCHEMA VARCHAR(64) Yes false \N +CONSTRAINT_NAME VARCHAR(64) Yes false \N +TABLE_SCHEMA VARCHAR(64) Yes false \N +TABLE_NAME VARCHAR(64) Yes false \N +CONSTRAINT_TYPE VARCHAR(64) Yes false \N + +-- !query_table_constraints -- + +-- !desc_schema_privileges -- +GRANTEE VARCHAR(81) Yes false \N +TABLE_CATALOG VARCHAR(512) Yes false \N +TABLE_SCHEMA VARCHAR(64) Yes false \N +PRIVILEGE_TYPE VARCHAR(64) Yes false \N +IS_GRANTABLE VARCHAR(3) Yes false \N + +-- !schema_privileges1 -- +'root'@'%' def mysql SELECT NO + +-- !schema_privileges2 -- +'cyw'@'%' def mysql SELECT NO + +-- !schema_privileges3 -- + +-- !desc_table_privileges -- +GRANTEE VARCHAR(81) Yes false \N +TABLE_CATALOG VARCHAR(512) Yes false \N +TABLE_SCHEMA VARCHAR(64) Yes false \N +TABLE_NAME VARCHAR(64) Yes false \N +PRIVILEGE_TYPE VARCHAR(64) Yes false \N +IS_GRANTABLE VARCHAR(3) Yes false \N + +-- !table_privileges -- + +-- !table_privileges2 -- +'cywtable'@'%' def table_privileges_demo test_table_privileges ALTER NO +'cywtable'@'%' def table_privileges_demo test_table_privileges INSERT NO +'cywtable'@'%' def table_privileges_demo test_table_privileges SELECT NO + +-- !table_privileges3 -- +'cywtable'@'%' def table_privileges_demo test_table_privileges ALTER NO +'cywtable'@'%' def table_privileges_demo test_table_privileges INSERT NO + +-- !desc_partitions -- +TABLE_CATALOG VARCHAR(64) Yes false \N +TABLE_SCHEMA VARCHAR(64) Yes false \N +TABLE_NAME VARCHAR(64) Yes false \N +PARTITION_NAME VARCHAR(64) Yes false \N +SUBPARTITION_NAME VARCHAR(64) Yes false \N +PARTITION_ORDINAL_POSITION INT Yes false \N +SUBPARTITION_ORDINAL_POSITION INT Yes false \N +PARTITION_METHOD VARCHAR(13) Yes false \N +SUBPARTITION_METHOD VARCHAR(13) Yes false \N +PARTITION_EXPRESSION VARCHAR(2048) Yes false \N +SUBPARTITION_EXPRESSION VARCHAR(2048) Yes false \N +PARTITION_DESCRIPTION TEXT Yes false \N +TABLE_ROWS BIGINT Yes false \N +AVG_ROW_LENGTH BIGINT Yes false \N +DATA_LENGTH BIGINT Yes false \N +MAX_DATA_LENGTH BIGINT Yes false \N +INDEX_LENGTH BIGINT Yes false \N +DATA_FREE BIGINT Yes false \N +CREATE_TIME BIGINT Yes false \N +UPDATE_TIME DATETIME Yes false \N +CHECK_TIME DATETIME Yes false \N +CHECKSUM BIGINT Yes false \N +PARTITION_COMMENT TEXT Yes false \N +NODEGROUP VARCHAR(256) Yes false \N +TABLESPACE_NAME VARCHAR(268) Yes false \N + +-- !select_partitions -- + +-- !desc_rowsets -- +BACKEND_ID BIGINT Yes false \N +ROWSET_ID VARCHAR(64) Yes false \N +TABLET_ID BIGINT Yes false \N +ROWSET_NUM_ROWS BIGINT Yes false \N +TXN_ID BIGINT Yes false \N +NUM_SEGMENTS BIGINT Yes false \N +START_VERSION BIGINT Yes false \N +END_VERSION BIGINT Yes false \N +INDEX_DISK_SIZE BIGINT Yes false \N +DATA_DISK_SIZE BIGINT Yes false \N +CREATION_TIME BIGINT Yes false \N +NEWEST_WRITE_TIMESTAMP BIGINT Yes false \N + +-- !rowsets1 -- +0 1 + +-- !rowsets2 -- +0 1 +2 2 + +-- !rowsets3 -- +0 1 +2 2 +3 3 +4 4 + -- !schemata -- internal test_query_sys_db_1 \N internal test_query_sys_db_2 \N internal test_query_sys_db_3 \N -- !tables -- +internal test_query_rowset BASE TABLE 0 \N \N internal test_query_sys_tb_1 BASE TABLE 0 \N \N internal test_query_sys_tb_2 BASE TABLE 0 \N \N internal test_query_sys_tb_3 BASE TABLE 0 \N \N @@ -58,3 +221,4 @@ test_view -- !sql -- -- !sql -- + diff --git a/regression-test/suites/query_p0/system/test_query_sys_tables.groovy b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy index baf5d3aa7ea110..ad4a5f12bb1dd5 100644 --- a/regression-test/suites/query_p0/system/test_query_sys_tables.groovy +++ b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy @@ -87,12 +87,85 @@ suite("test_query_sys_tables", "query,p0") { // test files // have no impl + qt_desc_files """desc `information_schema`.`files` """ + order_qt_query_files """ select * from `information_schema`.`files` """ + + //test information_schema.statistics + // have no impl + qt_desc_statistics """desc `information_schema`.`statistics` """ + order_qt_query_statistics """ select * from `information_schema`.`statistics` """ + + //test information_schema.table_constraints + // have no impl + qt_desc_statistics """desc `information_schema`.`table_constraints` """ + order_qt_query_table_constraints """ select * from `information_schema`.`table_constraints` """ + + + // test schema_privileges + sql """ DROP USER if exists 'cyw'; """ + qt_desc_schema_privileges """desc `information_schema`.`schema_privileges` """ + order_qt_schema_privileges1 """ select * from information_schema.schema_privileges where GRANTEE = "'root'@'%'" ; """ + sql """ CREATE USER 'cyw'; """ + order_qt_schema_privileges2 """ select * from information_schema.schema_privileges where GRANTEE = "'cyw'@'%'" ; """ + sql """ DROP USER 'cyw'; """ + order_qt_schema_privileges3 """ select * from information_schema.schema_privileges where GRANTEE = "'cyw'@'%'" ; """ + + + // test table_privileges + sql """ DROP USER if exists 'cywtable'; """ + qt_desc_table_privileges """desc `information_schema`.`table_privileges` """ + order_qt_table_privileges """ select * from information_schema.table_privileges """ + sql """ CREATE USER 'cywtable'; """ + sql """ CREATE DATABASE IF NOT EXISTS table_privileges_demo """ + sql """ create table IF NOT EXISTS table_privileges_demo.test_table_privileges( + a int , + b boolean , + c string ) + DISTRIBUTED BY HASH(`a`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true", + "enable_single_replica_compaction"="true" + );""" + + sql """ GRANT SELECT_PRIV,ALTER_PRIV,LOAD_PRIV ON table_privileges_demo.test_table_privileges TO 'cywtable'@'%'; """ + order_qt_table_privileges2 """ select * from information_schema.table_privileges order by PRIVILEGE_TYPE ; """ + sql """ REVOKE SELECT_PRIV ON table_privileges_demo.test_table_privileges FROM 'cywtable'@'%'; """ + order_qt_table_privileges3 """ select * from information_schema.table_privileges order by PRIVILEGE_TYPE ; """ + // test partitions // have no impl + qt_desc_partitions """ desc `information_schema`.`partitions` """ + order_qt_select_partitions """ select * from `information_schema`.`partitions`; """ + // test rowsets - // have no tablet system table, add this later + qt_desc_rowsets """ desc information_schema.rowsets """ + def rowsets_table_name = """ test_query_sys_db_1.test_query_rowset """ + sql """ drop table if exists ${rowsets_table_name} """ + + sql """ + create table ${rowsets_table_name}( + a int , + b boolean , + c string ) + DISTRIBUTED BY HASH(`a`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true", + "enable_single_replica_compaction"="true" + ); + """ + + List> rowsets_table_name_tablets = sql """ show tablets from ${rowsets_table_name} """ + order_qt_rowsets1 """ select START_VERSION,END_VERSION from information_schema.rowsets where TABLET_ID=${rowsets_table_name_tablets[0][0]} order by START_VERSION,END_VERSION; """ + sql """ insert into ${rowsets_table_name} values (1,0,"abc"); """ + order_qt_rowsets2 """ select START_VERSION,END_VERSION from information_schema.rowsets where TABLET_ID=${rowsets_table_name_tablets[0][0]} order by START_VERSION,END_VERSION; """ + sql """ insert into ${rowsets_table_name} values (2,1,"hello world"); """ + sql """ insert into ${rowsets_table_name} values (3,0,"dssadasdsafafdf"); """ + order_qt_rowsets3 """ select START_VERSION,END_VERSION from information_schema.rowsets where TABLET_ID=${rowsets_table_name_tablets[0][0]} order by START_VERSION,END_VERSION; """ + // test schemata // create test dbs