From 9144b48f52121c21a72ba3d6782579959b7377dc Mon Sep 17 00:00:00 2001 From: Satya Bodapati Date: Wed, 24 Apr 2019 14:09:13 -0700 Subject: [PATCH] FB8-59: Database admission control (#969) (#969) Summary: JIRA: https://jira.percona.com/browse/FB8-59 Reference patch: https://github.com/facebook/mysql-5.6/commit/5cbfab0 Reference patch: https://github.com/facebook/mysql-5.6/commit/0ebaf75 Reference patch: https://github.com/facebook/mysql-5.6/commit/cf7ce48 Reference patch: https://github.com/facebook/mysql-5.6/commit/3842a8f Reference patch: https://github.com/facebook/mysql-5.6/commit/bcbd77a Reference patch: https://github.com/facebook/mysql-5.6/commit/e6c2b87 Reference patch: https://github.com/facebook/mysql-5.6/commit/928c2a5 Reference patch: https://github.com/facebook/mysql-5.6/commit/86587af Reference patch: https://github.com/facebook/mysql-5.6/commit/7870828 Reference patch: https://github.com/facebook/mysql-5.6/commit/0151b45 Reference patch: https://github.com/facebook/mysql-5.6/commit/75de821 Reference patch: https://github.com/facebook/mysql-5.6/commit/08d4174 Reference patch: https://github.com/facebook/mysql-5.6/commit/c1e5b4f Reference patch: https://github.com/facebook/mysql-5.6/commit/08368d4 Reference patch: https://github.com/facebook/mysql-5.6/commit/a2198e0 Reference patch: https://github.com/facebook/mysql-5.6/commit/0730342 Reference patch: https://github.com/facebook/mysql-5.6/commit/d868c1b Reference patch: https://github.com/facebook/mysql-5.6/commit/645fc5e Reference patch: https://github.com/facebook/mysql-5.6/commit/0b130f1ebc9 Reference patch: https://github.com/facebook/mysql-5.6/commit/f49a4a74945 This patch ports Database admission control. The multi-tenancy plugin changes are not ported. This code will be eventually moved to new plugin of audit type. There are possible improvements with code. Below are the deferred items (makes it easier for review. Can compare with 5.6 code) 1. Remove MT_RESOURCE_ATTRS. we only need database name (also removes the typedefs) 2. Remove enums enum_multi_tenancy_resource_type & enum_multi_tenancy_return_type 3. Remove the usage of 'multi tenancy' word thd->is_real_trans is not ported from 5.6. Instead an equivalent is used. This is verified in 5.6 and it works well. (5.6 equivalent is: thd->transaction.all.ha_list == NULL;) "!thd->get_transaction()->is_active(Transaction_ctx::SESSION)" is used instead of thd->is_real_trans. The above check seems sufficient but after porting thd->is_real_trans, it should be possible to replace the above check with thd->is_real_trans ------- https://github.com/facebook/mysql-5.6/commit/5cbfab0 -------- Add two global sys vars max_running_queries and max_waiting_queries. They control maximum number of running queries on a database and maximum waiting queries when max_running_queries limit is crossed on that database. A value of 0 implies no limits are applied for the queries. If max_waiting_queries limit is crossed, then new queries will simply fail. For waiting threads SHOW PROCESSLIST will show the state as 'waiting for admission'. Only dagtabase set at the session level is considered in these checks. Admission checks are by-passed in the following cases 1. Query is run by super user. 2. Query is run by replication threads. 3. No database is set for the session. 4. max_running_queries is 0. Performance testing 1. start MySQL using mtr > mtr --start --mysqld=--default_storage_engine=InnoDB 2. Connect to mysql using > mysql --socket=mysql-test/var/tmp/mysqld.1.sock --user=root 3. Set innodb_flush_log_at_trx_commit=2. mysql> set global.innodb_flush_log_at_trx_commit=2 4. Check rows inserted using > mysqladmin extended-status --user=root -i 1 -r --socket=mysql-test/var/tmp/mysqld.1.sock status | grep "Innodb_rows_inserted" 5. Run mysqlslap using > mysqlslap --auto-generate-sql --number-of-queries=5000000 --concurrency=$concurrency --auto-generate-sql-load-type=write --auto-generate-sql-add-autoincrement --csv=a --user=root --socket=mysql-test/var/tmp/mysqld.1.sock == Innodb_rows_inserted with concurrency=100 == 38106 (max_running_queries=0, max_waiting_queries=0) 38733 (max_running_queries=1000, max_waiting_queries=0) 38365 (max_running_queries=100, max_waiting_queries=0) 40308 (max_running_queries=70, max_waiting_queries=0) 43790 (max_running_queries=50, max_waiting_queries=0) 47321 (max_running_queries=20, max_waiting_queries=0) 49473 (max_running_queries=10, max_waiting_queries=0) 54851 (max_running_queries=5, max_waiting_queries=0) == Innodb_rows_inserted with concurrency=100 == 26516 (max_running_queries=0, max_waiting_queries=0) 44494 (max_running_queries=10, max_waiting_queries=0) 51193 (max_running_queries=5, max_waiting_queries=0) The above numbers show the improvement in quality of service. == Innodb_rows_inserted with concurrency=20 == 47228 (max_running_queries=0, max_waiting_queries=0) 47107 (max_running_queries=100, max_waiting_queries=0) 47082 (max_running_queries=20, max_waiting_queries=0) 47515 (max_running_queries=10, max_waiting_queries=0) * There isn't much difference under low concurrency workloads. * There isn't much difference with non zero max_waiting_queries values greater than max_running_queries. With values lower than max_running_queries, lots of queries simply fail, so the qps in this scenario is of no value. Original Reviewers: tianx, kradhakrishnan, jtolmer, jkedgar ----- https://github.com/facebook/mysql-5.6/commit/0ebaf75 ----- Added the following status variables: 1. Database_admission_control_aborted_queries gives the total number of queries aborted because of queue overfill. 2. Database_admission_control_running_queries gives the total number of running queries across all the databases. 3. Database_admission_control_waiting_queries gives the total number of waiting queries across all the databases. ----- https://github.com/facebook/mysql-5.6/commit/cf7ce48 ------ In order to avoid possible mysqld hangs, we need an option to blacklist specific commands in admission control. Example usage is set global.admission_control_filter='BEGIN,COMMIT,SET'; An empty value for admission_control_filter implies no commands are blacklisted in the admission control. ---- https://github.com/facebook/mysql-5.6/commit/bcbd77a ------ Initial version of admission control applied limits on each query statement in multi query packet. The behavior is modified to the following: A thread enters admission control when executing the first non-filtered command in the multi query packet and exits after executing all the query statements in the multi query packet. This diff also fixes a bug where admission control limits are not initialized during mysqld startup. ----- https://github.com/facebook/mysql-5.6/commit/928c2a5 -------- Get accurate status variables Hold global admission control locks to get accurate value of running/waiting queries. Initial stats were not accurate causing test failures ------- https://github.com/facebook/mysql-5.6/commit/86587af ----------- Added SHOW command filter in admission control. ------- https://github.com/facebook/mysql-5.6/commit/7870828 ------ The original ac_map was a map of unique_ptr. When accessing the value we had to use a reference to the value. This diff changes that so that we use a shared_ptr instead so we can just pass it around have have reference counting automatically. ---- https://github.com/facebook/mysql-5.6/commit/75de821 ------ Fixing the following two issues in admission control when changing database is in a multi-statement query: - If the session hasn't set a database, admission control will allow arbitrary number of multi-query packet to run, even the packet starts with `use [db_name]`. - If the session already has a default database, the admission control will use the limit of the default database, even if we change database in the multi-query packet. This diff fixes the above issues in the following way: - The admission control flag (is_in_ac) is reset when a `use` command is executed. So in the multi-statement query, the subsequent sub-queries are not bypassed. - Admission control uses the `thd->db` directly instead of attribute map, so the attribute map doesn't need to store the session database. - `USE [db_name]` statement is added to `admission_control_filter`. In addition, I also changed the multitenancy plugin interface to use a wrapper object to pass down the connection_attrs_map, query_attrs_map, and session database info. Attribute maps will not be modified. --------- https://github.com/facebook/mysql-5.6/commit/c1e5b4f -------- Fix resource leaks by admission_control_by_trx - Rollback will need to mark is_real_trans - Query resource needs to be released when THD is terminated. Pull Request resolved: https://github.com/facebook/mysql-5.6/pull/969 Reviewed By: lloyd Differential Revision: D14567747 Pulled By: lth --- mysql-test/r/admission_control.result | 117 ++++++ mysql-test/r/admission_control_hang.result | 93 +++++ .../r/admission_control_multi_query.result | 8 + mysql-test/r/admission_control_stress.result | 5 + mysql-test/r/information_schema_ci.result | 3 + mysql-test/r/information_schema_cs.result | 3 + mysql-test/r/mysqld--help-notwin.result | 17 + .../perfschema/r/dml_setup_instruments.result | 2 +- .../r/admission_control_by_trx_basic.result | 42 ++ .../r/admission_control_filter_basic.result | 30 ++ .../r/max_running_queries_basic.result | 130 ++++++ .../r/max_waiting_queries_basic.result | 130 ++++++ .../t/admission_control_by_trx_basic.test | 32 ++ .../t/admission_control_filter_basic.test | 23 ++ .../sys_vars/t/max_running_queries_basic.test | 154 +++++++ .../sys_vars/t/max_waiting_queries_basic.test | 154 +++++++ mysql-test/t/admission_control.test | 379 +++++++++++++++++ mysql-test/t/admission_control_hang.test | 126 ++++++ .../admission_control_multi_query-master.opt | 1 + mysql-test/t/admission_control_multi_query.py | 121 ++++++ .../t/admission_control_multi_query.test | 16 + mysql-test/t/admission_control_stress.test | 95 +++++ share/messages_to_clients.txt | 4 +- sql/CMakeLists.txt | 1 + sql/mysqld.cc | 43 ++ sql/mysqld.h | 1 + sql/sql_admission_control.cc | 389 ++++++++++++++++++ sql/sql_admission_control.h | 366 ++++++++++++++++ sql/sql_class.cc | 9 + sql/sql_class.h | 6 + sql/sql_db.cc | 18 +- sql/sql_parse.cc | 24 +- sql/sys_vars.cc | 58 +++ 33 files changed, 2595 insertions(+), 5 deletions(-) create mode 100644 mysql-test/r/admission_control.result create mode 100644 mysql-test/r/admission_control_hang.result create mode 100644 mysql-test/r/admission_control_multi_query.result create mode 100644 mysql-test/r/admission_control_stress.result create mode 100644 mysql-test/suite/sys_vars/r/admission_control_by_trx_basic.result create mode 100644 mysql-test/suite/sys_vars/r/admission_control_filter_basic.result create mode 100644 mysql-test/suite/sys_vars/r/max_running_queries_basic.result create mode 100644 mysql-test/suite/sys_vars/r/max_waiting_queries_basic.result create mode 100644 mysql-test/suite/sys_vars/t/admission_control_by_trx_basic.test create mode 100644 mysql-test/suite/sys_vars/t/admission_control_filter_basic.test create mode 100644 mysql-test/suite/sys_vars/t/max_running_queries_basic.test create mode 100644 mysql-test/suite/sys_vars/t/max_waiting_queries_basic.test create mode 100644 mysql-test/t/admission_control.test create mode 100644 mysql-test/t/admission_control_hang.test create mode 100644 mysql-test/t/admission_control_multi_query-master.opt create mode 100644 mysql-test/t/admission_control_multi_query.py create mode 100644 mysql-test/t/admission_control_multi_query.test create mode 100644 mysql-test/t/admission_control_stress.test create mode 100644 sql/sql_admission_control.cc create mode 100644 sql/sql_admission_control.h diff --git a/mysql-test/r/admission_control.result b/mysql-test/r/admission_control.result new file mode 100644 index 000000000000..67ca29dd24c0 --- /dev/null +++ b/mysql-test/r/admission_control.result @@ -0,0 +1,117 @@ +create database test_db; +create user 'test_user'@'localhost'; +grant all on test_db.* to 'test_user'@'localhost'; +grant all on test.* to 'test_user'@'localhost'; +use test_db; +set @start_max_running_queries= @@global.max_running_queries; +set @start_max_waiting_queries= @@global.max_waiting_queries; +set @@global.max_running_queries=10; +set @@global.max_waiting_queries=5; +create table t1(a int) engine=InnoDB; +lock table t1 write; +Threads waiting for admission will have appropriate state set in processlist. +Super user is exempted from admission control checks. +select * from t1; +a +set @@global.admission_control_filter = 'USE'; +select @@global.admission_control_filter; +@@global.admission_control_filter +USE +Maximum waiting queries reached. So this would hit an error. +use test_db; +select * from t1|||| +ERROR HY000: Maximum waiting queries 5 reached for database `test_db` +Maximum waiting queries reached. So this would hit an error. +use test; +create table t1_test(aaa int); +insert into t1_test values (1); +select aaa from t1_test; +drop table t1_test; +use test_db; +select * from t1|||| +aaa +1 +ERROR HY000: Maximum waiting queries 5 reached for database `test_db` +use test_db; +select * from t1; +ERROR HY000: Maximum waiting queries 5 reached for database `test_db` +set @@global.admission_control_filter = ''; +select @@global.admission_control_filter; +@@global.admission_control_filter + +Check status variables +aborted_queries = 3 +running_queries = 10 +waiting_queries = 5 +Filled up queues on one db doesn't affect queries on other db. +use test; +set @@global.max_waiting_queries=6; +Kill a thread that is waiting for admission. +select count(*) from t1; +kill ID; +use test_db; +unlock tables; +Verify the waiting queries received wakeup signal. +select count(*) from t1; +count(*) +15 +set @save_admission_control_by_trx = @@global.admission_control_by_trx; +select @save_admission_control_by_trx; +@save_admission_control_by_trx +0 +set @@global.max_running_queries=5; +set @@global.max_waiting_queries=10; +# By default, open transaction has no effect on running queries +select count(*) from t1; +count(*) +15 +# Test: open transactions will take slots in running queries, +# and will not be blocked +set @@global.admission_control_filter = 'BEGIN,COMMIT,ROLLBACK'; +select @@global.admission_control_filter; +@@global.admission_control_filter +BEGIN,COMMIT,ROLLBACK +set @@global.admission_control_by_trx = true; +SELECT @@global.admission_control_by_trx; +@@global.admission_control_by_trx +1 +Open transaction is able to continue running queries +connection con_max_wait; +New queries will be rejected (waiting queue is full) +select * from t1; +ERROR HY000: Maximum waiting queries 10 reached for database `test_db` +New transactions will be rejected (waiting queue is full) +begin; +select * from t1; +ERROR HY000: Maximum waiting queries 10 reached for database `test_db` +aborted_queries will increase by 2 +Committing a transaction will free up the running query slots +The waiting queries will be unblocked +Check status variables +include/assert.inc [DB Admission control waiting queries should be zero] +include/assert.inc [DB Admission control running queries should be zero] +include/assert.inc [DB Admission control aborted queries should be five] +set @@global.admission_control_by_trx = @save_admission_control_by_trx; +select @@global.admission_control_by_trx; +@@global.admission_control_by_trx +0 +set @@global.admission_control_filter = ''; +select @@global.admission_control_filter; +@@global.admission_control_filter + +# End of open transaction test +reset global.max_running_queries and global.max_waiting_queries +set @@global.max_running_queries=10; +set @@global.max_waiting_queries=5; +Run parallel load and drop the database. +set @@global.max_waiting_queries=0; +Cleanup. +Verify there are no waiting threads. +select count(*) from information_schema.processlist where state='waiting for admission'; +count(*) +0 +Warnings: +Warning 1287 'INFORMATION_SCHEMA.PROCESSLIST' is deprecated and will be removed in a future release. Please use performance_schema.processlist instead +set @@global.max_running_queries=@start_max_running_queries; +set @@global.max_waiting_queries=@start_max_waiting_queries; +drop user test_user@localhost; diff --git a/mysql-test/r/admission_control_hang.result b/mysql-test/r/admission_control_hang.result new file mode 100644 index 000000000000..b41f2dc3e57a --- /dev/null +++ b/mysql-test/r/admission_control_hang.result @@ -0,0 +1,93 @@ +create database test_db; +create user test_user@localhost; +grant all on test_db.* to test_user@localhost; +set @start_max_running_queries = @@global.max_running_queries; +set @@global.max_running_queries = 4; +set @start_innodb_lock_wait_timeout = @@global.innodb_lock_wait_timeout; +set @@global.innodb_lock_wait_timeout = 10000; +set @start_admission_control_filter = @@global.admission_control_filter; +set @@global.admission_control_filter = 'COMMIT'; +create table t1 (a int) engine=innodb; +insert into t1 values(1); +begin; +update t1 set a=2 where a=1; +update t1 set a=2 where a=1; +update t1 set a=2 where a=1; +update t1 set a=2 where a=1; +update t1 set a=2 where a=1; +set @@global.admission_control_filter = 'USE'; +select @@global.admission_control_filter; +@@global.admission_control_filter +USE +use test; +use test_db; +set @@global.admission_control_filter = 'ALTER,BEGIN,COMMIT,CREATE,DELETE,DROP,INSERT,LOAD,SELECT,SET,REPLACE,TRUNCATE,UPDATE,SHOW,ROLLBACK'; +select @@global.admission_control_filter; +@@global.admission_control_filter +ALTER,BEGIN,COMMIT,CREATE,DELETE,DROP,INSERT,LOAD,SELECT,SET,REPLACE,ROLLBACK,TRUNCATE,UPDATE,SHOW +create table t2(a int) engine=innodb; +begin; +insert into t2 values(1); +update t2 set a=2 where a=1; +commit; +SHOW TABLES LIKE 't2'; +Tables_in_test_db (t2) +t2 +begin; +alter table t2 rename t3; +select * from t3; +a +2 +delete from t3; +set @val = 1; +truncate table t3; +rollback; +drop table t3; +set @save_admission_control_by_trx = @@global.admission_control_by_trx; +select @save_admission_control_by_trx; +@save_admission_control_by_trx +0 +# Turn on admission_control_by_trx +set @@global.admission_control_by_trx = true; +SELECT @@global.admission_control_by_trx; +@@global.admission_control_by_trx +1 +create table t2(a int) engine=innodb; +begin; +insert into t2 values(1); +update t2 set a=2 where a=1; +commit; +SHOW TABLES LIKE 't2'; +Tables_in_test_db (t2) +t2 +begin; +alter table t2 rename t3; +select * from t3; +a +2 +delete from t3; +set @val = 1; +truncate table t3; +rollback; +drop table t3; +set @@global.admission_control_filter = default; +select @@global.admission_control_filter; +@@global.admission_control_filter + +select count(*) from t1; +count(*) +1 +set @@global.admission_control_by_trx = @save_admission_control_by_trx; +select @@global.admission_control_by_trx; +@@global.admission_control_by_trx +0 +set @@global.admission_control_filter = 'COMMIT'; +select @@global.admission_control_filter; +@@global.admission_control_filter +COMMIT +commit; +set @@global.max_running_queries = @start_max_running_queries; +set @@global.innodb_lock_wait_timeout = @start_innodb_lock_wait_timeout; +set @@global.admission_control_filter = @start_admission_control_filter; +drop database test_db; +drop user test_user@localhost; diff --git a/mysql-test/r/admission_control_multi_query.result b/mysql-test/r/admission_control_multi_query.result new file mode 100644 index 000000000000..b9f3955224d3 --- /dev/null +++ b/mysql-test/r/admission_control_multi_query.result @@ -0,0 +1,8 @@ +create database test_db; +create user test_user@localhost identified with 'mysql_native_password' BY ''; +grant all on test_db.* to test_user@localhost; +grant all on test.* to test_user@localhost; +use test_db; +create table t1 (a int primary key, b int) engine=InnoDB; +drop database test_db; +drop user test_user@localhost; diff --git a/mysql-test/r/admission_control_stress.result b/mysql-test/r/admission_control_stress.result new file mode 100644 index 000000000000..dc150f3035ed --- /dev/null +++ b/mysql-test/r/admission_control_stress.result @@ -0,0 +1,5 @@ +Test setup. +Generate load. Toggle max_running_queries and randomly kill a query. +Cleanup +set global max_connections = @start_max_connections; +set global max_running_queries = @start_max_running_queries; diff --git a/mysql-test/r/information_schema_ci.result b/mysql-test/r/information_schema_ci.result index 40d5cceaa955..0f02f55657b6 100644 --- a/mysql-test/r/information_schema_ci.result +++ b/mysql-test/r/information_schema_ci.result @@ -778,6 +778,9 @@ mysql user 0 0 show status where variable_name like "%database%"; Variable_name Value Com_show_databases 3 +Database_admission_control_aborted_queries 0 +Database_admission_control_running_queries 0 +Database_admission_control_waiting_queries 0 show variables where variable_name like "skip_show_databas"; Variable_name Value show global status like "Threads_running"; diff --git a/mysql-test/r/information_schema_cs.result b/mysql-test/r/information_schema_cs.result index 5c4829bdfea6..786b12a0876a 100644 --- a/mysql-test/r/information_schema_cs.result +++ b/mysql-test/r/information_schema_cs.result @@ -778,6 +778,9 @@ mysql user 0 0 show status where variable_name like "%database%"; Variable_name Value Com_show_databases 3 +Database_admission_control_aborted_queries 0 +Database_admission_control_running_queries 0 +Database_admission_control_waiting_queries 0 show variables where variable_name like "skip_show_databas"; Variable_name Value show global status like "Threads_running"; diff --git a/mysql-test/r/mysqld--help-notwin.result b/mysql-test/r/mysqld--help-notwin.result index 53882bc1696c..032c3522f17e 100644 --- a/mysql-test/r/mysqld--help-notwin.result +++ b/mysql-test/r/mysqld--help-notwin.result @@ -38,6 +38,13 @@ The following options may be given as the first argument: --admin-ssl-crlpath=name --admin-ssl-key=name --admin-tls-ciphersuites=name + --admission-control-by-trx + Allow open transactions to go through admission control + --admission-control-filter=name + Commands that are skipped in admission control checks. + The legal values are: ALTER, BEGIN, COMMIT, CREATE, + DELETE, DROP, INSERT, LOAD, SELECT, SET, REPLACE, + ROLLBACK, TRUNCATE, UPDATE, SHOW and empty string --allow-noncurrent-db-rw=name Switch to allow/deny reads and writes to a table not in the current database. @@ -751,6 +758,9 @@ The following options may be given as the first argument: If non-zero: relay log will be rotated automatically when the size exceeds this value; if zero: when the size exceeds max_binlog_size + --max-running-queries=# + The maximum number of running queries allowed for a + database. If this value is 0, no such limits are applied. --max-seeks-for-key=# Limit assumed max number of seeks when looking up rows based on a key @@ -762,6 +772,9 @@ The following options may be given as the first argument: --max-user-connections=# The maximum number of active connections for a single user (0 = no limit) + --max-waiting-queries=# + The maximum number of waiting queries allowed for a + database.If this value is 0, no such limits are applied. --max-write-lock-count=# After this many write locks, allow some read locks to run in between @@ -2364,6 +2377,8 @@ admin-ssl-crl (No default value) admin-ssl-crlpath (No default value) admin-ssl-key (No default value) admin-tls-ciphersuites (No default value) +admission-control-by-trx FALSE +admission-control-filter allow-noncurrent-db-rw ON allow-suspicious-udfs FALSE authentication-policy *,, @@ -2549,10 +2564,12 @@ max-nonsuper-connections 0 max-points-in-geometry 65536 max-prepared-stmt-count 16382 max-relay-log-size 0 +max-running-queries 0 max-seeks-for-key 18446744073709551615 max-sort-length 1024 max-sp-recursion-depth 0 max-user-connections 0 +max-waiting-queries 0 max-write-lock-count 18446744073709551615 maximum-hlc-drift-ns 300000000000 memlock FALSE diff --git a/mysql-test/suite/perfschema/r/dml_setup_instruments.result b/mysql-test/suite/perfschema/r/dml_setup_instruments.result index 42647ae80a2b..8791ec174c85 100644 --- a/mysql-test/suite/perfschema/r/dml_setup_instruments.result +++ b/mysql-test/suite/perfschema/r/dml_setup_instruments.result @@ -21,6 +21,7 @@ where name like 'Wait/Synch/Rwlock/sql/%' 'wait/synch/rwlock/sql/LOCK_named_pipe_full_access_group') order by name limit 10; NAME ENABLED TIMED PROPERTIES FLAGS VOLATILITY DOCUMENTATION +wait/synch/rwlock/sql/AC::rwlock YES YES singleton NULL 0 NULL wait/synch/rwlock/sql/Binlog_relay_IO_delegate::lock YES YES singleton NULL 0 NULL wait/synch/rwlock/sql/Binlog_storage_delegate::lock YES YES singleton NULL 0 NULL wait/synch/rwlock/sql/Binlog_transmit_delegate::lock YES YES singleton NULL 0 NULL @@ -30,7 +31,6 @@ wait/synch/rwlock/sql/channel_to_filter_lock YES YES NULL 0 NULL wait/synch/rwlock/sql/gtid_commit_rollback YES YES singleton NULL 0 NULL wait/synch/rwlock/sql/gtid_mode_lock YES YES singleton NULL 0 NULL wait/synch/rwlock/sql/gtid_retrieved YES YES singleton NULL 0 NULL -wait/synch/rwlock/sql/LOCK_gap_lock_exceptions YES YES singleton NULL 0 NULL select * from performance_schema.setup_instruments where name like 'Wait/Synch/Cond/sql/%' and name not in ( diff --git a/mysql-test/suite/sys_vars/r/admission_control_by_trx_basic.result b/mysql-test/suite/sys_vars/r/admission_control_by_trx_basic.result new file mode 100644 index 000000000000..fc34d1b636e1 --- /dev/null +++ b/mysql-test/suite/sys_vars/r/admission_control_by_trx_basic.result @@ -0,0 +1,42 @@ +SET @start_admission_control_by_trx = @@global.admission_control_by_trx; +SELECT @start_admission_control_by_trx; +@start_admission_control_by_trx +0 +SET @@global.admission_control_by_trx = DEFAULT; +SELECT @@global.admission_control_by_trx; +@@global.admission_control_by_trx +0 +SET @@global.admission_control_by_trx = false; +SELECT @@global.admission_control_by_trx; +@@global.admission_control_by_trx +0 +SET @@global.admission_control_by_trx = true; +SELECT @@global.admission_control_by_trx; +@@global.admission_control_by_trx +1 +SET @@global.admission_control_by_trx = 1; +SELECT @@global.admission_control_by_trx; +@@global.admission_control_by_trx +1 +SET @@global.admission_control_by_trx = 0; +SELECT @@global.admission_control_by_trx; +@@global.admission_control_by_trx +0 +SET @@global.admission_control_by_trx = -1; +ERROR 42000: Variable 'admission_control_by_trx' can't be set to the value of '-1' +SELECT @@global.admission_control_by_trx; +@@global.admission_control_by_trx +0 +SET @@global.admission_control_by_trx = 100; +ERROR 42000: Variable 'admission_control_by_trx' can't be set to the value of '100' +SELECT @@global.admission_control_by_trx; +@@global.admission_control_by_trx +0 +SET @@session.admission_control_by_trx = 10; +ERROR HY000: Variable 'admission_control_by_trx' is a GLOBAL variable and should be set with SET GLOBAL +SELECT @@session.admission_control_by_trx; +ERROR HY000: Variable 'admission_control_by_trx' is a GLOBAL variable +SET @@global.admission_control_by_trx = @start_admission_control_by_trx; +SELECT @@global.admission_control_by_trx; +@@global.admission_control_by_trx +0 diff --git a/mysql-test/suite/sys_vars/r/admission_control_filter_basic.result b/mysql-test/suite/sys_vars/r/admission_control_filter_basic.result new file mode 100644 index 000000000000..1577db033326 --- /dev/null +++ b/mysql-test/suite/sys_vars/r/admission_control_filter_basic.result @@ -0,0 +1,30 @@ +set @@global.admission_control_filter = default; +select @@global.admission_control_filter; +@@global.admission_control_filter + +set @saved_admission_control_filter = @@global.admission_control_filter; +SELECT @@global.admission_control_filter; +@@global.admission_control_filter + +SET GLOBAL ADMISSION_CONTROL_FILTER=''; +SELECT @@global.admission_control_filter; +@@global.admission_control_filter + +SET GLOBAL ADMISSION_CONTROL_FILTER=''; +SELECT @@global.admission_control_filter; +@@global.admission_control_filter + +SET GLOBAL ADMISSION_CONTROL_FILTER='COMMIT,BEGIN,ALTER'; +SELECT @@global.admission_control_filter; +@@global.admission_control_filter +ALTER,BEGIN,COMMIT +SET GLOBAL ADMISSION_CONTROL_FILTER='ALTER,BEGIN,COMMIT,CREATE,DELETE,DROP,INSERT,LOAD,SELECT,SET,REPLACE,TRUNCATE,UPDATE,SHOW'; +SELECT @@global.admission_control_filter; +@@global.admission_control_filter +ALTER,BEGIN,COMMIT,CREATE,DELETE,DROP,INSERT,LOAD,SELECT,SET,REPLACE,TRUNCATE,UPDATE,SHOW +SET GLOBAL ADMISSION_CONTROL_FILTER='TRUNCATE,REPLACE,DROP,NONEXISTING_BIT'; +ERROR 42000: Variable 'admission_control_filter' can't be set to the value of 'NONEXISTING_BIT' +SELECT @@global.admission_control_filter; +@@global.admission_control_filter +ALTER,BEGIN,COMMIT,CREATE,DELETE,DROP,INSERT,LOAD,SELECT,SET,REPLACE,TRUNCATE,UPDATE,SHOW +set global admission_control_filter = @saved_admission_control_filter; diff --git a/mysql-test/suite/sys_vars/r/max_running_queries_basic.result b/mysql-test/suite/sys_vars/r/max_running_queries_basic.result new file mode 100644 index 000000000000..fcfa84e4d7b9 --- /dev/null +++ b/mysql-test/suite/sys_vars/r/max_running_queries_basic.result @@ -0,0 +1,130 @@ +SET @start_value = @@global.max_running_queries; +SELECT @start_value; +@start_value +0 +'#--------------------FN_DYNVARS_074_01------------------------#' +SET @@global.max_running_queries = 5000; +SET @@global.max_running_queries = DEFAULT; +SELECT @@global.max_running_queries; +@@global.max_running_queries +0 +'#---------------------FN_DYNVARS_074_02-------------------------#' +SET @@global.max_running_queries = @start_value; +SELECT @@global.max_running_queries = 151; +@@global.max_running_queries = 151 +0 +'#--------------------FN_DYNVARS_074_03------------------------#' +SET @@global.max_running_queries = 100000; +SELECT @@global.max_running_queries; +@@global.max_running_queries +100000 +SET @@global.max_running_queries = 99999; +SELECT @@global.max_running_queries; +@@global.max_running_queries +99999 +SET @@global.max_running_queries = 65536; +SELECT @@global.max_running_queries; +@@global.max_running_queries +65536 +SET @@global.max_running_queries = 1; +SELECT @@global.max_running_queries; +@@global.max_running_queries +1 +SET @@global.max_running_queries = 2; +SELECT @@global.max_running_queries; +@@global.max_running_queries +2 +'#--------------------FN_DYNVARS_074_04-------------------------#' +SET @@global.max_running_queries = -1; +Warnings: +Warning 1292 Truncated incorrect max_running_queries value: '-1' +SELECT @@global.max_running_queries; +@@global.max_running_queries +0 +SET @@global.max_running_queries = 100000000000; +Warnings: +Warning 1292 Truncated incorrect max_running_queries value: '100000000000' +SELECT @@global.max_running_queries; +@@global.max_running_queries +100000 +SET @@global.max_running_queries = 10000.01; +ERROR 42000: Incorrect argument type to variable 'max_running_queries' +SELECT @@global.max_running_queries; +@@global.max_running_queries +100000 +SET @@global.max_running_queries = -1024; +Warnings: +Warning 1292 Truncated incorrect max_running_queries value: '-1024' +SELECT @@global.max_running_queries; +@@global.max_running_queries +0 +SET @@global.max_running_queries = 0; +SELECT @@global.max_running_queries; +@@global.max_running_queries +0 +SET @@global.max_running_queries = 100001; +Warnings: +Warning 1292 Truncated incorrect max_running_queries value: '100001' +SELECT @@global.max_running_queries; +@@global.max_running_queries +100000 +SET @@global.max_running_queries = ON; +ERROR 42000: Incorrect argument type to variable 'max_running_queries' +SELECT @@global.max_running_queries; +@@global.max_running_queries +100000 +SET @@global.max_running_queries = 'test'; +ERROR 42000: Incorrect argument type to variable 'max_running_queries' +SELECT @@global.max_running_queries; +@@global.max_running_queries +100000 +'#-------------------FN_DYNVARS_074_05----------------------------#' +SET @@session.max_running_queries = 4096; +ERROR HY000: Variable 'max_running_queries' is a GLOBAL variable and should be set with SET GLOBAL +SELECT @@session.max_running_queries; +ERROR HY000: Variable 'max_running_queries' is a GLOBAL variable +'#----------------------FN_DYNVARS_074_06------------------------#' +SELECT @@global.max_running_queries = VARIABLE_VALUE +FROM performance_schema.global_variables +WHERE VARIABLE_NAME='max_running_queries'; +@@global.max_running_queries = VARIABLE_VALUE +1 +SELECT @@max_running_queries = VARIABLE_VALUE +FROM performance_schema.session_variables +WHERE VARIABLE_NAME='max_running_queries'; +@@max_running_queries = VARIABLE_VALUE +1 +'#---------------------FN_DYNVARS_074_07----------------------#' +SET @@global.max_running_queries = TRUE; +SELECT @@global.max_running_queries; +@@global.max_running_queries +1 +SET @@global.max_running_queries = FALSE; +SELECT @@global.max_running_queries; +@@global.max_running_queries +0 +'#---------------------FN_DYNVARS_074_08----------------------#' +SET @@global.max_running_queries = 5000; +SELECT @@max_running_queries = @@global.max_running_queries; +@@max_running_queries = @@global.max_running_queries +1 +'#---------------------FN_DYNVARS_074_09----------------------#' +SET max_running_queries = 6000; +ERROR HY000: Variable 'max_running_queries' is a GLOBAL variable and should be set with SET GLOBAL +SELECT @@max_running_queries; +@@max_running_queries +5000 +SET local.max_running_queries = 7000; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'local.max_running_queries = 7000' at line 1 +SELECT local.max_running_queries; +ERROR 42S02: Unknown table 'local' in field list +SET global.max_running_queries = 8000; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'global.max_running_queries = 8000' at line 1 +SELECT global.max_running_queries; +ERROR 42S02: Unknown table 'global' in field list +SELECT max_running_queries = @@session.max_running_queries; +ERROR HY000: Variable 'max_running_queries' is a GLOBAL variable +SET @@global.max_running_queries = @start_value; +SELECT @@global.max_running_queries; +@@global.max_running_queries +0 diff --git a/mysql-test/suite/sys_vars/r/max_waiting_queries_basic.result b/mysql-test/suite/sys_vars/r/max_waiting_queries_basic.result new file mode 100644 index 000000000000..f86fbd8491fd --- /dev/null +++ b/mysql-test/suite/sys_vars/r/max_waiting_queries_basic.result @@ -0,0 +1,130 @@ +SET @start_value = @@global.max_waiting_queries; +SELECT @start_value; +@start_value +0 +'#--------------------FN_DYNVARS_074_01------------------------#' +SET @@global.max_waiting_queries = 5000; +SET @@global.max_waiting_queries = DEFAULT; +SELECT @@global.max_waiting_queries; +@@global.max_waiting_queries +0 +'#---------------------FN_DYNVARS_074_02-------------------------#' +SET @@global.max_waiting_queries = @start_value; +SELECT @@global.max_waiting_queries = 151; +@@global.max_waiting_queries = 151 +0 +'#--------------------FN_DYNVARS_074_03------------------------#' +SET @@global.max_waiting_queries = 100000; +SELECT @@global.max_waiting_queries; +@@global.max_waiting_queries +100000 +SET @@global.max_waiting_queries = 99999; +SELECT @@global.max_waiting_queries; +@@global.max_waiting_queries +99999 +SET @@global.max_waiting_queries = 65536; +SELECT @@global.max_waiting_queries; +@@global.max_waiting_queries +65536 +SET @@global.max_waiting_queries = 1; +SELECT @@global.max_waiting_queries; +@@global.max_waiting_queries +1 +SET @@global.max_waiting_queries = 2; +SELECT @@global.max_waiting_queries; +@@global.max_waiting_queries +2 +'#--------------------FN_DYNVARS_074_04-------------------------#' +SET @@global.max_waiting_queries = -1; +Warnings: +Warning 1292 Truncated incorrect max_waiting_queries value: '-1' +SELECT @@global.max_waiting_queries; +@@global.max_waiting_queries +0 +SET @@global.max_waiting_queries = 100000000000; +Warnings: +Warning 1292 Truncated incorrect max_waiting_queries value: '100000000000' +SELECT @@global.max_waiting_queries; +@@global.max_waiting_queries +100000 +SET @@global.max_waiting_queries = 10000.01; +ERROR 42000: Incorrect argument type to variable 'max_waiting_queries' +SELECT @@global.max_waiting_queries; +@@global.max_waiting_queries +100000 +SET @@global.max_waiting_queries = -1024; +Warnings: +Warning 1292 Truncated incorrect max_waiting_queries value: '-1024' +SELECT @@global.max_waiting_queries; +@@global.max_waiting_queries +0 +SET @@global.max_waiting_queries = 0; +SELECT @@global.max_waiting_queries; +@@global.max_waiting_queries +0 +SET @@global.max_waiting_queries = 100001; +Warnings: +Warning 1292 Truncated incorrect max_waiting_queries value: '100001' +SELECT @@global.max_waiting_queries; +@@global.max_waiting_queries +100000 +SET @@global.max_waiting_queries = ON; +ERROR 42000: Incorrect argument type to variable 'max_waiting_queries' +SELECT @@global.max_waiting_queries; +@@global.max_waiting_queries +100000 +SET @@global.max_waiting_queries = 'test'; +ERROR 42000: Incorrect argument type to variable 'max_waiting_queries' +SELECT @@global.max_waiting_queries; +@@global.max_waiting_queries +100000 +'#-------------------FN_DYNVARS_074_05----------------------------#' +SET @@session.max_waiting_queries = 4096; +ERROR HY000: Variable 'max_waiting_queries' is a GLOBAL variable and should be set with SET GLOBAL +SELECT @@session.max_waiting_queries; +ERROR HY000: Variable 'max_waiting_queries' is a GLOBAL variable +'#----------------------FN_DYNVARS_074_06------------------------#' +SELECT @@global.max_waiting_queries = VARIABLE_VALUE +FROM performance_schema.global_variables +WHERE VARIABLE_NAME='max_waiting_queries'; +@@global.max_waiting_queries = VARIABLE_VALUE +1 +SELECT @@max_waiting_queries = VARIABLE_VALUE +FROM performance_schema.session_variables +WHERE VARIABLE_NAME='max_waiting_queries'; +@@max_waiting_queries = VARIABLE_VALUE +1 +'#---------------------FN_DYNVARS_074_07----------------------#' +SET @@global.max_waiting_queries = TRUE; +SELECT @@global.max_waiting_queries; +@@global.max_waiting_queries +1 +SET @@global.max_waiting_queries = FALSE; +SELECT @@global.max_waiting_queries; +@@global.max_waiting_queries +0 +'#---------------------FN_DYNVARS_074_08----------------------#' +SET @@global.max_waiting_queries = 5000; +SELECT @@max_waiting_queries = @@global.max_waiting_queries; +@@max_waiting_queries = @@global.max_waiting_queries +1 +'#---------------------FN_DYNVARS_074_09----------------------#' +SET max_waiting_queries = 6000; +ERROR HY000: Variable 'max_waiting_queries' is a GLOBAL variable and should be set with SET GLOBAL +SELECT @@max_waiting_queries; +@@max_waiting_queries +5000 +SET local.max_waiting_queries = 7000; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'local.max_waiting_queries = 7000' at line 1 +SELECT local.max_waiting_queries; +ERROR 42S02: Unknown table 'local' in field list +SET global.max_waiting_queries = 8000; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'global.max_waiting_queries = 8000' at line 1 +SELECT global.max_waiting_queries; +ERROR 42S02: Unknown table 'global' in field list +SELECT max_waiting_queries = @@session.max_waiting_queries; +ERROR HY000: Variable 'max_waiting_queries' is a GLOBAL variable +SET @@global.max_waiting_queries = @start_value; +SELECT @@global.max_waiting_queries; +@@global.max_waiting_queries +0 diff --git a/mysql-test/suite/sys_vars/t/admission_control_by_trx_basic.test b/mysql-test/suite/sys_vars/t/admission_control_by_trx_basic.test new file mode 100644 index 000000000000..67b28950caa7 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/admission_control_by_trx_basic.test @@ -0,0 +1,32 @@ +SET @start_admission_control_by_trx = @@global.admission_control_by_trx; +SELECT @start_admission_control_by_trx; + +SET @@global.admission_control_by_trx = DEFAULT; +SELECT @@global.admission_control_by_trx; + +SET @@global.admission_control_by_trx = false; +SELECT @@global.admission_control_by_trx; + +SET @@global.admission_control_by_trx = true; +SELECT @@global.admission_control_by_trx; + +SET @@global.admission_control_by_trx = 1; +SELECT @@global.admission_control_by_trx; + +SET @@global.admission_control_by_trx = 0; +SELECT @@global.admission_control_by_trx; + +--Error ER_WRONG_VALUE_FOR_VAR +SET @@global.admission_control_by_trx = -1; +SELECT @@global.admission_control_by_trx; +--Error ER_WRONG_VALUE_FOR_VAR +SET @@global.admission_control_by_trx = 100; +SELECT @@global.admission_control_by_trx; + +--ERROR ER_GLOBAL_VARIABLE +SET @@session.admission_control_by_trx = 10; +--ERROR ER_INCORRECT_GLOBAL_LOCAL_VAR +SELECT @@session.admission_control_by_trx; + +SET @@global.admission_control_by_trx = @start_admission_control_by_trx; +SELECT @@global.admission_control_by_trx; diff --git a/mysql-test/suite/sys_vars/t/admission_control_filter_basic.test b/mysql-test/suite/sys_vars/t/admission_control_filter_basic.test new file mode 100644 index 000000000000..8bcfcf9d23cc --- /dev/null +++ b/mysql-test/suite/sys_vars/t/admission_control_filter_basic.test @@ -0,0 +1,23 @@ +set @@global.admission_control_filter = default; +select @@global.admission_control_filter; +set @saved_admission_control_filter = @@global.admission_control_filter; + +SELECT @@global.admission_control_filter; +SET GLOBAL ADMISSION_CONTROL_FILTER=''; +SELECT @@global.admission_control_filter; + +SET GLOBAL ADMISSION_CONTROL_FILTER=''; +SELECT @@global.admission_control_filter; + +SET GLOBAL ADMISSION_CONTROL_FILTER='COMMIT,BEGIN,ALTER'; +SELECT @@global.admission_control_filter; + +SET GLOBAL ADMISSION_CONTROL_FILTER='ALTER,BEGIN,COMMIT,CREATE,DELETE,DROP,INSERT,LOAD,SELECT,SET,REPLACE,TRUNCATE,UPDATE,SHOW'; +SELECT @@global.admission_control_filter; + +# checking that setting variable to a non existing value raises error +--error ER_WRONG_VALUE_FOR_VAR +SET GLOBAL ADMISSION_CONTROL_FILTER='TRUNCATE,REPLACE,DROP,NONEXISTING_BIT'; +SELECT @@global.admission_control_filter; + +set global admission_control_filter = @saved_admission_control_filter; diff --git a/mysql-test/suite/sys_vars/t/max_running_queries_basic.test b/mysql-test/suite/sys_vars/t/max_running_queries_basic.test new file mode 100644 index 000000000000..1ca2822ef294 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/max_running_queries_basic.test @@ -0,0 +1,154 @@ +--source include/load_sysvars.inc + +############################################################### +# START OF max_running_queries TESTS # +############################################################### + + +################################################################### +# Saving initial value in a temporary variable # +################################################################### + +SET @start_value = @@global.max_running_queries; +SELECT @start_value; + + +--echo '#--------------------FN_DYNVARS_074_01------------------------#' +################################################################## +# Display the DEFAULT value of max_running_queries # +################################################################## + +SET @@global.max_running_queries = 5000; +SET @@global.max_running_queries = DEFAULT; +SELECT @@global.max_running_queries; + +--echo '#---------------------FN_DYNVARS_074_02-------------------------#' +############################################### +# Verify default value of variable # +############################################### + +SET @@global.max_running_queries = @start_value; +SELECT @@global.max_running_queries = 151; + + +--echo '#--------------------FN_DYNVARS_074_03------------------------#' +################################################################## +# Change the value of max_running_queries to a valid value # +################################################################## + +SET @@global.max_running_queries = 100000; +SELECT @@global.max_running_queries; +SET @@global.max_running_queries = 99999; +SELECT @@global.max_running_queries; +SET @@global.max_running_queries = 65536; +SELECT @@global.max_running_queries; +SET @@global.max_running_queries = 1; +SELECT @@global.max_running_queries; +SET @@global.max_running_queries = 2; +SELECT @@global.max_running_queries; + + +--echo '#--------------------FN_DYNVARS_074_04-------------------------#' +##################################################################### +# Change the value of max_running_queries to invalid value # +##################################################################### + +SET @@global.max_running_queries = -1; +SELECT @@global.max_running_queries; +SET @@global.max_running_queries = 100000000000; +SELECT @@global.max_running_queries; +--Error ER_WRONG_TYPE_FOR_VAR +SET @@global.max_running_queries = 10000.01; +SELECT @@global.max_running_queries; +SET @@global.max_running_queries = -1024; +SELECT @@global.max_running_queries; +SET @@global.max_running_queries = 0; +SELECT @@global.max_running_queries; +SET @@global.max_running_queries = 100001; +SELECT @@global.max_running_queries; + +--Error ER_WRONG_TYPE_FOR_VAR +SET @@global.max_running_queries = ON; +SELECT @@global.max_running_queries; +--Error ER_WRONG_TYPE_FOR_VAR +SET @@global.max_running_queries = 'test'; +SELECT @@global.max_running_queries; + + +--echo '#-------------------FN_DYNVARS_074_05----------------------------#' +##################################################################### +# Test if accessing session max_running_queries gives error # +##################################################################### + +--Error ER_GLOBAL_VARIABLE +SET @@session.max_running_queries = 4096; +--Error ER_INCORRECT_GLOBAL_LOCAL_VAR +SELECT @@session.max_running_queries; + + +--echo '#----------------------FN_DYNVARS_074_06------------------------#' +############################################################################## +# Check if the value in GLOBAL & SESSION Tables matches values in variable # +############################################################################## + +SELECT @@global.max_running_queries = VARIABLE_VALUE +FROM performance_schema.global_variables +WHERE VARIABLE_NAME='max_running_queries'; + +SELECT @@max_running_queries = VARIABLE_VALUE +FROM performance_schema.session_variables +WHERE VARIABLE_NAME='max_running_queries'; + + +--echo '#---------------------FN_DYNVARS_074_07----------------------#' +################################################################### +# Check if TRUE and FALSE values can be used on variable # +################################################################### + +SET @@global.max_running_queries = TRUE; +SELECT @@global.max_running_queries; +SET @@global.max_running_queries = FALSE; +SELECT @@global.max_running_queries; + + +--echo '#---------------------FN_DYNVARS_074_08----------------------#' +######################################################################################################## +# Check if accessing variable with SESSION,LOCAL and without SCOPE points to same session variable # +######################################################################################################## + +SET @@global.max_running_queries = 5000; +SELECT @@max_running_queries = @@global.max_running_queries; + + +--echo '#---------------------FN_DYNVARS_074_09----------------------#' +########################################################################## +# Check if max_running_queries can be accessed with and without @@ sign # +########################################################################## + +--Error ER_GLOBAL_VARIABLE +SET max_running_queries = 6000; +SELECT @@max_running_queries; +--Error ER_PARSE_ERROR +SET local.max_running_queries = 7000; +--Error ER_UNKNOWN_TABLE +SELECT local.max_running_queries; +--Error ER_PARSE_ERROR +SET global.max_running_queries = 8000; +--Error ER_UNKNOWN_TABLE +SELECT global.max_running_queries; +--Error ER_INCORRECT_GLOBAL_LOCAL_VAR +SELECT max_running_queries = @@session.max_running_queries; + + +############################## +# Restore initial value # +############################## + +SET @@global.max_running_queries = @start_value; +SELECT @@global.max_running_queries; + + +################################################################## +# END OF max_running_queries TESTS # +################################################################## + diff --git a/mysql-test/suite/sys_vars/t/max_waiting_queries_basic.test b/mysql-test/suite/sys_vars/t/max_waiting_queries_basic.test new file mode 100644 index 000000000000..2cd2f010bf47 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/max_waiting_queries_basic.test @@ -0,0 +1,154 @@ +--source include/load_sysvars.inc + +############################################################### +# START OF max_waiting_queries TESTS # +############################################################### + + +################################################################### +# Saving initial value in a temporary variable # +################################################################### + +SET @start_value = @@global.max_waiting_queries; +SELECT @start_value; + + +--echo '#--------------------FN_DYNVARS_074_01------------------------#' +################################################################## +# Display the DEFAULT value of max_waiting_queries # +################################################################## + +SET @@global.max_waiting_queries = 5000; +SET @@global.max_waiting_queries = DEFAULT; +SELECT @@global.max_waiting_queries; + +--echo '#---------------------FN_DYNVARS_074_02-------------------------#' +############################################### +# Verify default value of variable # +############################################### + +SET @@global.max_waiting_queries = @start_value; +SELECT @@global.max_waiting_queries = 151; + + +--echo '#--------------------FN_DYNVARS_074_03------------------------#' +################################################################## +# Change the value of max_waiting_queries to a valid value # +################################################################## + +SET @@global.max_waiting_queries = 100000; +SELECT @@global.max_waiting_queries; +SET @@global.max_waiting_queries = 99999; +SELECT @@global.max_waiting_queries; +SET @@global.max_waiting_queries = 65536; +SELECT @@global.max_waiting_queries; +SET @@global.max_waiting_queries = 1; +SELECT @@global.max_waiting_queries; +SET @@global.max_waiting_queries = 2; +SELECT @@global.max_waiting_queries; + + +--echo '#--------------------FN_DYNVARS_074_04-------------------------#' +##################################################################### +# Change the value of max_waiting_queries to invalid value # +##################################################################### + +SET @@global.max_waiting_queries = -1; +SELECT @@global.max_waiting_queries; +SET @@global.max_waiting_queries = 100000000000; +SELECT @@global.max_waiting_queries; +--Error ER_WRONG_TYPE_FOR_VAR +SET @@global.max_waiting_queries = 10000.01; +SELECT @@global.max_waiting_queries; +SET @@global.max_waiting_queries = -1024; +SELECT @@global.max_waiting_queries; +SET @@global.max_waiting_queries = 0; +SELECT @@global.max_waiting_queries; +SET @@global.max_waiting_queries = 100001; +SELECT @@global.max_waiting_queries; + +--Error ER_WRONG_TYPE_FOR_VAR +SET @@global.max_waiting_queries = ON; +SELECT @@global.max_waiting_queries; +--Error ER_WRONG_TYPE_FOR_VAR +SET @@global.max_waiting_queries = 'test'; +SELECT @@global.max_waiting_queries; + + +--echo '#-------------------FN_DYNVARS_074_05----------------------------#' +##################################################################### +# Test if accessing session max_waiting_queries gives error # +##################################################################### + +--Error ER_GLOBAL_VARIABLE +SET @@session.max_waiting_queries = 4096; +--Error ER_INCORRECT_GLOBAL_LOCAL_VAR +SELECT @@session.max_waiting_queries; + + +--echo '#----------------------FN_DYNVARS_074_06------------------------#' +############################################################################## +# Check if the value in GLOBAL & SESSION Tables matches values in variable # +############################################################################## + +SELECT @@global.max_waiting_queries = VARIABLE_VALUE +FROM performance_schema.global_variables +WHERE VARIABLE_NAME='max_waiting_queries'; + +SELECT @@max_waiting_queries = VARIABLE_VALUE +FROM performance_schema.session_variables +WHERE VARIABLE_NAME='max_waiting_queries'; + + +--echo '#---------------------FN_DYNVARS_074_07----------------------#' +################################################################### +# Check if TRUE and FALSE values can be used on variable # +################################################################### + +SET @@global.max_waiting_queries = TRUE; +SELECT @@global.max_waiting_queries; +SET @@global.max_waiting_queries = FALSE; +SELECT @@global.max_waiting_queries; + + +--echo '#---------------------FN_DYNVARS_074_08----------------------#' +######################################################################################################## +# Check if accessing variable with SESSION,LOCAL and without SCOPE points to same session variable # +######################################################################################################## + +SET @@global.max_waiting_queries = 5000; +SELECT @@max_waiting_queries = @@global.max_waiting_queries; + + +--echo '#---------------------FN_DYNVARS_074_09----------------------#' +########################################################################## +# Check if max_waiting_queries can be accessed with and without @@ sign # +########################################################################## + +--Error ER_GLOBAL_VARIABLE +SET max_waiting_queries = 6000; +SELECT @@max_waiting_queries; +--Error ER_PARSE_ERROR +SET local.max_waiting_queries = 7000; +--Error ER_UNKNOWN_TABLE +SELECT local.max_waiting_queries; +--Error ER_PARSE_ERROR +SET global.max_waiting_queries = 8000; +--Error ER_UNKNOWN_TABLE +SELECT global.max_waiting_queries; +--Error ER_INCORRECT_GLOBAL_LOCAL_VAR +SELECT max_waiting_queries = @@session.max_waiting_queries; + + +############################## +# Restore initial value # +############################## + +SET @@global.max_waiting_queries = @start_value; +SELECT @@global.max_waiting_queries; + + +################################################################## +# END OF max_waiting_queries TESTS # +################################################################## + diff --git a/mysql-test/t/admission_control.test b/mysql-test/t/admission_control.test new file mode 100644 index 000000000000..4022f84cdee7 --- /dev/null +++ b/mysql-test/t/admission_control.test @@ -0,0 +1,379 @@ +--source include/count_sessions.inc +create database test_db; +create user 'test_user'@'localhost'; +grant all on test_db.* to 'test_user'@'localhost'; +grant all on test.* to 'test_user'@'localhost'; + +let $start_aborted_queries = query_get_value(show global status like "Database_admission_control_aborted_queries", Value, 1); +let $start_running_queries = query_get_value(show global status like "Database_admission_control_running_queries", Value, 1); +let $start_waiting_queries = query_get_value(show global status like "Database_admission_control_waiting_queries", Value, 1); + +connection default; +use test_db; +set @start_max_running_queries= @@global.max_running_queries; +set @start_max_waiting_queries= @@global.max_waiting_queries; +set @@global.max_running_queries=10; +set @@global.max_waiting_queries=5; +create table t1(a int) engine=InnoDB; + +# Write lock here ensures the insert queries will block on row lock and be +# in running queue until unlock tables is executed. +lock table t1 write; + +disable_query_log; +disable_result_log; +let $i= 15; +while ($i) +{ + connect (con$i, localhost, test_user,,test_db); + dec $i; +} +let $i= 10; +while($i) +{ + connection con$i; + # These queries will be in running queue + send insert into t1 values(1); + dec $i; +} +let $i= 15; +while ($i != 10) +{ + # These queries will be in waiting queue + connection con$i; + send insert into t1 values(1); + dec $i; +} +enable_query_log; +enable_result_log; + +connection default; +--echo Threads waiting for admission will have appropriate state set in processlist. +let $wait_condition= + select count(*)=5 from information_schema.processlist where state='waiting for admission'; +source include/wait_condition.inc; +--echo Super user is exempted from admission control checks. +select * from t1; + +# set filter to bypass USE statement +--let $saved_admission_control_filter = `SELECT @@global.admission_control_filter` +set @@global.admission_control_filter = 'USE'; +select @@global.admission_control_filter; + +# Test failure case +# Default db is intentionally not set when creating the connection +connect (con_max_wait, localhost, test_user,,); +connection con_max_wait; + +# Test multi-statement packet with use statement. +delimiter ||||; +--echo Maximum waiting queries reached. So this would hit an error. +--error ER_DB_ADMISSION_CONTROL +# switch the session's default db to test_db, which (the use statement) should +# be bypassed in admission control. The select statement will fail. +use test_db; +select * from t1|||| + +# Query will only fail on test_db, but not on test +--echo Maximum waiting queries reached. So this would hit an error. +--error ER_DB_ADMISSION_CONTROL +use test; +create table t1_test(aaa int); +insert into t1_test values (1); +select aaa from t1_test; +drop table t1_test; +use test_db; +select * from t1|||| + +delimiter ;|||| + +# USE is ok +use test_db; + +# Single statement +--error ER_DB_ADMISSION_CONTROL +select * from t1; + +disconnect con_max_wait; + +connection default; +# reset admission_control_filter +--eval set @@global.admission_control_filter = '$saved_admission_control_filter' +select @@global.admission_control_filter; + +--echo Check status variables +let $aborted_queries = query_get_value(show global status like "Database_admission_control_aborted_queries", Value, 1); +let $running_queries = query_get_value(show global status like "Database_admission_control_running_queries", Value, 1); +let $waiting_queries = query_get_value(show global status like "Database_admission_control_waiting_queries", Value, 1); +let $aborted_queries = `select $aborted_queries - $start_aborted_queries`; +let $running_queries = `select $running_queries - $start_running_queries`; +let $waiting_queries = `select $waiting_queries - $start_waiting_queries`; +--echo aborted_queries = $aborted_queries +--echo running_queries = $running_queries +--echo waiting_queries = $waiting_queries + +--echo Filled up queues on one db doesn't affect queries on other db. +connect (con_test, localhost, test_user,,test); +connection con_test; +use test; +disconnect con_test; + +connection default; +set @@global.max_waiting_queries=6; + +--echo Kill a thread that is waiting for admission. +connect (killed_connection, localhost, test_user,,test_db); +connection killed_connection; +send select count(*) from t1; +connection default; +# Need to first wait for the state in processlist, otherwise we run the risk +# of a race condition and test will fail because $id would be empty. +let $wait_timeout= 300; +let $wait_condition= + select count(*) = 1 from information_schema.processlist where state='waiting for admission' and info='select count(*) from t1'; +source include/wait_condition.inc; +let $id= + `select id from information_schema.processlist where state='waiting for admission' and info='select count(*) from t1'`; +--replace_result $id ID +eval kill $id; +disconnect killed_connection; + +# Insert queries sent above will be unblocked by this. +use test_db; +unlock tables; + +disable_query_log; +disable_result_log; +let $i= 15; +while ($i) +{ + connection con$i; + reap; + dec $i; +} +enable_query_log; +enable_result_log; +connection default; +--echo Verify the waiting queries received wakeup signal. +select count(*) from t1; + +set @save_admission_control_by_trx = @@global.admission_control_by_trx; +select @save_admission_control_by_trx; +set @@global.max_running_queries=5; +set @@global.max_waiting_queries=10; +--echo # By default, open transaction has no effect on running queries +disable_query_log; +disable_result_log; +let $i= 5; # max_running_queries is 5 +while($i) +{ + connection con$i; + # 5 Open transactions + begin; + select count(*) from t1; + dec $i; +} +enable_query_log; +enable_result_log; +# query will succeed +connection con11; +select count(*) from t1; + +--echo # Test: open transactions will take slots in running queries, +--echo # and will not be blocked +connection default; +--let $saved_admission_control_filter = `SELECT @@global.admission_control_filter` +set @@global.admission_control_filter = 'BEGIN,COMMIT,ROLLBACK'; +select @@global.admission_control_filter; +set @@global.admission_control_by_trx = true; +SELECT @@global.admission_control_by_trx; +disable_query_log; +disable_result_log; +let $i= 5; # max_running_queries is 5 +while($i) +{ + connection con$i; + # Open transactions will take up slots in running queue + begin; + select count(*) from t1; + dec $i; +} +let $i= 15; # max_waiting_queries is 10 +while ($i != 5) +{ + # These queries will be in waiting queue + connection con$i; + send select count(*) from t1; + dec $i; +} +enable_query_log; +enable_result_log; + +connection default; +let $wait_timeout= 300; +let $wait_condition= + select count(*)=10 from information_schema.processlist where state='waiting for admission'; +source include/wait_condition.inc; + +--echo Open transaction is able to continue running queries +disable_query_log; +disable_result_log; +let $i= 5; +while($i) +{ + connection con$i; + select count(*) from t1; + dec $i; +} +enable_query_log; +enable_result_log; + +connect (con_max_wait, localhost, test_user,,test_db); +--echo connection con_max_wait; +connection con_max_wait; +--echo New queries will be rejected (waiting queue is full) +error ER_DB_ADMISSION_CONTROL; +select * from t1; +--echo New transactions will be rejected (waiting queue is full) +begin; +error ER_DB_ADMISSION_CONTROL; +select * from t1; +disconnect con_max_wait; +--echo aborted_queries will increase by 2 + +--echo Committing a transaction will free up the running query slots +disable_query_log; +let $i= 5; +# commit con4 and con5 +while($i > 3) +{ + connection con$i; + commit; + dec $i; +} +# rollback con2 explicitly +connection con2; +rollback; +# close con1 to rollback +connection con1; +disconnect con1; +connection con3; +# get connection id of con3 +let $con3_id = `select connection_id()`; +# kill con3 to rollback +connection default; +eval kill $con3_id; +enable_query_log; + +--echo The waiting queries will be unblocked +disable_query_log; +disable_result_log; +let $i= 15; +while ($i != 5) +{ + connection con$i; + reap; + dec $i; +} +enable_query_log; +enable_result_log; + +connection default; +let $wait_timeout= 300; + +# Total 15-2 = 13 test_user connections (con1 & con3 were closed) +# wait for con1 and con3 to be killed or disconnected +let $wait_condition= + select count(*)=13 from information_schema.processlist where user='test_user'; +source include/wait_condition.inc; + +# wait for all test_user connections becoming idle. +let $wait_condition= + select count(*)=13 from information_schema.processlist where command='Sleep' and state='' and user='test_user'; +source include/wait_condition.inc; + +--echo Check status variables +let $aborted_queries = query_get_value(show global status like "Database_admission_control_aborted_queries", Value, 1); +let $running_queries = query_get_value(show global status like "Database_admission_control_running_queries", Value, 1); +let $waiting_queries = query_get_value(show global status like "Database_admission_control_waiting_queries", Value, 1); +let $aborted_queries = `select $aborted_queries - $start_aborted_queries`; +let $running_queries = `select $running_queries - $start_running_queries`; +let $waiting_queries = `select $waiting_queries - $start_waiting_queries`; + +--let $assert_text= DB Admission control waiting queries should be zero +--let $assert_cond= $waiting_queries = 0 +--source include/assert.inc + +--let $assert_text= DB Admission control running queries should be zero +--let $assert_cond= $running_queries = 0 +--source include/assert.inc + +--let $assert_text= DB Admission control aborted queries should be five +--let $assert_cond= $aborted_queries = 5 +--source include/assert.inc + + +# restore admission_control_by_trx +connection default; +set @@global.admission_control_by_trx = @save_admission_control_by_trx; +select @@global.admission_control_by_trx; +# reset admission_control_filter +--eval set @@global.admission_control_filter = '$saved_admission_control_filter' +select @@global.admission_control_filter; +# recreate con1 and con3 +disconnect con3; +connect (con1, localhost, test_user,,test_db); +connect (con3, localhost, test_user,,test_db); +--echo # End of open transaction test + +connection default; +--echo reset global.max_running_queries and global.max_waiting_queries +set @@global.max_running_queries=10; +set @@global.max_waiting_queries=5; +--echo Run parallel load and drop the database. +# Verify the waiting queries will receive the signal from DROP DATABASE +# and exit with appropriate error ER_NO_SUCH_TABLE. max_waiting_queries=0 +# is used to not have a limit on number of waiting queries. +set @@global.max_waiting_queries=0; # No limit on waiting queue. +disable_query_log; +disable_result_log; +let $i= 15; +while($i) +{ + connection con$i; + # 10 queries (max_running_queries is 10) will be in running queue. + # Using sleep(10) in the insert ensures some queries end up in waiting queue. + send insert into t1(select sleep(10)); + dec $i; +} +connection default; +drop database test_db; +let $i= 15; +while($i) +{ + connection con$i; + error 0,ER_BAD_DB_ERROR,ER_LOCK_DEADLOCK; + reap; + dec $i; +} +enable_query_log; +enable_result_log; + +--echo Cleanup. +connection default; +--echo Verify there are no waiting threads. +select count(*) from information_schema.processlist where state='waiting for admission'; +set @@global.max_running_queries=@start_max_running_queries; +set @@global.max_waiting_queries=@start_max_waiting_queries; +drop user test_user@localhost; +disable_query_log; +disable_result_log; +let $i= 15; +while ($i) +{ + disconnect con$i; + dec $i; +} +enable_query_log; +enable_result_log; +--source include/wait_until_count_sessions.inc diff --git a/mysql-test/t/admission_control_hang.test b/mysql-test/t/admission_control_hang.test new file mode 100644 index 000000000000..c6a780fbf99e --- /dev/null +++ b/mysql-test/t/admission_control_hang.test @@ -0,0 +1,126 @@ +--source include/count_sessions.inc +create database test_db; +create user test_user@localhost; +grant all on test_db.* to test_user@localhost; + +set @start_max_running_queries = @@global.max_running_queries; +set @@global.max_running_queries = 4; +set @start_innodb_lock_wait_timeout = @@global.innodb_lock_wait_timeout; +set @@global.innodb_lock_wait_timeout = 10000; +set @start_admission_control_filter = @@global.admission_control_filter; +set @@global.admission_control_filter = 'COMMIT'; + +let $i= 6; +while ($i) +{ + connect (con$i, localhost, test_user,,test_db); + dec $i; +} + +connection con5; +create table t1 (a int) engine=innodb; +insert into t1 values(1); +begin; +update t1 set a=2 where a=1; + +connection default; +let $i= 4; +# Fill up the admission control running limit. +while($i) +{ + connection con$i; + # Blocked on innodb row lock + send update t1 set a=2 where a=1; + dec $i; +} + +# Test filter USE statement +connection default; +set @@global.admission_control_filter = 'USE'; +select @@global.admission_control_filter; + +connection con6; +use test; +use test_db; + +connection default; +set @@global.admission_control_filter = 'ALTER,BEGIN,COMMIT,CREATE,DELETE,DROP,INSERT,LOAD,SELECT,SET,REPLACE,TRUNCATE,UPDATE,SHOW,ROLLBACK'; +select @@global.admission_control_filter; + +# Verify the commands filtered above run fine. +connection con6; +create table t2(a int) engine=innodb; +begin; +insert into t2 values(1); +update t2 set a=2 where a=1; +commit; +SHOW TABLES LIKE 't2'; +begin; +alter table t2 rename t3; +select * from t3; +delete from t3; +set @val = 1; +truncate table t3; +rollback; +drop table t3; + +# Verify that admission_control_by_trx will honor the filters +connection default; +set @save_admission_control_by_trx = @@global.admission_control_by_trx; +select @save_admission_control_by_trx; +--echo # Turn on admission_control_by_trx +set @@global.admission_control_by_trx = true; +SELECT @@global.admission_control_by_trx; + +connection con6; +create table t2(a int) engine=innodb; +begin; +insert into t2 values(1); +update t2 set a=2 where a=1; +commit; +SHOW TABLES LIKE 't2'; +begin; +alter table t2 rename t3; +select * from t3; +delete from t3; +set @val = 1; +truncate table t3; +rollback; +drop table t3; + +connection default; +set @@global.admission_control_filter = default; +select @@global.admission_control_filter; + +# open transaction will be able to go through +connection con5; +select count(*) from t1; + +# restore admission_control_by_trx +connection default; +set @@global.admission_control_by_trx = @save_admission_control_by_trx; +select @@global.admission_control_by_trx; + +connection default; +set @@global.admission_control_filter = 'COMMIT'; +select @@global.admission_control_filter; + +connection con5; +# Without the COMMIT filter set above, this query gets blocked until conflicting +# queries hit timeout. +commit; + +let $i= 6; +while ($i) +{ + disconnect con$i; + dec $i; +} + +connection default; +set @@global.max_running_queries = @start_max_running_queries; +set @@global.innodb_lock_wait_timeout = @start_innodb_lock_wait_timeout; +set @@global.admission_control_filter = @start_admission_control_filter; +drop database test_db; +drop user test_user@localhost; +--source include/wait_until_count_sessions.inc diff --git a/mysql-test/t/admission_control_multi_query-master.opt b/mysql-test/t/admission_control_multi_query-master.opt new file mode 100644 index 000000000000..efe5ff0cc88c --- /dev/null +++ b/mysql-test/t/admission_control_multi_query-master.opt @@ -0,0 +1 @@ +--innodb_lock_wait_timeout=4 --innodb_deadlock_detect=0 --max_running_queries=5 --max_waiting_queries=2000 diff --git a/mysql-test/t/admission_control_multi_query.py b/mysql-test/t/admission_control_multi_query.py new file mode 100644 index 000000000000..f980d6220388 --- /dev/null +++ b/mysql-test/t/admission_control_multi_query.py @@ -0,0 +1,121 @@ +import time +import sys +import MySQLdb +import argparse +import random +import threading +import traceback + +NUM_WORKERS = 50 +NUM_TRANSACTIONS = 2500 + +def parse_args(): + parser = argparse.ArgumentParser( + 'multi_query', + description='Generate Multi query load', + ) + parser.add_argument( + '--host', + type=str, + help='host name') + parser.add_argument( + '--port', + type=int, + help='port number') + parser.add_argument( + '--user', + type=str, + help='user name') + parser.add_argument( + '--password', + default='', + type=str, + help='password') + parser.add_argument( + '--database', + type=str, + help='database to use') + + return parser.parse_args() + +def generate_load(args): + con = MySQLdb.connect(user=args.user, + passwd=args.password, + host=args.host, + port=args.port, + db=args.database) + for i in range(NUM_TRANSACTIONS): + cursor = con.cursor() + cursor.execute('begin;') + sql = '' + values = [] + for i in range(3): + val = random.randrange(1, 50) + values.append(val) + values = sorted(values) + for val in values: + sql += 'insert into t1 values(%d, 1) on duplicate key ' \ + 'update b=greatest(b+1, 0);' % (val) + sql += 'commit;' + cursor.execute(sql) + cursor.close() + con.close() + +def run_admin_checks(args): + con = MySQLdb.connect(user=args.user, + passwd=args.password, + host=args.host, + port=args.port, + db=args.database) + cursor=con.cursor() + cursor.execute("select @@global.max_running_queries") + rows = cursor.fetchone() + max_running_queries = int(rows[0]) + for i in range(NUM_TRANSACTIONS): + cursor=con.cursor() + cursor.execute("show status like '%admission%'") + rows = cursor.fetchall() + if int(rows[1][1]) > max_running_queries: + raise Exception('Current running queries %s is more than ' \ + 'max_running_queries %d' % (rows[1][1], + max_running_queries)) + +class worker_thread(threading.Thread): + def __init__(self, args, admin): + threading.Thread.__init__(self) + self.args = args + self.exception = None + self.admin = admin + self.start() + + def run(self): + try: + if self.admin: + run_admin_checks(self.args) + else: + generate_load(self.args) + except Exception as e: + self.exception = traceback.format_exc() + +def main(): + args = parse_args() + workers = [] + for i in range(NUM_WORKERS): + worker = worker_thread(args, False) + workers.append(worker) + + admin_worker = worker_thread(args, True) + workers.append(admin_worker) + + worker_failed = False + for w in workers: + w.join() + if w.exception: + print("worker hit an exception:\n%s\n'" % w.exception) + worker_failed = True + + if worker_failed: + sys.exit(1) + +if __name__ == '__main__': + main() diff --git a/mysql-test/t/admission_control_multi_query.test b/mysql-test/t/admission_control_multi_query.test new file mode 100644 index 000000000000..c23a4ae5dd6f --- /dev/null +++ b/mysql-test/t/admission_control_multi_query.test @@ -0,0 +1,16 @@ +source include/big_test.inc; +source include/not_valgrind.inc; +create database test_db; +create user test_user@localhost identified with 'mysql_native_password' BY ''; +grant all on test_db.* to test_user@localhost; +grant all on test.* to test_user@localhost; + +use test_db; +create table t1 (a int primary key, b int) engine=InnoDB; + +let $exec = python $MYSQL_TEST_DIR/t/admission_control_multi_query.py --user='test_user' --host=127.0.0.1 --port=$MASTER_MYPORT --database='test_db'; +exec $exec > $MYSQLTEST_VARDIR/tmp/admission_control_multi_query.output; + +drop database test_db; +drop user test_user@localhost; +--remove_file $MYSQLTEST_VARDIR/tmp/admission_control_multi_query.output diff --git a/mysql-test/t/admission_control_stress.test b/mysql-test/t/admission_control_stress.test new file mode 100644 index 000000000000..d6e58647d21b --- /dev/null +++ b/mysql-test/t/admission_control_stress.test @@ -0,0 +1,95 @@ +source include/big_test.inc; +# valgrind is pretty slow with lot of threads. +source include/not_valgrind.inc; +source include/count_sessions.inc; + + +--echo Test setup. + +let $num_databases = 2; +let $num_connections = 100; +let $num_connection_per_db = 50; +let $num_iterations = 4000; + +--disable_query_log +--disable_result_log + +set @start_max_connections = @@global.max_connections; +eval set @@global.max_connections = $num_connections + 10; +set @start_max_running_queries = @@global.max_running_queries; +set global max_running_queries = 2; + +# Create databases and tables. +connection default; +let $i = $num_databases; +while ($i) { + eval create database test_db$i; + eval use test_db$i; + eval create table test_table (a int auto_increment primary key, b text) engine=InnoDB; + dec $i; +} + +# Create test user and connection pool. +create user test_user@localhost; +grant all on *.* to test_user@localhost; +let $i = $num_connections; +while ($i) { + let $db = `select ceiling($i/$num_connection_per_db)`; + connect (con$i, localhost, test_user,,test_db$db); + dec $i; +} + +--echo Generate load. Toggle max_running_queries and randomly kill a query. +let $j = $num_iterations; +while ($j) { + let $i = $num_connections; + while ($i) { + connection con$i; + send insert into test_table values (NULL, uuid()); + dec $i; + } + + connection default; + let $i = 20; + while ($i) { + let $val = `select 5*floor(10*rand())`; + eval set global max_running_queries = $val; + let $id = `select id from information_schema.processlist where user='test_user' order by rand() limit 1;`; + eval kill query $id; + dec $i; + } + + let $i = $num_connections; + while ($i) { + connection con$i; + error 0, ER_QUERY_INTERRUPTED; # Error may be caused by kill query. + reap; + dec $i; + } + + dec $j; +} + +--echo Cleanup +connection default; +let $i = $num_databases; +while ($i) { + eval select count(*) from test_db$i.test_table; + eval drop database if exists test_db$i; + dec $i; +} + +drop user test_user@localhost; + +let $i = $num_connections; +while ($i) { + disconnect con$i; + dec $i; +} + +--enable_result_log +--enable_query_log + +set global max_connections = @start_max_connections; +set global max_running_queries = @start_max_running_queries; +--source include/wait_until_count_sessions.inc diff --git a/share/messages_to_clients.txt b/share/messages_to_clients.txt index 7df835f6f2f8..b0a6046d992e 100644 --- a/share/messages_to_clients.txt +++ b/share/messages_to_clients.txt @@ -10431,8 +10431,8 @@ ER_PLACEHOLDER_50026 ER_DB_READ_ONLY eng "Database '%s' is in read-only mode. %s" -ER_PLACEHOLDER_50028 - eng "Placeholder" +ER_DB_ADMISSION_CONTROL + eng "Maximum waiting queries %lu reached for database `%s`" ER_PLACEHOLDER_50029 eng "Placeholder" diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 089fab1d93e4..6b562c76f22e 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -523,6 +523,7 @@ SET(SQL_SHARED_SOURCES spatial.cc string_service.cc sql_admin.cc + sql_admission_control.cc sql_alloc_error_handler.cc sql_alter.cc sql_alter_instance.cc diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 0d7fb2dc7687..51832638fd27 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -984,6 +984,7 @@ MySQL clients support the protocol: #include "sql/server_component/mysql_server_keyring_lockable_imp.h" #include "sql/server_component/mysql_thd_store_imp.h" #include "sql/server_component/persistent_dynamic_loader_imp.h" +#include "sql/sql_admission_control.h" #include "sql/srv_session.h" #ifdef HAVE_JEMALLOC @@ -1444,6 +1445,7 @@ ulong binlog_stmt_cache_use = 0, binlog_stmt_cache_disk_use = 0; ulong max_connections, max_connect_errors; ulong max_nonsuper_connections = 0; std::atomic nonsuper_connections(0); +ulong opt_max_running_queries = 0, opt_max_waiting_queries = 0; ulong rpl_stop_replica_timeout = LONG_TIMEOUT; bool thread_cache_size_specified = false; bool host_cache_size_specified = false; @@ -2826,6 +2828,7 @@ static void clean_up(bool print_message) { ha_pre_dd_shutdown(); dd::shutdown(); + delete db_ac; Events::deinit(); stop_handle_manager(); @@ -9930,6 +9933,10 @@ int mysqld_main(int argc, char **argv) /* Determine default TCP port and unix socket name */ set_ports(); + db_ac = new AC(); + db_ac->update_max_running_queries(opt_max_running_queries); + db_ac->update_max_waiting_queries(opt_max_waiting_queries); + if (init_server_components()) unireg_abort(MYSQLD_ABORT_EXIT); if (!server_id_supplied) @@ -11636,6 +11643,33 @@ static int show_connection_errors_net_ER_NET_WRITE_INTERRUPTED(THD *, return 0; } +static int get_db_ac_total_aborted_queries(THD *thd MY_ATTRIBUTE((unused)), + SHOW_VAR *var, char *buff) { + var->type = SHOW_LONGLONG; + var->value = buff; + ulonglong *value = reinterpret_cast(buff); + *value = db_ac->get_total_aborted_queries(); + return 0; +} + +static int get_db_ac_total_running_queries(THD *thd MY_ATTRIBUTE((unused)), + SHOW_VAR *var, char *buff) { + var->type = SHOW_LONG; + var->value = buff; + long *value = reinterpret_cast(buff); + *value = db_ac->get_total_running_queries(); + return 0; +} + +static int get_db_ac_total_waiting_queries(THD *thd MY_ATTRIBUTE((unused)), + SHOW_VAR *var, char *buff) { + var->type = SHOW_LONG; + var->value = buff; + long *value = reinterpret_cast(buff); + *value = db_ac->get_total_waiting_queries(); + return 0; +} + #ifdef ENABLED_PROFILING static int show_flushstatustime(THD *thd, SHOW_VAR *var, char *buff) { var->type = SHOW_LONGLONG; @@ -11995,6 +12029,12 @@ SHOW_VAR status_vars[] = { {"Created_tmp_tables", (char *)offsetof(System_status_var, created_tmp_tables), SHOW_LONGLONG_STATUS, SHOW_SCOPE_ALL}, + {"Database_admission_control_aborted_queries", + (char *)&get_db_ac_total_aborted_queries, SHOW_FUNC, SHOW_SCOPE_GLOBAL}, + {"Database_admission_control_running_queries", + (char *)&get_db_ac_total_running_queries, SHOW_FUNC, SHOW_SCOPE_GLOBAL}, + {"Database_admission_control_waiting_queries", + (char *)&get_db_ac_total_waiting_queries, SHOW_FUNC, SHOW_SCOPE_GLOBAL}, {"Delayed_errors", (char *)&delayed_insert_errors, SHOW_LONG, SHOW_SCOPE_GLOBAL}, {"Delayed_insert_threads", (char *)&delayed_insert_threads, @@ -14569,6 +14609,8 @@ extern PSI_stage_info stage_waiting_for_disk_space; #ifdef HAVE_PSI_INTERFACE PSI_stage_info *all_server_stages[] = { + &stage_admission_control_enter, + &stage_admission_control_exit, &stage_after_create, &stage_alter_inplace_prepare, &stage_alter_inplace, @@ -14642,6 +14684,7 @@ PSI_stage_info *all_server_stages[] = { &stage_updating_reference_tables, &stage_user_sleep, &stage_verifying_table, + &stage_waiting_for_admission, &stage_waiting_for_gtid_to_be_committed, &stage_waiting_for_handler_commit, &stage_waiting_for_source_to_send_event, diff --git a/sql/mysqld.h b/sql/mysqld.h index fe7fb763e1c8..b1b76323251f 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -329,6 +329,7 @@ extern ulong stored_program_def_size; extern ulong table_def_size; extern ulong tablespace_def_size; extern MYSQL_PLUGIN_IMPORT ulong max_connections; +extern ulong opt_max_running_queries, opt_max_waiting_queries; extern ulong max_digest_length; extern ulong max_connect_errors, connect_timeout; extern ulong max_nonsuper_connections; diff --git a/sql/sql_admission_control.cc b/sql/sql_admission_control.cc new file mode 100644 index 000000000000..ce72df7f08f9 --- /dev/null +++ b/sql/sql_admission_control.cc @@ -0,0 +1,389 @@ +/* Copyright (c) 2016, Facebook. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "sql/sql_admission_control.h" +#include "sql/auth/auth_acls.h" +#include "sql/sql_lex.h" + +bool opt_admission_control_by_trx = false; +ulonglong admission_control_filter; +AC *db_ac; +#ifdef HAVE_PSI_INTERFACE +PSI_stage_info stage_admission_control_enter = {0, "Admission control enter", 0, + PSI_DOCUMENT_ME}; +PSI_stage_info stage_admission_control_exit = {0, "Admission control exit", 0, + PSI_DOCUMENT_ME}; +PSI_stage_info stage_waiting_for_admission = {0, "waiting for admission", 0, + PSI_DOCUMENT_ME}; +#endif + +/* + @param sql_command command the thread is currently executing + + @return true Skips the current query in admission control + false Admission control checks are applied for this query +*/ +static bool filter_command(enum_sql_command sql_command) { + switch (sql_command) { + case SQLCOM_ALTER_TABLE: + case SQLCOM_ALTER_DB: + case SQLCOM_ALTER_PROCEDURE: + case SQLCOM_ALTER_FUNCTION: + case SQLCOM_ALTER_TABLESPACE: + case SQLCOM_ALTER_SERVER: + case SQLCOM_ALTER_EVENT: + case SQLCOM_ALTER_USER: + case SQLCOM_RENAME_TABLE: + case SQLCOM_RENAME_USER: + case SQLCOM_ALTER_INSTANCE: + case SQLCOM_ALTER_USER_DEFAULT_ROLE: + case SQLCOM_ALTER_RESOURCE_GROUP: + return IS_BIT_SET(admission_control_filter, ADMISSION_CONTROL_ALTER); + + case SQLCOM_BEGIN: + return IS_BIT_SET(admission_control_filter, ADMISSION_CONTROL_BEGIN); + + case SQLCOM_COMMIT: + return IS_BIT_SET(admission_control_filter, ADMISSION_CONTROL_COMMIT); + + case SQLCOM_CREATE_TABLE: + case SQLCOM_CREATE_INDEX: + case SQLCOM_CREATE_DB: + case SQLCOM_CREATE_FUNCTION: + case SQLCOM_CREATE_USER: + case SQLCOM_CREATE_PROCEDURE: + case SQLCOM_CREATE_SPFUNCTION: + case SQLCOM_CREATE_VIEW: + case SQLCOM_CREATE_TRIGGER: + case SQLCOM_CREATE_SERVER: + case SQLCOM_CREATE_EVENT: + case SQLCOM_CREATE_ROLE: + case SQLCOM_CREATE_RESOURCE_GROUP: + case SQLCOM_CREATE_SRS: + return IS_BIT_SET(admission_control_filter, ADMISSION_CONTROL_CREATE); + + case SQLCOM_DELETE: + case SQLCOM_DELETE_MULTI: + return IS_BIT_SET(admission_control_filter, ADMISSION_CONTROL_DELETE); + + case SQLCOM_DROP_TABLE: + case SQLCOM_DROP_INDEX: + case SQLCOM_DROP_DB: + case SQLCOM_DROP_FUNCTION: + case SQLCOM_DROP_USER: + case SQLCOM_DROP_PROCEDURE: + case SQLCOM_DROP_VIEW: + case SQLCOM_DROP_TRIGGER: + case SQLCOM_DROP_SERVER: + case SQLCOM_DROP_EVENT: + case SQLCOM_DROP_ROLE: + case SQLCOM_DROP_SRS: + return IS_BIT_SET(admission_control_filter, ADMISSION_CONTROL_DROP); + + case SQLCOM_INSERT: + case SQLCOM_INSERT_SELECT: + return IS_BIT_SET(admission_control_filter, ADMISSION_CONTROL_INSERT); + + case SQLCOM_LOAD: + return IS_BIT_SET(admission_control_filter, ADMISSION_CONTROL_LOAD); + + case SQLCOM_SELECT: + return IS_BIT_SET(admission_control_filter, ADMISSION_CONTROL_SELECT); + + case SQLCOM_SET_OPTION: + return IS_BIT_SET(admission_control_filter, ADMISSION_CONTROL_SET); + + case SQLCOM_REPLACE: + case SQLCOM_REPLACE_SELECT: + return IS_BIT_SET(admission_control_filter, ADMISSION_CONTROL_REPLACE); + + case SQLCOM_ROLLBACK: + return IS_BIT_SET(admission_control_filter, ADMISSION_CONTROL_ROLLBACK); + + case SQLCOM_TRUNCATE: + return IS_BIT_SET(admission_control_filter, ADMISSION_CONTROL_TRUNCATE); + + case SQLCOM_UPDATE: + case SQLCOM_UPDATE_MULTI: + return IS_BIT_SET(admission_control_filter, ADMISSION_CONTROL_UPDATE); + + case SQLCOM_SHOW_DATABASES: + case SQLCOM_SHOW_TABLES: + case SQLCOM_SHOW_FIELDS: + case SQLCOM_SHOW_KEYS: + case SQLCOM_SHOW_VARIABLES: + case SQLCOM_SHOW_STATUS: + case SQLCOM_SHOW_ENGINE_LOGS: + case SQLCOM_SHOW_ENGINE_STATUS: + case SQLCOM_SHOW_ENGINE_MUTEX: + case SQLCOM_SHOW_PROCESSLIST: + case SQLCOM_SHOW_MASTER_STAT: + case SQLCOM_SHOW_SLAVE_STAT: + case SQLCOM_SHOW_GRANTS: + case SQLCOM_SHOW_CREATE: + case SQLCOM_SHOW_CHARSETS: + case SQLCOM_SHOW_COLLATIONS: + case SQLCOM_SHOW_CREATE_DB: + case SQLCOM_SHOW_TABLE_STATUS: + case SQLCOM_SHOW_TRIGGERS: + case SQLCOM_SHOW_BINLOGS: + case SQLCOM_SHOW_OPEN_TABLES: + case SQLCOM_SHOW_SLAVE_HOSTS: + case SQLCOM_SHOW_BINLOG_EVENTS: + case SQLCOM_SHOW_WARNS: + case SQLCOM_SHOW_ERRORS: + case SQLCOM_SHOW_STORAGE_ENGINES: + case SQLCOM_SHOW_PRIVILEGES: + case SQLCOM_SHOW_CREATE_PROC: + case SQLCOM_SHOW_CREATE_FUNC: + case SQLCOM_SHOW_STATUS_PROC: + case SQLCOM_SHOW_STATUS_FUNC: + case SQLCOM_SHOW_PROC_CODE: + case SQLCOM_SHOW_FUNC_CODE: + case SQLCOM_SHOW_PLUGINS: + case SQLCOM_SHOW_CREATE_EVENT: + case SQLCOM_SHOW_EVENTS: + case SQLCOM_SHOW_CREATE_TRIGGER: + case SQLCOM_SHOW_PROFILE: + case SQLCOM_SHOW_PROFILES: + case SQLCOM_SHOW_RELAYLOG_EVENTS: + case SQLCOM_SHOW_ENGINE_TRX: + case SQLCOM_SHOW_MEMORY_STATUS: + return IS_BIT_SET(admission_control_filter, ADMISSION_CONTROL_SHOW); + + case SQLCOM_CHANGE_DB: + return IS_BIT_SET(admission_control_filter, ADMISSION_CONTROL_USE); + + default: + return false; + } +} + +/* + * Admit a query based on database entity + * + * @param thd THD structure + * @param attrs Resource attributes + * + * @return 0 if the query is admitted, 1 if otherwise + */ +int multi_tenancy_admit_query(THD *thd, const MT_RESOURCE_ATTRS *attrs) { + bool admission_check = false; + + /* + * Admission control check will be enforced if ALL of the following + * conditions are satisfied + * 1. The query is run by regular (non-super) user + * 2. The THD is not a replication thread + * 3. The query is not part of a transaction (controlled by global var + * opt_admission_control_by_trx), nor the THD is already in an admission + * control (e.g. part of a multi query packet) + * 4. Session database is set for THD + * 5. sys var max_running_queries > 0 + * 6. The command is not filtered by admission_control_filter + */ + bool beginning_of_trx = + !thd->get_transaction()->is_active(Transaction_ctx::SESSION); + + if (!thd->security_context()->check_access(SUPER_ACL) && /* 1 */ + !thd->rli_slave && /* 2 */ + ((!opt_admission_control_by_trx || beginning_of_trx) && + !thd->is_in_ac) && /* 3 */ + attrs->database && /* 4 */ + db_ac->get_max_running_queries() && /* 5 */ + !filter_command(thd->lex->sql_command) /* 6 */ + ) { + admission_check = true; + } + + if (admission_check && db_ac->admission_control_enter(thd, attrs)) return 1; + + if (admission_check) thd->is_in_ac = true; + + return 0; +} + +/* + * Exit a query based on database entity + * + * @param thd THD structure + * @param attrs Resource attributes + * + * @return 0 if successful, 1 if otherwise + */ +int multi_tenancy_exit_query(THD *thd, const MT_RESOURCE_ATTRS *attrs) { + if (!attrs->database) return 0; // no limiting + + db_ac->admission_control_exit(thd, attrs); + return 0; +} + +/** + * @param thd THD structure. + * @param attrs session resource attributes + * + * Applies admission control checks for the entity. Outline of + * the steps in this function: + * 1. Error out if we crossed the max waiting limit. + * 2. Put the thd in a queue. + * 3. If we crossed the max running limit then wait for signal from threads + * that completed their query execution. + * + * Note current implementation assumes the admission control entity is + * database. We will lift the assumption and implement the entity logic in + * multitenancy plugin. + * + * @return False Run this query. + True maximum waiting queries limit reached. Error out this query. +*/ +bool AC::admission_control_enter(THD *thd, const MT_RESOURCE_ATTRS *attrs) { + bool error = false; + std::string entity = attrs->database; + const char *prev_proc_info = thd->proc_info(); + THD_STAGE_INFO(thd, stage_admission_control_enter); + bool release_lock_ac = true; + // Unlock this before waiting. + mysql_rwlock_rdlock(&LOCK_ac); + if (max_running_queries) { + auto it = ac_map.find(entity); + if (it == ac_map.end()) { + // New DB. + mysql_rwlock_unlock(&LOCK_ac); + insert(entity); + mysql_rwlock_rdlock(&LOCK_ac); + it = ac_map.find(entity); + } + + if (!thd->ac_node) { + // Both THD and the admission control queue will share the object + // created here. + thd->ac_node = std::make_shared(); + } + auto ac_info = it->second; + MT_RETURN_TYPE ret = MT_RETURN_TYPE::MULTI_TENANCY_RET_FALLBACK; + mysql_mutex_lock(&ac_info->lock); + // if plugin is disabled, fallback to check global query limit + if (ret == MT_RETURN_TYPE::MULTI_TENANCY_RET_FALLBACK) { + if (ac_info->queue.size() < max_running_queries) + ret = MT_RETURN_TYPE::MULTI_TENANCY_RET_ACCEPT; + else { + if (max_waiting_queries && + ac_info->queue.size() >= max_running_queries + max_waiting_queries) + ret = MT_RETURN_TYPE::MULTI_TENANCY_RET_REJECT; + else + ret = MT_RETURN_TYPE::MULTI_TENANCY_RET_WAIT; + } + } + + switch (ret) { + case MT_RETURN_TYPE::MULTI_TENANCY_RET_REJECT: + ++total_aborted_queries; + // We reached max waiting limit. Error out + mysql_mutex_unlock(&ac_info->lock); + error = true; + break; + + case MT_RETURN_TYPE::MULTI_TENANCY_RET_WAIT: + ac_info->queue.emplace_back(thd->ac_node); + ++ac_info->waiting_queries; + /** + Inserting or deleting in std::map will not invalidate existing + iterators except of course if the current iterator is erased. If the + db corresponding to this iterator is getting dropped, these waiting + queries are given signal to abort before the iterator + is erased. See AC::remove(). + So, we don't need LOCK_ac here. The motivation to unlock the read lock + is that waiting queries here shouldn't block other operations + modifying ac_map or max_running_queries/max_waiting_queries. + */ + mysql_rwlock_unlock(&LOCK_ac); + release_lock_ac = false; + wait_for_signal(thd, thd->ac_node, ac_info); + --ac_info->waiting_queries; + break; + + case MT_RETURN_TYPE::MULTI_TENANCY_RET_ACCEPT: + // We are below the max running limit. + ac_info->queue.emplace_back(thd->ac_node); + mysql_mutex_unlock(&ac_info->lock); + break; + + default: + // unreachable branch + assert(0); + } + } + if (release_lock_ac) { + mysql_rwlock_unlock(&LOCK_ac); + } + thd->set_proc_info(prev_proc_info); + return error; +} + +void AC::wait_for_signal(THD *thd, std::shared_ptr &ac_node, + const Ac_info_ptr &ac_info) { + PSI_stage_info old_stage; + mysql_mutex_lock(&ac_node->lock); + /** + The locking order followed during admission_control_enter() is + lock ac_info + lock ac_node + unlock ac_info + unlock ac_node + The locks are interleaved to avoid possible races which makes + this waiting thread miss the signal from admission_control_exit(). + */ + mysql_mutex_unlock(&ac_info->lock); + thd->ENTER_COND(&ac_node->cond, &ac_node->lock, &stage_waiting_for_admission, + &old_stage); + // Spurious wake-ups are rare and fine in this design. + mysql_cond_wait(&ac_node->cond, &ac_node->lock); + mysql_mutex_unlock(&ac_node->lock); + thd->EXIT_COND(&old_stage); +} + +/** + @param thd THD structure + @param attrs session resource attributes + + Signals one waiting thread. Pops out the first THD in the queue. +*/ +void AC::admission_control_exit(THD *thd, const MT_RESOURCE_ATTRS *attrs) { + std::string entity = attrs->database; + const char *prev_proc_info = thd->proc_info(); + THD_STAGE_INFO(thd, stage_admission_control_exit); + mysql_rwlock_rdlock(&LOCK_ac); + auto it = ac_map.find(entity); + if (it != ac_map.end()) { + auto ac_info = it->second.get(); + mysql_mutex_lock(&ac_info->lock); + if (max_running_queries && ac_info->queue.size() > max_running_queries) { + signal(ac_info->queue[max_running_queries]); + } + // The queue is empty if max_running_queries is toggled to 0 + // when this THD is inside admission_control_enter(). + if (ac_info->queue.size()) { + /** + The popped value here doesn't necessarily give the ac_node of the + current THD. It is better if the popped value is not accessed at all. + */ + ac_info->queue.pop_front(); + } + mysql_mutex_unlock(&ac_info->lock); + } + mysql_rwlock_unlock(&LOCK_ac); + thd->set_proc_info(prev_proc_info); +} diff --git a/sql/sql_admission_control.h b/sql/sql_admission_control.h new file mode 100644 index 000000000000..c946da130601 --- /dev/null +++ b/sql/sql_admission_control.h @@ -0,0 +1,366 @@ +/* Copyright (c) 2016, Facebook. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* + * sql_admission_control.h/cc + * + * This module handles multi-tenancy resource allocation on the server side. + * Functions will call into multi-tenancy plugin interfaces (if installed) to + * make a decision of whether or not a resource can be allocated. Connections + * and query limits are among the resources multi-tenancy plugin allocates. + * Note that admission control is now part of the multi-tenancy module to + * hanlde query throttling. + * + * The isolation level is defined by an entity. The entity could be a database, + * a user, or any thing that multi-tenancy plugin defines to isolate the + * resource allocation. + * + * See sql_admission_control.cc for implementation. + */ + +#ifndef _sql_admission_control_h +#define _sql_admission_control_h + +#include "mysql/psi/mysql_mutex.h" + +#include +#include "sql/sql_class.h" + +#define IS_BIT_SET(val, n) ((val) & (1 << (n))) + +extern bool opt_admission_control_by_trx; +extern ulonglong admission_control_filter; +class AC; +extern AC *db_ac; + +#ifdef HAVE_PSI_INTERFACE +extern PSI_stage_info stage_admission_control_enter; +extern PSI_stage_info stage_admission_control_exit; +extern PSI_stage_info stage_waiting_for_admission; +#endif + +// TODO: remove this block eventually when this becomes part of audit plugin? +// TBD later +// Resource isolation types that a multi-tenancy plugin will handle. +// Currently connection and query limits are two resource types. More will be +// supported in the future. +enum class enum_multi_tenancy_resource_type : int32_t { + MULTI_TENANCY_RESOURCE_CONNECTION, + MULTI_TENANCY_RESOURCE_QUERY, + + MULTI_TENANCY_NUM_RESOURCE_TYPES +}; + +// TODO: Remove this enum when code becomes part of plugin +// Callback function return types. +// - ACCEPT: resource can be granted +// - WAIT: may need to wait for resource to be freed up +// - REJECT: resource cannot be granted +// - FALLBACK: plugin is disabled +enum class enum_multi_tenancy_return_type : int32_t { + MULTI_TENANCY_RET_ACCEPT = 0, + MULTI_TENANCY_RET_WAIT, + MULTI_TENANCY_RET_REJECT, + + MULTI_TENANCY_RET_FALLBACK, + MULTI_TENANCY_NUM_RETURN_TYPES +}; + +typedef enum enum_multi_tenancy_resource_type MT_RESOURCE_TYPE; +typedef enum enum_multi_tenancy_return_type MT_RETURN_TYPE; +typedef std::unordered_map ATTRS_MAP_T; +typedef std::vector> ATTRS_VEC_T; + +struct multi_tenancy_resource_attributes { + const ATTRS_MAP_T *connection_attrs_map; + const ATTRS_VEC_T *query_attrs_vec; + const char *database; +}; + +typedef struct multi_tenancy_resource_attributes MT_RESOURCE_ATTRS; + +// The contents here must match entries in admission_control_filter_names array +enum enum_admission_control_filter { + ADMISSION_CONTROL_ALTER, + ADMISSION_CONTROL_BEGIN, + ADMISSION_CONTROL_COMMIT, + ADMISSION_CONTROL_CREATE, + ADMISSION_CONTROL_DELETE, + ADMISSION_CONTROL_DROP, + ADMISSION_CONTROL_INSERT, + ADMISSION_CONTROL_LOAD, + ADMISSION_CONTROL_SELECT, + ADMISSION_CONTROL_SET, + ADMISSION_CONTROL_REPLACE, + ADMISSION_CONTROL_ROLLBACK, + ADMISSION_CONTROL_TRUNCATE, + ADMISSION_CONTROL_UPDATE, + ADMISSION_CONTROL_SHOW, + ADMISSION_CONTROL_USE, + ADMISSION_CONTROL_END = 64 +}; + +extern int multi_tenancy_admit_query(THD *, const MT_RESOURCE_ATTRS *); +extern int multi_tenancy_exit_query(THD *, const MT_RESOURCE_ATTRS *); + +/** + Per-thread information used in admission control. +*/ + +#ifdef HAVE_PSI_INTERFACE + +static PSI_mutex_key key_ac_node_lock; +static PSI_cond_key key_ac_node_cond; + +static PSI_mutex_info all_ac_node_mutexes[] = { + {&key_ac_node_lock, "st_ac_node::lock", PSI_FLAG_SINGLETON, 0, + PSI_DOCUMENT_ME}}; + +static PSI_cond_info all_ac_node_conds[] = { + {&key_ac_node_cond, "st_ac_node::cond", PSI_FLAG_SINGLETON, 0, + PSI_DOCUMENT_ME}}; + +static void init_st_ac_node_keys() { + const char *const category = "sql"; + int count; + + count = static_cast(array_elements(all_ac_node_mutexes)); + mysql_mutex_register(category, all_ac_node_mutexes, count); + + count = static_cast(array_elements(all_ac_node_conds)); + mysql_cond_register(category, all_ac_node_conds, count); +} +#endif + +struct st_ac_node { + st_ac_node(const st_ac_node &) = delete; + st_ac_node &operator=(const st_ac_node &) = delete; + mysql_mutex_t lock; + mysql_cond_t cond; + st_ac_node() { +#ifdef HAVE_PSI_INTERFACE + init_st_ac_node_keys(); +#endif + mysql_mutex_init(key_ac_node_lock, &lock, MY_MUTEX_INIT_FAST); + mysql_cond_init(key_ac_node_cond, &cond); + } + + ~st_ac_node() { + mysql_mutex_destroy(&lock); + mysql_cond_destroy(&cond); + } +}; + +using st_ac_node_ptr = std::shared_ptr; + +/** + Class used in admission control. + + Every entity (database or table or user name) will have this + object created and stored in the global map AC::ac_map. + +*/ +class Ac_info { + friend class AC; +#ifdef HAVE_PSI_INTERFACE + PSI_mutex_key key_lock; + PSI_mutex_info key_lock_info[1] = { + {&key_lock, "Ac_info::lock", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME}}; +#endif + // Queue to track the running and waiting threads. + using ac_node_ptr_container = std::deque; + ac_node_ptr_container queue; + std::atomic_ulong waiting_queries; + // Protects the queue. + mysql_mutex_t lock; + + public: + Ac_info() : waiting_queries(0) { +#ifdef HAVE_PSI_INTERFACE + mysql_mutex_register("sql", key_lock_info, array_elements(key_lock_info)); +#endif + mysql_mutex_init(key_lock, &lock, MY_MUTEX_INIT_FAST); + } + + ~Ac_info() { mysql_mutex_destroy(&lock); } + // Disable copy constructor. + Ac_info(const Ac_info &) = delete; + Ac_info &operator=(const Ac_info &) = delete; +}; + +using Ac_info_ptr = std::shared_ptr; + +/** + Global class used to enforce per admission control limits. +*/ +class AC { + // This map is protected by the rwlock LOCK_ac. + using Ac_info_ptr_container = std::unordered_map; + Ac_info_ptr_container ac_map; + // Variables to track global limits + ulong max_running_queries, max_waiting_queries; + /** + Protects ac_map and max_running_queries/max_waiting_queries. + + Locking order followed is LOCK_ac, Ac_info::lock, st_ac_node::lock. + */ + mutable mysql_rwlock_t LOCK_ac; +#ifdef HAVE_PSI_INTERFACE + PSI_rwlock_key key_rwlock_LOCK_ac; + PSI_rwlock_info key_rwlock_LOCK_ac_info[1] = { + {&key_rwlock_LOCK_ac, "AC::rwlock", PSI_FLAG_SINGLETON, 0, + PSI_DOCUMENT_ME}}; +#endif + + std::atomic_ullong total_aborted_queries; + + public: + AC() + : max_running_queries(0), + max_waiting_queries(0), + total_aborted_queries(0) { +#ifdef HAVE_PSI_INTERFACE + mysql_rwlock_register("sql", key_rwlock_LOCK_ac_info, + array_elements(key_rwlock_LOCK_ac_info)); +#endif + mysql_rwlock_init(key_rwlock_LOCK_ac, &LOCK_ac); + } + + ~AC() { mysql_rwlock_destroy(&LOCK_ac); } + // Disable copy constructor. + AC(const AC &) = delete; + AC &operator=(const AC &) = delete; + + // TODO: check if this can be moved to st_ac_node + inline void signal(st_ac_node_ptr &ac_node) { + assert(ac_node && ac_node.get()); + mysql_mutex_lock(&ac_node->lock); + mysql_cond_signal(&ac_node->cond); + mysql_mutex_unlock(&ac_node->lock); + } + + /* + * Removes a dropped entity info from the global map. + */ + void remove(const char *entity) { + std::string str(entity); + // First take a read lock to unblock any waiting queries. + mysql_rwlock_rdlock(&LOCK_ac); + auto it = ac_map.find(str); + if (it != ac_map.end()) { + auto ac_info = it->second; + mysql_mutex_lock(&ac_info->lock); + while (ac_info->waiting_queries) { + for (uint i = max_running_queries; i < ac_info->queue.size(); ++i) { + signal(ac_info->queue[i]); + } + } + mysql_mutex_unlock(&ac_info->lock); + } + mysql_rwlock_unlock(&LOCK_ac); + mysql_rwlock_wrlock(&LOCK_ac); + it = ac_map.find(std::string(str)); + if (it != ac_map.end()) { + ac_map.erase(it); + } + mysql_rwlock_unlock(&LOCK_ac); + } + + void insert(const std::string &entity) { + mysql_rwlock_wrlock(&LOCK_ac); + if (ac_map.find(entity) == ac_map.end()) { + ac_map.emplace(entity, std::make_shared()); + } + mysql_rwlock_unlock(&LOCK_ac); + } + + void update_max_running_queries(ulong val) { + // lock to protect against erasing map iterators. + mysql_rwlock_wrlock(&LOCK_ac); + ulong old_val = max_running_queries; + max_running_queries = val; + // Signal any waiting threads which are below the new limit. Note 0 is a + // special case where every waiting thread needs to be signalled. + if (val > old_val || !val) { + for (auto &it : ac_map) { + auto &ac_info = it.second; + mysql_mutex_lock(&ac_info->lock); + for (uint i = old_val; (!val || i < val) && i < ac_info->queue.size(); + ++i) { + signal(ac_info->queue[i]); + } + mysql_mutex_unlock(&ac_info->lock); + } + } + mysql_rwlock_unlock(&LOCK_ac); + } + + void update_max_waiting_queries(ulong val) { + mysql_rwlock_wrlock(&LOCK_ac); + max_waiting_queries = val; + mysql_rwlock_unlock(&LOCK_ac); + } + + ulong get_max_running_queries() const { + mysql_rwlock_rdlock(&LOCK_ac); + ulong res = max_running_queries; + mysql_rwlock_unlock(&LOCK_ac); + return res; + } + + ulong get_max_waiting_queries() const { + mysql_rwlock_rdlock(&LOCK_ac); + ulong res = max_waiting_queries; + mysql_rwlock_unlock(&LOCK_ac); + return res; + } + + bool admission_control_enter(THD *, const MT_RESOURCE_ATTRS *); + void admission_control_exit(THD *, const MT_RESOURCE_ATTRS *); + void wait_for_signal(THD *, st_ac_node_ptr &, const Ac_info_ptr &ac_info); + + ulonglong get_total_aborted_queries() const { return total_aborted_queries; } + + ulong get_total_running_queries() { + ulonglong res = 0; + mysql_rwlock_rdlock(&LOCK_ac); + for (auto it : ac_map) { + auto &ac_info = it.second; + mysql_mutex_lock(&ac_info->lock); + res += ac_info->queue.size() < max_running_queries ? ac_info->queue.size() + : max_running_queries; + mysql_mutex_unlock(&ac_info->lock); + } + mysql_rwlock_unlock(&LOCK_ac); + return res; + } + + ulong get_total_waiting_queries() { + ulonglong res = 0; + mysql_rwlock_rdlock(&LOCK_ac); + for (auto it : ac_map) { + auto &ac_info = it.second; + mysql_mutex_lock(&ac_info->lock); + if (ac_info->queue.size() > max_running_queries) + res += ac_info->queue.size() - max_running_queries; + mysql_mutex_unlock(&ac_info->lock); + } + mysql_rwlock_unlock(&LOCK_ac); + return res; + } +}; + +#endif /* _sql_admission_control_h */ diff --git a/sql/sql_class.cc b/sql/sql_class.cc index ea3b1fc3732d..a7f0bca0665c 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -115,6 +115,7 @@ #include "sql/transaction.h" // trans_rollback #include "sql/transaction_info.h" +#include "sql/sql_admission_control.h" #include "sql/xa.h" #include "sql/xa/sql_cmd_xa.h" // Sql_cmd_xa_* #include "sql/xa/transaction_cache.h" // xa::Transaction_cache @@ -1470,6 +1471,14 @@ void THD::release_resources() { // Mark THD life cycle state as "SCHEDULED_FOR_DISPOSAL". start_disposal(); + /* if we are still in admission control, release it */ + if (is_in_ac) { + MT_RESOURCE_ATTRS attrs = {&connection_attrs_map, &query_attrs_list, + m_db.str}; + multi_tenancy_exit_query(this, &attrs); + is_in_ac = false; + } + /* Close connection */ if (is_classic_protocol() && get_protocol_classic()->get_vio()) { vio_delete(get_protocol_classic()->get_vio()); diff --git a/sql/sql_class.h b/sql/sql_class.h index abaf45c53b31..2565643d65dc 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -175,6 +175,8 @@ struct LOG_INFO; typedef struct user_conn USER_CONN; struct MYSQL_LOCK; +struct st_ac_node; +using st_ac_node_ptr = std::shared_ptr; enum enum_slave_use_idempotent_for_recovery { SLAVE_USE_IDEMPOTENT_FOR_RECOVERY_NO, @@ -4833,6 +4835,10 @@ class THD : public MDL_context_owner, bool is_secondary_engine_forced() const { return m_secondary_engine_forced; } + /* whether the session is already in admission control for queries */ + bool is_in_ac = false; + st_ac_node_ptr ac_node; + private: /** This flag tells if a secondary storage engine can be used to diff --git a/sql/sql_db.cc b/sql/sql_db.cc index c202fd2694d2..8c459cebf57f 100644 --- a/sql/sql_db.cc +++ b/sql/sql_db.cc @@ -89,7 +89,8 @@ #include "sql/rpl_gtid.h" #include "sql/rpl_replica_commit_order_manager.h" // Commit_order_manager #include "sql/session_tracker.h" -#include "sql/sp.h" // lock_db_routines +#include "sql/sp.h" // lock_db_routines +#include "sql/sql_admission_control.h" #include "sql/sql_base.h" // lock_table_names #include "sql/sql_class.h" // THD #include "sql/sql_const.h" @@ -1128,6 +1129,10 @@ bool mysql_rm_db(THD *thd, const LEX_CSTRING &db, bool if_exists) { } } + if (!error) { + db_ac->remove(db.str); + } + thd->server_status |= SERVER_STATUS_DB_DROPPED; my_ok(thd, deleted_tables); return false; @@ -1389,6 +1394,17 @@ long mysql_rm_arc_files(THD *thd, MY_DIR *dirp, const char *org_path) { static void mysql_change_db_impl(THD *thd, const LEX_CSTRING &new_db_name, ulong new_db_access, const CHARSET_INFO *new_db_charset) { + if (thd->is_in_ac) { + // current admission control is per database + assert(thd->db().str != nullptr); + // if the thd is already in admission control for the previous + // database, we need to release it before resetting the value. + MT_RESOURCE_ATTRS attrs = {nullptr, nullptr, nullptr}; + attrs.database = thd->db().str; + multi_tenancy_exit_query(thd, &attrs); + thd->is_in_ac = false; + } + /* 1. Change current database in THD. */ if (new_db_name.str == nullptr) { diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index b100feb9cd7e..e4998b979fc3 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -144,6 +144,7 @@ #include "sql/sp_cache.h" // sp_cache_enforce_limit #include "sql/sp_head.h" // sp_head #include "sql/sql_admin.h" +#include "sql/sql_admission_control.h" #include "sql/sql_alter.h" #include "sql/sql_audit.h" // MYSQL_AUDIT_NOTIFY_CONNECTION_CHANGE_USER #include "sql/sql_base.h" // find_temporary_table @@ -2373,6 +2374,17 @@ bool dispatch_command(THD *thd, const COM_DATA *com_data, } } + if (thd->is_in_ac && + (!opt_admission_control_by_trx || + !thd->get_transaction()->is_active( + Transaction_ctx::SESSION) /* thd->is_real_trans equivalent */)) { + MT_RESOURCE_ATTRS mattrs = {&thd->connection_attrs_map, + &thd->query_attrs_list, thd->db().str}; + multi_tenancy_exit_query(thd, &mattrs); + + thd->is_in_ac = false; + } + thd->bind_parameter_values = nullptr; thd->bind_parameter_values_count = 0; @@ -5528,7 +5540,17 @@ void dispatch_sql_command(THD *thd, Parser_state *parser_state, const bool switched = mgr_ptr->switch_resource_group_if_needed( thd, &src_res_grp, &dest_res_grp, &ticket, &cur_ticket); - error = mysql_execute_command(thd, true, last_timer); + MT_RESOURCE_ATTRS attrs = {&thd->connection_attrs_map, + &thd->query_attrs_list, thd->db().str}; + + if (multi_tenancy_admit_query(thd, &attrs)) { + my_error(ER_DB_ADMISSION_CONTROL, MYF(0), + db_ac->get_max_waiting_queries(), + thd->db().str ? thd->db().str : "unknown database"); + error = 1; + } else { + error = mysql_execute_command(thd, true, last_timer); + } if (switched) mgr_ptr->restore_original_resource_group(thd, src_res_grp, diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index b933e7b828fb..e915ede39f65 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -128,6 +128,7 @@ #include "sql/server_component/log_builtins_imp.h" #include "sql/session_tracker.h" #include "sql/sp_head.h" // SP_PSI_STATEMENT_INFO_COUNT +#include "sql/sql_admission_control.h" #include "sql/sql_lex.h" #include "sql/sql_locale.h" // my_locale_by_number #include "sql/sql_parse.h" // killall_non_super_threads @@ -2956,6 +2957,63 @@ static Sys_var_ulong Sys_max_connections( /* max_connections is used as a sizing hint by the performance schema. */ sys_var::PARSE_EARLY); +static bool update_max_running_queries( + sys_var *self MY_ATTRIBUTE((unused)), THD *thd MY_ATTRIBUTE((unused)), + enum_var_type type MY_ATTRIBUTE((unused))) { + db_ac->update_max_running_queries(opt_max_running_queries); + + return false; +} + +static bool update_max_waiting_queries( + sys_var *self MY_ATTRIBUTE((unused)), THD *thd MY_ATTRIBUTE((unused)), + enum_var_type type MY_ATTRIBUTE((unused))) { + db_ac->update_max_waiting_queries(opt_max_waiting_queries); + + return false; +} + +static Sys_var_ulong Sys_max_running_queries( + "max_running_queries", + "The maximum number of running queries allowed for a database. " + "If this value is 0, no such limits are applied.", + GLOBAL_VAR(opt_max_running_queries), CMD_LINE(REQUIRED_ARG), + VALID_RANGE(0, 100000), DEFAULT(0), BLOCK_SIZE(1), NO_MUTEX_GUARD, + NOT_IN_BINLOG, ON_CHECK(nullptr), ON_UPDATE(update_max_running_queries)); + +static Sys_var_ulong Sys_max_waiting_queries( + "max_waiting_queries", + "The maximum number of waiting queries allowed for a database." + "If this value is 0, no such limits are applied.", + GLOBAL_VAR(opt_max_waiting_queries), CMD_LINE(REQUIRED_ARG), + VALID_RANGE(0, 100000), DEFAULT(0), BLOCK_SIZE(1), NO_MUTEX_GUARD, + NOT_IN_BINLOG, ON_CHECK(nullptr), ON_UPDATE(update_max_waiting_queries)); + +/* + Do not add more than 63 entries here. + There should be corresponding entry for each of these in + enum_admission_control_filter. +*/ +const char *admission_control_filter_names[] = { + "ALTER", "BEGIN", "COMMIT", "CREATE", "DELETE", "DROP", + "INSERT", "LOAD", "SELECT", "SET", "REPLACE", "ROLLBACK", + "TRUNCATE", "UPDATE", "SHOW", "USE", 0}; +static Sys_var_set Admission_control_options( + "admission_control_filter", + "Commands that are skipped in admission control checks. The legal " + "values are: " + "ALTER, BEGIN, COMMIT, CREATE, DELETE, DROP, INSERT, LOAD, " + "SELECT, SET, REPLACE, ROLLBACK, TRUNCATE, UPDATE, SHOW " + "and empty string", + GLOBAL_VAR(admission_control_filter), CMD_LINE(REQUIRED_ARG), + admission_control_filter_names, DEFAULT(0)); + +static Sys_var_bool Sys_admission_control_by_trx( + "admission_control_by_trx", + "Allow open transactions to go through admission control", + GLOBAL_VAR(opt_admission_control_by_trx), CMD_LINE(OPT_ARG), + DEFAULT(false)); + static Sys_var_ulong Sys_max_connect_errors( "max_connect_errors", "If there is more than this number of interrupted connections from "