From ef7b25061b95a70458c110c46ddca0aa9058e8fc Mon Sep 17 00:00:00 2001 From: Jason Rahman Date: Sun, 1 Mar 2020 17:25:51 -0800 Subject: [PATCH] Implement resultset checksums Summary: Implement a CRC32 checksum over the resultset field headers and row contents. Enablement is controlled by a global variable + the query checksum feature being enabled. The query_checksum attribute was renamed to checksum for symmetry. The idea being the an attribute with key 'checksum' contains a value reflecting a checksum of the entity (query or resultset) to which that attribute is attached. Reviewed By: jkedgar Differential Revision: D20186509 --- client/client_priv.h | 1 + client/mysql.cc | 37 +++- include/mysql.h | 8 +- include/mysql.h.pp | 2 + mysql-test/r/mysqld--help-notwin.result | 11 +- mysql-test/r/query_checksum.result | 3 + mysql-test/r/resultset_checksum.result | 89 ++++++++++ .../r/enable_resultset_checksum_basic.result | 15 ++ .../t/enable_resultset_checksum_basic.test | 23 +++ mysql-test/t/query_checksum.test | 17 +- mysql-test/t/resultset_checksum.test | 168 ++++++++++++++++++ share/messages_to_clients.txt | 6 +- sql-common/client.cc | 99 +++++++++-- sql/mysqld.cc | 6 + sql/mysqld.h | 3 + sql/protocol_classic.cc | 41 ++++- sql/protocol_classic.h | 12 +- sql/sql_parse.cc | 4 +- sql/sys_vars.cc | 7 +- 19 files changed, 522 insertions(+), 30 deletions(-) create mode 100644 mysql-test/r/resultset_checksum.result create mode 100644 mysql-test/suite/sys_vars/r/enable_resultset_checksum_basic.result create mode 100644 mysql-test/suite/sys_vars/t/enable_resultset_checksum_basic.test create mode 100644 mysql-test/t/resultset_checksum.test diff --git a/client/client_priv.h b/client/client_priv.h index db0a8f061859..07b41c2cbb65 100644 --- a/client/client_priv.h +++ b/client/client_priv.h @@ -199,6 +199,7 @@ enum options_client { OPT_MYSQLBINLOG_SKIP_EMPTY_TRANS, OPT_PRINT_GTIDS, OPT_MINIMUM_HLC, + OPT_CHECKSUM, /* Add new option above this */ OPT_MAX_CLIENT_OPTION }; diff --git a/client/mysql.cc b/client/mysql.cc index 0318212f8f71..5462b3c9c9c0 100644 --- a/client/mysql.cc +++ b/client/mysql.cc @@ -165,6 +165,7 @@ static bool ignore_errors = false, wait_flag = false, quick = false, ignore_spaces = false, sigint_received = false, opt_syslog = false, opt_binhex = false; static bool opt_binary_as_hex_set_explicitly = false; +static bool opt_checksum = false; static bool debug_info_flag, debug_check_flag; static bool column_types_flag; static bool preserve_comments = false; @@ -323,7 +324,8 @@ static int com_quit(String *str, char *), com_go(String *str, char *), com_prompt(String *str, char *), com_delimiter(String *str, char *), com_warnings(String *str, char *), com_nowarnings(String *str, char *), com_resetconnection(String *str, char *), com_attr(String *str, char *), - com_query_attributes(String *str, char *); + com_query_attributes(String *str, char *), + com_resp_attr(String *str, char *); static int com_shell(String *str, char *); #ifdef USE_POPEN @@ -399,6 +401,7 @@ static COMMANDS commands[] = { {"quit", 'q', com_quit, false, "Quit mysql."}, {"rehash", '#', com_rehash, false, "Rebuild completion hash."}, {"setattr", 'z', com_attr, true, "Set query attribute."}, + {"getattr", 'z', com_resp_attr, true, "Get response attribute."}, {"source", '.', com_source, true, "Execute an SQL script file. Takes a file name as an argument."}, {"status", 's', com_status, false, @@ -1824,6 +1827,11 @@ static struct my_option my_long_options[] = { "with --disable-reconnect. This option is enabled by default.", &opt_reconnect, &opt_reconnect, nullptr, GET_BOOL, NO_ARG, 1, 0, 0, nullptr, 0, nullptr}, + {"checksum", OPT_CHECKSUM, + "Use query and resultset checksums to verify the integrity of the query " + "and resultset in transit. This is disabled by default.", + &opt_checksum, &opt_checksum, 0, GET_BOOL, NO_ARG, 0, 0, 0, nullptr, 0, + nullptr}, {"silent", 's', "Be more silent. Print results with a tab as separator, " "each row on new line.", @@ -3075,6 +3083,8 @@ static int mysql_real_query_for_lazy(const char *buf, size_t length, error = 0; if (set_params && global_attrs->set_params(&mysql)) break; + if (opt_checksum) + mysql_options4(&mysql, MYSQL_OPT_QUERY_ATTR_ADD, "checksum", "ON"); if (!mysql_real_query(&mysql, buf, (ulong)length)) break; error = put_error(&mysql); if ((mysql_errno(&mysql) != CR_SERVER_GONE_ERROR && @@ -4146,6 +4156,31 @@ static int com_attr(String *buffer [[maybe_unused]], char *line) { return 0; } +static int com_resp_attr(String *, char *line) { + static const char *delim = " \t"; + char *ptr = nullptr; + char *buf = strdup(line); + const char *cmd __attribute__((unused)) = strtok_r(buf, delim, &ptr); + const char *key = strtok_r(nullptr, delim, &ptr); + + if (!key) { + put_info("Usage: getattr key", INFO_ERROR); + free(buf); + return -1; + } + + const char *value; + size_t len; + if (!mysql_resp_attr_find(&mysql, key, &value, &len)) { + char *tmp = strndup(value, len); + put_info(tmp, INFO_INFO); + free(tmp); + } + + free(buf); + return 0; +} + static int com_print(String *buffer, char *line [[maybe_unused]]) { tee_puts("--------------", stdout); (void)tee_fputs(buffer->c_ptr(), stdout); diff --git a/include/mysql.h b/include/mysql.h index 2d15b4b319f7..dd3acaf77107 100644 --- a/include/mysql.h +++ b/include/mysql.h @@ -312,8 +312,12 @@ typedef struct MYSQL { MYSQL_FIELD *fields; struct MEM_ROOT *field_alloc; uint64_t affected_rows; - uint64_t insert_id; /* id if insert on table with NEXTNR */ - uint64_t extra_info; /* Not used */ + uint64_t insert_id; /* id if insert on table with NEXTNR */ + uint64_t extra_info; /* Not used */ + /* Should the resultset checksum be calculated */ + bool should_record_checksum; + /* Store the computed checksum from the resultset */ + unsigned long checksum; unsigned long thread_id; /* Id for connection in server */ unsigned long packet_length; unsigned int port; diff --git a/include/mysql.h.pp b/include/mysql.h.pp index d5bf6f09a011..2486df9cefd0 100644 --- a/include/mysql.h.pp +++ b/include/mysql.h.pp @@ -556,6 +556,8 @@ uint64_t affected_rows; uint64_t insert_id; uint64_t extra_info; + bool should_record_checksum; + unsigned long checksum; unsigned long thread_id; unsigned long packet_length; unsigned int port; diff --git a/mysql-test/r/mysqld--help-notwin.result b/mysql-test/r/mysqld--help-notwin.result index 10194b42a3b7..bf8d9514ba49 100644 --- a/mysql-test/r/mysqld--help-notwin.result +++ b/mysql-test/r/mysqld--help-notwin.result @@ -363,9 +363,13 @@ The following options may be given as the first argument: --enable-binlog-hlc Enable logging HLC timestamp as part of Metadata log event --enable-query-checksum - Enable query checksums for queries that have the - query_checksum query attribute set. Uses a CRC32 checksum - of the query contents, but not the attributes + Enable query checksums for queries that have the checksum + query attribute set. Uses a CRC32 checksum of the query + contents, but not the attributes + --enable-resultset-checksum + Enable CRC32 resultset checksums if requested by the + client sending the checksum query attribute, set to the + query checksum --end-markers-in-json In JSON output ("EXPLAIN FORMAT=JSON" and optimizer trace), if variable is set to 1, repeats the structure's @@ -1932,6 +1936,7 @@ disconnect-slave-event-count 0 div-precision-increment 4 enable-binlog-hlc FALSE enable-query-checksum FALSE +enable-resultset-checksum FALSE end-markers-in-json FALSE enforce-gtid-consistency FALSE eq-range-index-dive-limit 200 diff --git a/mysql-test/r/query_checksum.result b/mysql-test/r/query_checksum.result index 7d2905d906f5..c1d573524574 100644 --- a/mysql-test/r/query_checksum.result +++ b/mysql-test/r/query_checksum.result @@ -12,6 +12,9 @@ SELECT 1;|||| /* */ SELECT 1;;|||| 1 1 +/* */ SELECT 1;;|||| +1 +1 set @@global.enable_query_checksum = FALSE; SELECT 1;SELECT 2;|||| 1 diff --git a/mysql-test/r/resultset_checksum.result b/mysql-test/r/resultset_checksum.result new file mode 100644 index 000000000000..442e59aaddca --- /dev/null +++ b/mysql-test/r/resultset_checksum.result @@ -0,0 +1,89 @@ +SET @@global.enable_resultset_checksum = ON; +SET @@session.session_track_response_attributes = ON; +Creating a table for later tests +CREATE TABLE data (a VARCHAR(1024), b TEXT, c INT); +No Checksum: ; +Check charset, which impacts encoding and thus checksums +SET NAMES 'latin1'; +No Checksum: ; +SELECT @@session.character_set_client; +@@session.character_set_client +latin1 +Verify SELECTS of variables +SELECT @@session.character_set_connection; +@@session.character_set_connection +latin1 +Checksum: 1525358488; +Add some dummy data +INSERT INTO data (a, b, c) VALUES ('testing', 'result checksums', 0); +No Checksum: ; +INSERT INTO data (a, b, c) VALUES ('testing', 'result checksums', 1); +INSERT INTO data (a, b, c) VALUES ('testing', 'result checksums', 2); +INSERT INTO data (a, b, c) VALUES ('testing', 'result checksums', 3); +INSERT INTO data (a, b, c) VALUES ('testing', 'result checksums', 4); +INSERT INTO data (a, b, c) VALUES ('testing', 'result checksums', 5); +INSERT INTO data (a, b, c) VALUES ('testing', 'result checksums', 6); +Verify multiple fields in a single row +SELECT * FROM data WHERE c = 1; +a b c +testing result checksums 1 +Checksum: 1134867259; +Verify multiple fields in multiple rows +SELECT * FROM data ORDER BY c; +a b c +testing result checksums 0 +testing result checksums 1 +testing result checksums 2 +testing result checksums 3 +testing result checksums 4 +testing result checksums 5 +testing result checksums 6 +Checksum: 4204762635; +Verify single massive result row +Checksum: 2267222966; +Verify multiple massive rows +Checksum: 2166497371; +Test around 16MB packet boundaries +Checksum: 2512256286; +Checksum: 982468368; +Checksum: 1437851346; +Checksum: 2699967101; +Checksum: 261522601; +Checksum: 1879284586; +Checksum: 2282180928; +Several random row sizes +SELECT REPEAT('a', 255); +REPEAT('a', 255) +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +Checksum: 2275529303; +SELECT REPEAT('a', 256); +REPEAT('a', 256) +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +Checksum: 1932162976; +SELECT REPEAT('a', 257); +REPEAT('a', 257) +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +Checksum: 2889758595; +verify SHOW SLAVE HOSTS output +SHOW REPLICAS; +Server_Id Host Port Source_Id Replica_UUID Is_semi_sync_slave Replication_status +Checksum: 3702298611; +Verify SHOW TABLES output +SHOW TABLES; +Tables_in_test +data +Checksum: 3251735689; +Verify SHOW GRANTS output +SHOW GRANTS; +Grants for root@localhost +GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, SHUTDOWN, PROCESS, FILE, REFERENCES, INDEX, ALTER, SHOW DATABASES, SUPER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER, CREATE TABLESPACE, CREATE ROLE, DROP ROLE ON *.* TO `root`@`localhost` WITH GRANT OPTION +GRANT APPLICATION_PASSWORD_ADMIN,AUDIT_ABORT_EXEMPT,AUDIT_ADMIN,AUTHENTICATION_POLICY_ADMIN,BACKUP_ADMIN,BINLOG_ADMIN,BINLOG_ENCRYPTION_ADMIN,CLONE_ADMIN,CONNECTION_ADMIN,ENCRYPTION_KEY_ADMIN,FLUSH_OPTIMIZER_COSTS,FLUSH_STATUS,FLUSH_TABLES,FLUSH_USER_RESOURCES,GROUP_REPLICATION_ADMIN,GROUP_REPLICATION_STREAM,INNODB_REDO_LOG_ARCHIVE,INNODB_REDO_LOG_ENABLE,PASSWORDLESS_USER_ADMIN,PERSIST_RO_VARIABLES_ADMIN,REPLICATION_APPLIER,REPLICATION_SLAVE_ADMIN,RESOURCE_GROUP_ADMIN,RESOURCE_GROUP_USER,ROLE_ADMIN,SERVICE_CONNECTION_ADMIN,SESSION_VARIABLES_ADMIN,SET_USER_ID,SHOW_ROUTINE,SYSTEM_USER,SYSTEM_VARIABLES_ADMIN,TABLE_ENCRYPTION_ADMIN,XA_RECOVER_ADMIN ON *.* TO `root`@`localhost` WITH GRANT OPTION +GRANT PROXY ON ``@`` TO `root`@`localhost` WITH GRANT OPTION +Checksum: 428322074; +Verify CHECK TABLE output +CHECK TABLE data; +Table Op Msg_type Msg_text +test.data check status OK +Checksum: 2020821564; +SET @@global.enable_resultset_checksum = default; +DROP TABLE data; diff --git a/mysql-test/suite/sys_vars/r/enable_resultset_checksum_basic.result b/mysql-test/suite/sys_vars/r/enable_resultset_checksum_basic.result new file mode 100644 index 000000000000..f376d240b437 --- /dev/null +++ b/mysql-test/suite/sys_vars/r/enable_resultset_checksum_basic.result @@ -0,0 +1,15 @@ +Default value of enable_resultset_checksum is false +SELECT @@global.enable_resultset_checksum; +@@global.enable_resultset_checksum +0 +SELECT @@session.enable_resultset_checksum; +ERROR HY000: Variable 'enable_resultset_checksum' is a GLOBAL variable +Expected error 'Variable is a GLOBAL variable' +SET @@global.enable_resultset_checksum = true; +SELECT @@global.enable_resultset_checksum; +@@global.enable_resultset_checksum +1 +SET @@global.enable_resultset_checksum = default; +SELECT @@global.enable_resultset_checksum; +@@global.enable_resultset_checksum +0 diff --git a/mysql-test/suite/sys_vars/t/enable_resultset_checksum_basic.test b/mysql-test/suite/sys_vars/t/enable_resultset_checksum_basic.test new file mode 100644 index 000000000000..36a939b8f181 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/enable_resultset_checksum_basic.test @@ -0,0 +1,23 @@ +-- source include/load_sysvars.inc + +#### +# Verify default value false +#### +--echo Default value of enable_resultset_checksum is false +SELECT @@global.enable_resultset_checksum; + +#### +# Verify that this is not a session variable +#### +--Error ER_INCORRECT_GLOBAL_LOCAL_VAR +SELECT @@session.enable_resultset_checksum; +--echo Expected error 'Variable is a GLOBAL variable' + +#### +## Actual tests which enables this are in different file which test this feature +#### +SET @@global.enable_resultset_checksum = true; +SELECT @@global.enable_resultset_checksum; + +SET @@global.enable_resultset_checksum = default; +SELECT @@global.enable_resultset_checksum; diff --git a/mysql-test/t/query_checksum.test b/mysql-test/t/query_checksum.test index 849015d816ed..881c5efcb45e 100644 --- a/mysql-test/t/query_checksum.test +++ b/mysql-test/t/query_checksum.test @@ -1,7 +1,7 @@ set @@global.enable_query_checksum = TRUE; # Add a checksum that fails -query_attrs_add query_checksum 0; +query_attrs_add checksum 0; delimiter ||||; --error ER_QUERY_CHECKSUM_FAILED SELECT 1;SELECT 2;|||| @@ -9,21 +9,28 @@ delimiter ;|||| # Add a checksum that succeeds query_attrs_reset; -query_attrs_add query_checksum 2629868284; +query_attrs_add checksum 2629868284; delimiter ||||; SELECT 1;SELECT 2;|||| delimiter ;|||| # Add a checksum that succeeds, on a single query query_attrs_reset; -query_attrs_add query_checksum 0078787420; +query_attrs_add checksum 0078787420; delimiter ||||; SELECT 1;|||| delimiter ;|||| # Add a checksum that passes, with leading comment and trailing ; query_attrs_reset; -query_attrs_add query_checksum 3498200060; +query_attrs_add checksum 3498200060; +delimiter ||||; +/* */ SELECT 1;;|||| +delimiter ;|||| + +# Check that the automatic checksumming works as expected +query_attrs_reset; +query_attrs_add checksum ON; delimiter ||||; /* */ SELECT 1;;|||| delimiter ;|||| @@ -31,7 +38,7 @@ delimiter ;|||| # Add a checksum that fails, but the global variable is off query_attrs_reset; set @@global.enable_query_checksum = FALSE; -query_attrs_add query_checksum 0; +query_attrs_add checksum 0; delimiter ||||; SELECT 1;SELECT 2;|||| delimiter ;|||| diff --git a/mysql-test/t/resultset_checksum.test b/mysql-test/t/resultset_checksum.test new file mode 100644 index 000000000000..65697fe40bc5 --- /dev/null +++ b/mysql-test/t/resultset_checksum.test @@ -0,0 +1,168 @@ +SET @@global.enable_resultset_checksum = ON; +SET @@session.session_track_response_attributes = ON; + +--echo Creating a table for later tests +query_attrs_reset; +query_attrs_add checksum ON; +CREATE TABLE data (a VARCHAR(1024), b TEXT, c INT); +let $checksum = get_response_attribute(checksum); +--echo No Checksum: $checksum; + +--echo Check charset, which impacts encoding and thus checksums +query_attrs_reset; +query_attrs_add checksum ON; +SET NAMES 'latin1'; +let $checksum = get_response_attribute(checksum); +--echo No Checksum: $checksum; + +SELECT @@session.character_set_client; + +--echo Verify SELECTS of variables +query_attrs_reset; +query_attrs_add checksum ON; +SELECT @@session.character_set_connection; +let $checksum = get_response_attribute(checksum); +--echo Checksum: $checksum; + +--echo Add some dummy data +query_attrs_reset; +query_attrs_add checksum ON; +INSERT INTO data (a, b, c) VALUES ('testing', 'result checksums', 0); +let $checksum = get_response_attribute(checksum); +--echo No Checksum: $checksum; + +INSERT INTO data (a, b, c) VALUES ('testing', 'result checksums', 1); +INSERT INTO data (a, b, c) VALUES ('testing', 'result checksums', 2); +INSERT INTO data (a, b, c) VALUES ('testing', 'result checksums', 3); +INSERT INTO data (a, b, c) VALUES ('testing', 'result checksums', 4); +INSERT INTO data (a, b, c) VALUES ('testing', 'result checksums', 5); +INSERT INTO data (a, b, c) VALUES ('testing', 'result checksums', 6); + +--echo Verify multiple fields in a single row +query_attrs_reset; +query_attrs_add checksum ON; +SELECT * FROM data WHERE c = 1; +let $checksum = get_response_attribute(checksum); +--echo Checksum: $checksum; + +--echo Verify multiple fields in multiple rows +query_attrs_reset; +query_attrs_add checksum ON; +SELECT * FROM data ORDER BY c; +let $checksum = get_response_attribute(checksum); +--echo Checksum: $checksum; + +# The following results are massive and we should avoid logging them +--disable_result_log +--disable_query_log + +--echo Verify single massive result row +query_attrs_reset; +query_attrs_add checksum ON; +SELECT REPEAT('a', 33554432); +let $checksum = get_response_attribute(checksum); +--echo Checksum: $checksum; + +--echo Verify multiple massive rows +query_attrs_reset; +query_attrs_add checksum ON; +SELECT a, REPEAT('a', 33554432) FROM data ORDER BY c; +let $checksum = get_response_attribute(checksum); +--echo Checksum: $checksum; + +--echo Test around 16MB packet boundaries +query_attrs_reset; +query_attrs_add checksum ON; +eval SELECT REPEAT('a', 16777190); +let $checksum = get_response_attribute(checksum); +--echo Checksum: $checksum; + +query_attrs_reset; +query_attrs_add checksum ON; +eval SELECT REPEAT('a', 16777195); +let $checksum = get_response_attribute(checksum); +--echo Checksum: $checksum; + +query_attrs_reset; +query_attrs_add checksum ON; +eval SELECT REPEAT('a', 16777200); +let $checksum = get_response_attribute(checksum); +--echo Checksum: $checksum; + +query_attrs_reset; +query_attrs_add checksum ON; +eval SELECT REPEAT('a', 16777205); +let $checksum = get_response_attribute(checksum); +--echo Checksum: $checksum; + +query_attrs_reset; +query_attrs_add checksum ON; +eval SELECT REPEAT('a', 16777210); +let $checksum = get_response_attribute(checksum); +--echo Checksum: $checksum; + +query_attrs_reset; +query_attrs_add checksum ON; +eval SELECT REPEAT('a', 16777215); +let $checksum = get_response_attribute(checksum); +--echo Checksum: $checksum; + +query_attrs_reset; +query_attrs_add checksum ON; +eval SELECT REPEAT('a', 16777220); +let $checksum = get_response_attribute(checksum); +--echo Checksum: $checksum; + + +--enable_query_log +--enable_result_log + +--echo Several random row sizes +query_attrs_reset; +query_attrs_add checksum ON; +SELECT REPEAT('a', 255); +let $checksum = get_response_attribute(checksum); +--echo Checksum: $checksum; + +query_attrs_reset; +query_attrs_add checksum ON; +SELECT REPEAT('a', 256); +let $checksum = get_response_attribute(checksum); +--echo Checksum: $checksum; + +query_attrs_reset; +query_attrs_add checksum ON; +SELECT REPEAT('a', 257); +let $checksum = get_response_attribute(checksum); +--echo Checksum: $checksum; + +--echo verify SHOW SLAVE HOSTS output +query_attrs_reset; +query_attrs_add checksum ON; +SHOW REPLICAS; +let $checksum = get_response_attribute(checksum); +--echo Checksum: $checksum; + +--echo Verify SHOW TABLES output +query_attrs_reset; +query_attrs_add checksum ON; +SHOW TABLES; +let $checksum = get_response_attribute(checksum); +--echo Checksum: $checksum; + +--echo Verify SHOW GRANTS output +query_attrs_reset; +query_attrs_add checksum ON; +SHOW GRANTS; +let $checksum = get_response_attribute(checksum); +--echo Checksum: $checksum; + +--echo Verify CHECK TABLE output +query_attrs_reset; +query_attrs_add checksum ON; +CHECK TABLE data; +let $checksum = get_response_attribute(checksum); +--echo Checksum: $checksum; + +SET @@global.enable_resultset_checksum = default; +DROP TABLE data; diff --git a/share/messages_to_clients.txt b/share/messages_to_clients.txt index 2b886fb0dac6..fcca5478fd90 100644 --- a/share/messages_to_clients.txt +++ b/share/messages_to_clients.txt @@ -9981,10 +9981,10 @@ ER_PLACEHOLDER_50080 eng "Placeholder" ER_QUERY_CHECKSUM_FAILED - eng "Expected query checksum %u but found %u" + eng "Expected query checksum %lu but found %lu" -ER_PLACEHOLDER_50082 - eng "Placeholder" +ER_RESULTSET_CHECKSUM_FAILED + eng "Expected resultset checksum %lu but found %lu" # # FB error codes specifically for 8.0 only start at 50100. diff --git a/sql-common/client.cc b/sql-common/client.cc index eb40d5669f49..2a873f5b01f3 100644 --- a/sql-common/client.cc +++ b/sql-common/client.cc @@ -783,6 +783,32 @@ static int read_resp_attrs(MYSQL *mysql, STATE_INFO *info, uchar **pos) { return 0; } +int validate_checksum(MYSQL *mysql) { + const char *checksumStr; + size_t checksumLen; + if (!mysql_resp_attr_find(mysql, "checksum", &checksumStr, &checksumLen)) { + // checksumStr is not null-terminated, unfortunately + char buff[32]; + if (checksumLen >= sizeof(buff)) return 1; + memcpy(buff, checksumStr, checksumLen); + buff[checksumLen] = 0; + unsigned long checksum = strtoul(buff, NULL, 10); + if (checksum != mysql->checksum) { + set_mysql_extended_error(mysql, ER_RESULTSET_CHECKSUM_FAILED, + unknown_sqlstate, + "Expected resultset checksum %lu but found %lu", + checksum, mysql->checksum); + return 1; + } + } + return 0; +} + +void update_checksum(MYSQL *mysql, uchar *pkt, size_t pkt_len) { + if (mysql->should_record_checksum) + mysql->checksum = crc32(mysql->checksum, pkt, pkt_len); +} + /** Read Ok packet along with the server state change information. */ @@ -1262,6 +1288,7 @@ ulong cli_safe_read_with_ok_complete(MYSQL *mysql, bool parse_ok, if (net->read_pos[0] == 0) { if (parse_ok) { read_ok_ex(mysql, len); + if (validate_checksum(mysql)) return packet_error; return len; } } @@ -1283,6 +1310,8 @@ ulong cli_safe_read_with_ok_complete(MYSQL *mysql, bool parse_ok, if (is_data_packet) *is_data_packet = false; /* parse it if requested */ if (parse_ok) read_ok_ex(mysql, len); + // Validate resultset checksum (if sent from server) + if (parse_ok && validate_checksum(mysql)) return packet_error; return len; } /* for old client detect EOF packet */ @@ -1653,6 +1682,7 @@ static net_async_status flush_one_result_nonblocking(MYSQL *mysql, bool *res) { if (mysql->server_capabilities & CLIENT_DEPRECATE_EOF && !is_data_packet) { read_ok_ex(mysql, packet_length); + if (validate_checksum(mysql)) return NET_ASYNC_ERROR; } else { mysql->warning_count = uint2korr(pos); pos += 2; @@ -1662,6 +1692,8 @@ static net_async_status flush_one_result_nonblocking(MYSQL *mysql, bool *res) { } break; } + /* Update the checksum as we flush results */ + update_checksum(mysql, mysql->net.read_pos, packet_length); } return NET_ASYNC_COMPLETE; } @@ -1693,15 +1725,18 @@ static bool flush_one_result(MYSQL *mysql) { cli_safe_read() has set an error for us, just return. */ if (packet_length == packet_error) return true; + if (is_data_packet) + update_checksum(mysql, mysql->net.read_pos, packet_length); } while (mysql->net.read_pos[0] == 0 || is_data_packet); /* Analyse final OK packet (EOF packet if it is old client) */ if (protocol_41(mysql)) { uchar *pos = mysql->net.read_pos + 1; - if (mysql->server_capabilities & CLIENT_DEPRECATE_EOF && !is_data_packet) + if (mysql->server_capabilities & CLIENT_DEPRECATE_EOF && !is_data_packet) { read_ok_ex(mysql, packet_length); - else { + if (validate_checksum(mysql)) return true; + } else { mysql->warning_count = uint2korr(pos); pos += 2; mysql->server_status = uint2korr(pos); @@ -1755,6 +1790,7 @@ static bool opt_flush_ok_packet(MYSQL *mysql, bool *is_ok_packet) { *is_ok_packet = is_OK_packet(mysql, packet_length); if (*is_ok_packet) { read_ok_ex(mysql, packet_length); + if (validate_checksum(mysql)) return true; #if defined(CLIENT_PROTOCOL_TRACING) if (mysql->server_status & SERVER_MORE_RESULTS_EXISTS) MYSQL_TRACE_STAGE(mysql, WAIT_FOR_RESULT); @@ -2931,6 +2967,8 @@ net_async_status cli_read_rows_nonblocking(MYSQL *mysql, async_context->prev_row_ptr = &cur->next; to = (char *)(cur->data + fields + 1); end_to = to + pkt_len - 1; + /* Calculate checksum if requested */ + update_checksum(mysql, cp, pkt_len); for (field = 0; field < fields; field++) { if ((len = (ulong)net_field_length(&cp)) == NULL_LENGTH) { /* null field */ @@ -2971,9 +3009,10 @@ net_async_status cli_read_rows_nonblocking(MYSQL *mysql, *async_context->prev_row_ptr = nullptr; /* last pointer is null */ /* read EOF packet or OK packet if it is new client */ if (pkt_len > 1) { - if (mysql->server_capabilities & CLIENT_DEPRECATE_EOF && !is_data_packet) + if (mysql->server_capabilities & CLIENT_DEPRECATE_EOF && !is_data_packet) { read_ok_ex(mysql, pkt_len); - else { + if (validate_checksum(mysql)) return NET_ASYNC_ERROR; + } else { mysql->warning_count = uint2korr(cp + 1); mysql->server_status = uint2korr(cp + 3); } @@ -3050,6 +3089,8 @@ MYSQL_DATA *cli_read_rows(MYSQL *mysql, MYSQL_FIELD *mysql_fields, prev_ptr = &cur->next; to = (char *)(cur->data + fields + 1); end_to = to + pkt_len - 1; + /* Calculate checksum if requested */ + update_checksum(mysql, cp, pkt_len); for (field = 0; field < fields; field++) { uint length_len = 0; if (packet_left < 1 || @@ -3093,9 +3134,10 @@ MYSQL_DATA *cli_read_rows(MYSQL *mysql, MYSQL_FIELD *mysql_fields, *prev_ptr = nullptr; /* last pointer is null */ /* read EOF packet or OK packet if it is new client */ if (pkt_len > 1) { - if (mysql->server_capabilities & CLIENT_DEPRECATE_EOF && !is_data_packet) + if (mysql->server_capabilities & CLIENT_DEPRECATE_EOF && !is_data_packet) { read_ok_ex(mysql, pkt_len); - else { + if (validate_checksum(mysql)) return nullptr; + } else { mysql->warning_count = uint2korr(cp + 1); mysql->server_status = uint2korr(cp + 3); } @@ -3126,9 +3168,10 @@ static int read_one_row_complete(MYSQL *mysql, ulong pkt_len, if (net->read_pos[0] != 0x00 && !is_data_packet) { if (pkt_len > 1) /* MySQL 4.1 protocol */ { - if (mysql->server_capabilities & CLIENT_DEPRECATE_EOF) + if (mysql->server_capabilities & CLIENT_DEPRECATE_EOF) { read_ok_ex(mysql, pkt_len); - else { + if (validate_checksum(mysql)) return -1; + } else { mysql->warning_count = uint2korr(net->read_pos + 1); mysql->server_status = uint2korr(net->read_pos + 3); } @@ -3144,6 +3187,8 @@ static int read_one_row_complete(MYSQL *mysql, ulong pkt_len, prev_pos = nullptr; /* allowed to write at packet[-1] */ pos = net->read_pos; end_pos = pos + pkt_len; + /* Calculate checksum if requested */ + update_checksum(mysql, (uchar *)pos, pkt_len); for (field = 0; field < fields; field++) { if (pos >= end_pos) { set_mysql_error(mysql, CR_MALFORMED_PACKET, unknown_sqlstate); @@ -7519,6 +7564,7 @@ static bool cli_read_query_result(MYSQL *mysql) { pos = (uchar *)mysql->net.read_pos; if ((field_count = net_field_length(&pos)) == 0) { read_ok_ex(mysql, length); + if (validate_checksum(mysql)) return true; #if defined(CLIENT_PROTOCOL_TRACING) if (mysql->server_status & SERVER_MORE_RESULTS_EXISTS) MYSQL_TRACE_STAGE(mysql, WAIT_FOR_RESULT); @@ -7596,6 +7642,7 @@ static net_async_status cli_read_query_result_nonblocking(MYSQL *mysql) { pos = (uchar *)mysql->net.read_pos; if ((field_count = net_field_length(&pos)) == 0) { read_ok_ex(mysql, length); + if (validate_checksum(mysql)) return NET_ASYNC_ERROR; #if defined(CLIENT_PROTOCOL_TRACING) if (mysql->server_status & SERVER_MORE_RESULTS_EXISTS) MYSQL_TRACE_STAGE(mysql, WAIT_FOR_RESULT); @@ -7739,6 +7786,37 @@ static int mysql_prepare_com_query_parameters(MYSQL *mysql, } return 0; } + +int STDCALL handle_checksums(MYSQL *mysql, const char *query, ulong length) { + mysql->should_record_checksum = false; + auto extension = mysql->options.extension; + if (extension && extension->query_attributes) { + /* + * checksum being set here indicates we should also checksum the resultset + * client can optionally set the value of the checksum with a precomputed + * query checksum for testing purposes + */ + auto &query_attrs = extension->query_attributes->hash; + auto it = query_attrs.find("checksum"); + auto exists = it != query_attrs.end(); + mysql->should_record_checksum = exists; + mysql->checksum = 0; + + /* If checksum = ON then also perform the crc32 in libmysql */ + if (mysql->should_record_checksum && it->second == "ON") { + unsigned long checksum = crc32(0, (const uchar *)query, length); + char buf[32]; + snprintf(buf, sizeof(buf), "%lu", checksum); + // Remove and re-add the attribute + if (mysql_options(mysql, MYSQL_OPT_QUERY_ATTR_DELETE, "checksum")) + return 1; + if (mysql_options4(mysql, MYSQL_OPT_QUERY_ATTR_ADD, "checksum", buf)) + return 1; + } + } + return 0; +} + /* Send the query and return so we can do something else. Needs to be followed by mysql_read_query_result() when we want to @@ -7755,7 +7833,7 @@ int STDCALL mysql_send_query(MYSQL *mysql, const char *query, ulong length) { assert(ext); if ((info = STATE_DATA(mysql))) free_state_change_info(ext); - + if (handle_checksums(mysql, query, length)) return 1; size_t query_attrs_len = mysql->options.extension ? mysql->options.extension->query_attributes_length @@ -7808,7 +7886,7 @@ static net_async_status mysql_send_query_nonblocking_inner(MYSQL *mysql, if ((info = STATE_DATA(mysql))) free_state_change_info(static_cast(mysql->extension)); - + if (handle_checksums(mysql, query, length)) return NET_ASYNC_ERROR; bool ret; MYSQL_ASYNC *async_context = ASYNC_DATA(mysql); @@ -8209,6 +8287,7 @@ static MYSQL_RES *cli_use_result(MYSQL *mysql) { result->handle = mysql; result->current_row = nullptr; mysql->fields = nullptr; /* fields is now in result */ + mysql->checksum = 0; mysql->status = MYSQL_STATUS_USE_RESULT; mysql->unbuffered_fetch_owner = &result->unbuffered_fetch_cancelled; return result; /* Data is read to be fetched */ diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 93e6e4ea8fc4..c87f38456b68 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -1321,6 +1321,12 @@ ulonglong maximum_hlc_drift_ns = 0; /* Enable query checksum validation for queries with a checksum sent */ bool enable_query_checksum = false; +/* + * Enable resultset checksum for resultsets of queries with the checksum + * query attr set + */ +bool enable_resultset_checksum = false; + #if defined(ENABLED_DEBUG_SYNC) MYSQL_PLUGIN_IMPORT uint opt_debug_sync_timeout = 0; #endif /* defined(ENABLED_DEBUG_SYNC) */ diff --git a/sql/mysqld.h b/sql/mysqld.h index 9b17ae8c7530..48e03812de38 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -416,6 +416,9 @@ extern ulonglong maximum_hlc_drift_ns; /* Enable query checksum validation for queries with a checksum sent */ extern bool enable_query_checksum; +/* Enable resultset checksums */ +extern bool enable_resultset_checksum; + extern uint net_compression_level; extern uint zstd_net_compression_level; diff --git a/sql/protocol_classic.cc b/sql/protocol_classic.cc index 0507af492ac1..d55a6eca78f7 100644 --- a/sql/protocol_classic.cc +++ b/sql/protocol_classic.cc @@ -430,6 +430,7 @@ #include #include +#include // crc32 #include #include @@ -467,6 +468,7 @@ using std::max; using std::min; +static const char *CHECKSUM = "checksum"; static const unsigned int PACKET_BUFFER_EXTRA_ALLOC = 1024; static bool net_send_error_packet(THD *, uint, const char *, const char *); static bool net_send_error_packet(NET *, uint, const char *, const char *, bool, @@ -1307,11 +1309,14 @@ bool Protocol_classic::send_ok(uint server_status, uint statement_warn_count, ulonglong affected_rows, ulonglong last_insert_id, const char *message) { DBUG_TRACE; + record_checksum(); const bool retval = net_send_ok(m_thd, server_status, statement_warn_count, affected_rows, last_insert_id, message, false); // Reclaim some memory convert.shrink(m_thd->variables.net_buffer_length); + checksum = 0; + should_record_checksum = false; return retval; } @@ -1331,13 +1336,16 @@ bool Protocol_classic::send_eof(uint server_status, uint statement_warn_count) { */ if (has_client_capability(CLIENT_DEPRECATE_EOF) && (m_thd->get_command() != COM_BINLOG_DUMP && - m_thd->get_command() != COM_BINLOG_DUMP_GTID)) + m_thd->get_command() != COM_BINLOG_DUMP_GTID)) { + record_checksum(); retval = net_send_ok(m_thd, server_status, statement_warn_count, 0, 0, nullptr, true); - else + } else retval = net_send_eof(m_thd, server_status, statement_warn_count); // Reclaim some memory convert.shrink(m_thd->variables.net_buffer_length); + checksum = 0; + should_record_checksum = false; return retval; } @@ -1354,6 +1362,8 @@ bool Protocol_classic::send_error(uint sql_errno, const char *err_msg, net_send_error_packet(m_thd, sql_errno, err_msg, sql_state); // Reclaim some memory convert.shrink(m_thd->variables.net_buffer_length); + checksum = 0; + should_record_checksum = false; return retval; } @@ -3112,6 +3122,15 @@ bool Protocol_classic::start_result_metadata(uint num_cols_arg, uint flags, field_count = num_cols; sending_flags = flags; + should_record_checksum = false; + for (const auto &p : m_thd->query_attrs_list) { + if (enable_resultset_checksum && p.first == CHECKSUM) { + should_record_checksum = true; + checksum = 0; + break; + } + } + DBUG_EXECUTE_IF("send_large_column_count_in_metadata", num_cols = 50397184;); /* We don't send number of column for PS, as it's sent in a preceding packet. @@ -3387,6 +3406,10 @@ bool Protocol_classic::send_field_metadata(Send_field *field, bool Protocol_classic::end_row() { DBUG_TRACE; + // Only checksum the data rows + if (should_record_checksum) + checksum = crc32(checksum, (uchar *)packet->ptr(), packet->length()); + return my_net_write(&m_thd->net, pointer_cast(packet->ptr()), packet->length()); } @@ -3423,6 +3446,20 @@ bool Protocol_classic::connection_alive() const { return m_thd->net.vio != nullptr; } +void Protocol_classic::record_checksum() { + if (should_record_checksum) { + char buf[32]; + snprintf(buf, sizeof(buf), "%lu", checksum); + auto &tracker = m_thd->session_tracker; + auto sess_tracker = tracker.get_tracker(SESSION_RESP_ATTR_TRACKER); + if (sess_tracker->is_enabled()) { + LEX_CSTRING key = {CHECKSUM, strlen(CHECKSUM)}; + LEX_CSTRING value = {buf, strlen(buf)}; + sess_tracker->mark_as_changed(m_thd, &key, &value); + } + } +} + void Protocol_text::start_row() { field_pos = 0; packet->length(0); diff --git a/sql/protocol_classic.h b/sql/protocol_classic.h index 764f1f3ecab4..510e5072cfbb 100644 --- a/sql/protocol_classic.h +++ b/sql/protocol_classic.h @@ -70,6 +70,8 @@ class Protocol_classic : public Protocol { uint sending_flags; ulong input_packet_length; uchar *input_raw_packet; + unsigned long checksum; + bool should_record_checksum; const CHARSET_INFO *result_cs; bool send_ok(uint server_status, uint statement_warn_count, @@ -86,11 +88,17 @@ class Protocol_classic : public Protocol { public: bool bad_packet; Protocol_classic() - : send_metadata(false), input_packet_length(0), bad_packet(true) {} + : send_metadata(false), + input_packet_length(0), + checksum(0), + should_record_checksum(false), + bad_packet(true) {} Protocol_classic(THD *thd) : send_metadata(false), input_packet_length(0), input_raw_packet(nullptr), + checksum(0), + should_record_checksum(false), bad_packet(true) { init(thd); } @@ -178,6 +186,8 @@ class Protocol_classic : public Protocol { bool has_client_capability(unsigned long client_capability) override { return (bool)(m_client_capabilities & client_capability); } + /* Set the checksum response attribute */ + void record_checksum(); // TODO: temporary functions. Will be removed. // DON'T USE IN ANY NEW FEATURES. /* Return NET */ diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 457485cbb509..45024393c761 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2034,11 +2034,11 @@ bool dispatch_command(THD *thd, const COM_DATA *com_data, if (enable_query_checksum) { bool failed = false; for (const auto &p : thd->query_attrs_list) { - if (p.first == "query_checksum") { + if (p.first == "checksum") { unsigned long checksum = crc32(0, (const uchar *)com_data->com_query.query, com_data->com_query.length); - unsigned long expected = std::stoul(p.second); + unsigned long expected = strtoul(p.second.c_str(), nullptr, 10); if (expected != checksum) { my_error(ER_QUERY_CHECKSUM_FAILED, MYF(0), expected, checksum); failed = true; diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 45354b8c5b7e..ef171fafed51 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -8231,6 +8231,11 @@ static Sys_var_bool Sys_rpl_slave_flow_control( static Sys_var_bool Sys_enable_query_checksum( "enable_query_checksum", "Enable query checksums for queries that have " - "the query_checksum query attribute set. Uses a CRC32 checksum of the " + "the checksum query attribute set. Uses a CRC32 checksum of the " "query contents, but not the attributes", GLOBAL_VAR(enable_query_checksum), CMD_LINE(OPT_ARG), DEFAULT(false)); +static Sys_var_bool Sys_enable_resultset_checksum( + "enable_resultset_checksum", + "Enable CRC32 resultset checksums if requested by the client sending the " + "checksum query attribute, set to the query checksum", + GLOBAL_VAR(enable_resultset_checksum), CMD_LINE(OPT_ARG), DEFAULT(false));