Implement resultset checksums

Summary:
Implement a CRC32 checksum over the resultset field headers and
row contents. Enablement is controlled by a global variable + the query checksum feature being enabled.

The query_checksum attribute was renamed to checksum for symmetry. The idea being the an attribute with key 'checksum' contains a value reflecting a checksum of the entity (query or resultset) to which that attribute is attached.

Reviewed By: jkedgar

Differential Revision: D20186509

client/client_priv.h

@@ -199,6 +199,7 @@ enum options_client {
   OPT_MYSQLBINLOG_SKIP_EMPTY_TRANS,
   OPT_PRINT_GTIDS,
   OPT_MINIMUM_HLC,
+  OPT_CHECKSUM,
   /* Add new option above this */
   OPT_MAX_CLIENT_OPTION
 };

client/mysql.cc

@@ -165,6 +165,7 @@ static bool ignore_errors = false, wait_flag = false, quick = false,
             ignore_spaces = false, sigint_received = false, opt_syslog = false,
             opt_binhex = false;
 static bool opt_binary_as_hex_set_explicitly = false;
+static bool opt_checksum = false;
 static bool debug_info_flag, debug_check_flag;
 static bool column_types_flag;
 static bool preserve_comments = false;
@@ -323,7 +324,8 @@ static int com_quit(String *str, char *), com_go(String *str, char *),
     com_prompt(String *str, char *), com_delimiter(String *str, char *),
     com_warnings(String *str, char *), com_nowarnings(String *str, char *),
     com_resetconnection(String *str, char *), com_attr(String *str, char *),
-    com_query_attributes(String *str, char *);
+    com_query_attributes(String *str, char *),
+    com_resp_attr(String *str, char *);
 static int com_shell(String *str, char *);
 
 #ifdef USE_POPEN
@@ -399,6 +401,7 @@ static COMMANDS commands[] = {
     {"quit", 'q', com_quit, false, "Quit mysql."},
     {"rehash", '#', com_rehash, false, "Rebuild completion hash."},
     {"setattr", 'z', com_attr, true, "Set query attribute."},
+    {"getattr", 'z', com_resp_attr, true, "Get response attribute."},
     {"source", '.', com_source, true,
      "Execute an SQL script file. Takes a file name as an argument."},
     {"status", 's', com_status, false,
@@ -1824,6 +1827,11 @@ static struct my_option my_long_options[] = {
      "with --disable-reconnect. This option is enabled by default.",
      &opt_reconnect, &opt_reconnect, nullptr, GET_BOOL, NO_ARG, 1, 0, 0,
      nullptr, 0, nullptr},
+    {"checksum", OPT_CHECKSUM,
+     "Use query and resultset checksums to verify the integrity of the query "
+     "and resultset in transit. This is disabled by default.",
+     &opt_checksum, &opt_checksum, 0, GET_BOOL, NO_ARG, 0, 0, 0, nullptr, 0,
+     nullptr},
     {"silent", 's',
      "Be more silent. Print results with a tab as separator, "
      "each row on new line.",
@@ -3075,6 +3083,8 @@ static int mysql_real_query_for_lazy(const char *buf, size_t length,
     error = 0;
 
     if (set_params && global_attrs->set_params(&mysql)) break;
+    if (opt_checksum)
+      mysql_options4(&mysql, MYSQL_OPT_QUERY_ATTR_ADD, "checksum", "ON");
     if (!mysql_real_query(&mysql, buf, (ulong)length)) break;
     error = put_error(&mysql);
     if ((mysql_errno(&mysql) != CR_SERVER_GONE_ERROR &&
@@ -4146,6 +4156,31 @@ static int com_attr(String *buffer [[maybe_unused]], char *line) {
   return 0;
 }
 
+static int com_resp_attr(String *, char *line) {
+  static const char *delim = " \t";
+  char *ptr = nullptr;
+  char *buf = strdup(line);
+  const char *cmd __attribute__((unused)) = strtok_r(buf, delim, &ptr);
+  const char *key = strtok_r(nullptr, delim, &ptr);
+
+  if (!key) {
+    put_info("Usage: getattr key", INFO_ERROR);
+    free(buf);
+    return -1;
+  }
+
+  const char *value;
+  size_t len;
+  if (!mysql_resp_attr_find(&mysql, key, &value, &len)) {
+    char *tmp = strndup(value, len);
+    put_info(tmp, INFO_INFO);
+    free(tmp);
+  }
+
+  free(buf);
+  return 0;
+}
+
 static int com_print(String *buffer, char *line [[maybe_unused]]) {
   tee_puts("--------------", stdout);
   (void)tee_fputs(buffer->c_ptr(), stdout);

include/mysql.h

@@ -312,8 +312,12 @@ typedef struct MYSQL {
   MYSQL_FIELD *fields;
   struct MEM_ROOT *field_alloc;
   uint64_t affected_rows;
-  uint64_t insert_id;      /* id if insert on table with NEXTNR */
-  uint64_t extra_info;     /* Not used */
+  uint64_t insert_id;  /* id if insert on table with NEXTNR */
+  uint64_t extra_info; /* Not used */
+  /* Should the resultset checksum be calculated */
+  bool should_record_checksum;
+  /* Store the computed checksum from the resultset */
+  unsigned long checksum;
   unsigned long thread_id; /* Id for connection in server */
   unsigned long packet_length;
   unsigned int port;

include/mysql.h.pp

@@ -556,6 +556,8 @@
   uint64_t affected_rows;
   uint64_t insert_id;
   uint64_t extra_info;
+  bool should_record_checksum;
+  unsigned long checksum;
   unsigned long thread_id;
   unsigned long packet_length;
   unsigned int port;

mysql-test/r/mysqld--help-notwin.result

@@ -363,9 +363,13 @@ The following options may be given as the first argument:
  --enable-binlog-hlc Enable logging HLC timestamp as part of Metadata log
  event
  --enable-query-checksum 
- Enable query checksums for queries that have the
- query_checksum query attribute set. Uses a CRC32 checksum
- of the query contents, but not the attributes
+ Enable query checksums for queries that have the checksum
+ query attribute set. Uses a CRC32 checksum of the query
+ contents, but not the attributes
+ --enable-resultset-checksum 
+ Enable CRC32 resultset checksums if requested by the
+ client sending the checksum query attribute, set to the
+ query checksum
  --end-markers-in-json 
  In JSON output ("EXPLAIN FORMAT=JSON" and optimizer
  trace), if variable is set to 1, repeats the structure's
@@ -1932,6 +1936,7 @@ disconnect-slave-event-count 0
 div-precision-increment 4
 enable-binlog-hlc FALSE
 enable-query-checksum FALSE
+enable-resultset-checksum FALSE
 end-markers-in-json FALSE
 enforce-gtid-consistency FALSE
 eq-range-index-dive-limit 200

mysql-test/r/query_checksum.result

@@ -12,6 +12,9 @@ SELECT 1;||||
 /* */ SELECT 1;;||||
 1
 1
+/* */ SELECT 1;;||||
+1
+1
 set @@global.enable_query_checksum = FALSE;
 SELECT 1;SELECT 2;||||
 1

mysql-test/r/resultset_checksum.result

@@ -0,0 +1,89 @@
+SET @@global.enable_resultset_checksum = ON;
+SET @@session.session_track_response_attributes = ON;
+Creating a table for later tests
+CREATE TABLE data (a VARCHAR(1024), b TEXT, c INT);
+No Checksum: ;
+Check charset, which impacts encoding and thus checksums
+SET NAMES 'latin1';
+No Checksum: ;
+SELECT @@session.character_set_client;
+@@session.character_set_client
+latin1
+Verify SELECTS of variables
+SELECT @@session.character_set_connection;
+@@session.character_set_connection
+latin1
+Checksum: 1525358488;
+Add some dummy data
+INSERT INTO data (a, b, c) VALUES ('testing', 'result checksums', 0);
+No Checksum: ;
+INSERT INTO data (a, b, c) VALUES ('testing', 'result checksums', 1);
+INSERT INTO data (a, b, c) VALUES ('testing', 'result checksums', 2);
+INSERT INTO data (a, b, c) VALUES ('testing', 'result checksums', 3);
+INSERT INTO data (a, b, c) VALUES ('testing', 'result checksums', 4);
+INSERT INTO data (a, b, c) VALUES ('testing', 'result checksums', 5);
+INSERT INTO data (a, b, c) VALUES ('testing', 'result checksums', 6);
+Verify multiple fields in a single row
+SELECT * FROM data WHERE c = 1;
+a	b	c
+testing	result checksums	1
+Checksum: 1134867259;
+Verify multiple fields in multiple rows
+SELECT * FROM data ORDER BY c;
+a	b	c
+testing	result checksums	0
+testing	result checksums	1
+testing	result checksums	2
+testing	result checksums	3
+testing	result checksums	4
+testing	result checksums	5
+testing	result checksums	6
+Checksum: 4204762635;
+Verify single massive result row
+Checksum: 2267222966;
+Verify multiple massive rows
+Checksum: 2166497371;
+Test around 16MB packet boundaries
+Checksum: 2512256286;
+Checksum: 982468368;
+Checksum: 1437851346;
+Checksum: 2699967101;
+Checksum: 261522601;
+Checksum: 1879284586;
+Checksum: 2282180928;
+Several random row sizes
+SELECT REPEAT('a', 255);
+REPEAT('a', 255)
+aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+Checksum: 2275529303;
+SELECT REPEAT('a', 256);
+REPEAT('a', 256)
+aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+Checksum: 1932162976;
+SELECT REPEAT('a', 257);
+REPEAT('a', 257)
+aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+Checksum: 2889758595;
+verify SHOW SLAVE HOSTS output
+SHOW REPLICAS;
+Server_Id	Host	Port	Source_Id	Replica_UUID	Is_semi_sync_slave	Replication_status
+Checksum: 3702298611;
+Verify SHOW TABLES output
+SHOW TABLES;
+Tables_in_test
+data
+Checksum: 3251735689;
+Verify SHOW GRANTS output
+SHOW GRANTS;
+Grants for root@localhost
+GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, SHUTDOWN, PROCESS, FILE, REFERENCES, INDEX, ALTER, SHOW DATABASES, SUPER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER, CREATE TABLESPACE, CREATE ROLE, DROP ROLE ON *.* TO `root`@`localhost` WITH GRANT OPTION
+GRANT APPLICATION_PASSWORD_ADMIN,AUDIT_ABORT_EXEMPT,AUDIT_ADMIN,AUTHENTICATION_POLICY_ADMIN,BACKUP_ADMIN,BINLOG_ADMIN,BINLOG_ENCRYPTION_ADMIN,CLONE_ADMIN,CONNECTION_ADMIN,ENCRYPTION_KEY_ADMIN,FLUSH_OPTIMIZER_COSTS,FLUSH_STATUS,FLUSH_TABLES,FLUSH_USER_RESOURCES,GROUP_REPLICATION_ADMIN,GROUP_REPLICATION_STREAM,INNODB_REDO_LOG_ARCHIVE,INNODB_REDO_LOG_ENABLE,PASSWORDLESS_USER_ADMIN,PERSIST_RO_VARIABLES_ADMIN,REPLICATION_APPLIER,REPLICATION_SLAVE_ADMIN,RESOURCE_GROUP_ADMIN,RESOURCE_GROUP_USER,ROLE_ADMIN,SERVICE_CONNECTION_ADMIN,SESSION_VARIABLES_ADMIN,SET_USER_ID,SHOW_ROUTINE,SYSTEM_USER,SYSTEM_VARIABLES_ADMIN,TABLE_ENCRYPTION_ADMIN,XA_RECOVER_ADMIN ON *.* TO `root`@`localhost` WITH GRANT OPTION
+GRANT PROXY ON ``@`` TO `root`@`localhost` WITH GRANT OPTION
+Checksum: 428322074;
+Verify CHECK TABLE output
+CHECK TABLE data;
+Table	Op	Msg_type	Msg_text
+test.data	check	status	OK
+Checksum: 2020821564;
+SET @@global.enable_resultset_checksum = default;
+DROP TABLE data;

mysql-test/suite/sys_vars/r/enable_resultset_checksum_basic.result

@@ -0,0 +1,15 @@
+Default value of enable_resultset_checksum is false
+SELECT @@global.enable_resultset_checksum;
+@@global.enable_resultset_checksum
+0
+SELECT @@session.enable_resultset_checksum;
+ERROR HY000: Variable 'enable_resultset_checksum' is a GLOBAL variable
+Expected error 'Variable is a GLOBAL variable'
+SET @@global.enable_resultset_checksum = true;
+SELECT @@global.enable_resultset_checksum;
+@@global.enable_resultset_checksum
+1
+SET @@global.enable_resultset_checksum = default;
+SELECT @@global.enable_resultset_checksum;
+@@global.enable_resultset_checksum
+0

mysql-test/suite/sys_vars/t/enable_resultset_checksum_basic.test

@@ -0,0 +1,23 @@
+-- source include/load_sysvars.inc
+
+####
+# Verify default value false
+####
+--echo Default value of enable_resultset_checksum is false
+SELECT @@global.enable_resultset_checksum;
+
+####
+# Verify that this is not a session variable
+####
+--Error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SELECT @@session.enable_resultset_checksum;
+--echo Expected error 'Variable is a GLOBAL variable'
+
+####
+## Actual tests which enables this are in different file which test this feature
+####
+SET @@global.enable_resultset_checksum = true;
+SELECT @@global.enable_resultset_checksum;
+
+SET @@global.enable_resultset_checksum = default;
+SELECT @@global.enable_resultset_checksum;

mysql-test/t/query_checksum.test

@@ -1,37 +1,44 @@
 set @@global.enable_query_checksum = TRUE;
 
 # Add a checksum that fails
-query_attrs_add query_checksum 0;
+query_attrs_add checksum 0;
 delimiter ||||;
 --error ER_QUERY_CHECKSUM_FAILED
 SELECT 1;SELECT 2;||||
 delimiter ;||||
 
 # Add a checksum that succeeds
 query_attrs_reset;
-query_attrs_add query_checksum 2629868284;
+query_attrs_add checksum 2629868284;
 delimiter ||||;
 SELECT 1;SELECT 2;||||
 delimiter ;||||
 
 # Add a checksum that succeeds, on a single query
 query_attrs_reset;
-query_attrs_add query_checksum 0078787420;
+query_attrs_add checksum 0078787420;
 delimiter ||||;
 SELECT 1;||||
 delimiter ;||||
 
 # Add a checksum that passes, with leading comment and trailing ;
 query_attrs_reset;
-query_attrs_add query_checksum 3498200060;
+query_attrs_add checksum 3498200060;
+delimiter ||||;
+/* */ SELECT 1;;||||
+delimiter ;||||
+
+# Check that the automatic checksumming works as expected
+query_attrs_reset;
+query_attrs_add checksum ON;
 delimiter ||||;
 /* */ SELECT 1;;||||
 delimiter ;||||
 
 # Add a checksum that fails, but the global variable is off
 query_attrs_reset;
 set @@global.enable_query_checksum = FALSE;
-query_attrs_add query_checksum 0;
+query_attrs_add checksum 0;
 delimiter ||||;
 SELECT 1;SELECT 2;||||
 delimiter ;||||