Removing unchanged columns from AI when when row image is not FULL

Summary:
Some update and upsert (insert ... on duplicate key update) queries
used to log columns which have not changed in the after image (when using RBR)
even when the image format is not FULL.

Test Plan: Extended rpl_row_img_sanity to check these special cases

Reviewers: santoshb

Reviewed By: santoshb

Subscribers: webscalesql-eng

Differential Revision: https://reviews.facebook.net/D65133

Author: abhinav04sharma

mysql-test/suite/rpl/r/rpl_row_img_sanity.result

@@ -448,6 +448,9 @@ include/rpl_reset.inc
 include/sync_slave_sql_with_master.inc
 ####### MINIMAL OTHER PARTICULAR SCENARIO ######
 include/sync_slave_sql_with_master.inc
+####### MINIMAL AND COMPLETE PARTICULAR SCENARIO SETTING SAME VALUE ######
+include/sync_slave_sql_with_master.inc
+include/sync_slave_sql_with_master.inc
 CON: 'master', IMG: 'NOBLOB', RESTART SLAVE: 'N'
 Variable_name	Value
 binlog_row_image	NOBLOB
@@ -489,6 +492,9 @@ include/rpl_reset.inc
 include/sync_slave_sql_with_master.inc
 include/sync_slave_sql_with_master.inc
 include/rpl_reset.inc
+####### MINIMAL AND COMPLETE PARTICULAR SCENARIO SETTING SAME VALUE ######
+include/sync_slave_sql_with_master.inc
+include/sync_slave_sql_with_master.inc
 ################## SPECIAL CASES #########################
 include/rpl_reset.inc
 CON: 'master', IMG: 'NOBLOB', RESTART SLAVE: 'N'

mysql-test/suite/rpl/t/rpl_row_img_sanity.test

@@ -333,7 +333,7 @@ while (`SELECT HEX(@img_types) != HEX('')`)
     #      1. columns that are set to default value are in AI
     #
     #    UPDATE/MINIMAL:
-    #      2. columns that are set to the same value are in AI
+    #      2. columns that are set to the same value are not in AI
     #      3. columns that are not in WHERE clause but in PKE are in BI
     #      4. on slave, columns that are not in master's BI but are in slave's
     #         PKE are in slave's BI
@@ -359,11 +359,11 @@ while (`SELECT HEX(@img_types) != HEX('')`)
     -- source include/rpl_row_img_parts_master_slave.inc
 
     -- let $row_img_query= UPDATE t SET c2=100, c3=1000 WHERE c2=100;
-    #   2. columns that are set to the same value are in AI
+    #   2. columns that are set to the same value are not in AI
     #   3. columns that are not in WHERE clause but in PKE are in BI
     #   4. on slave, columns that are not in master's BI but are in slave's PKE are in slave's BI
-    -- let $row_img_expected_master= 1:1 | 2:100 3:1000
-    -- let $row_img_expected_slave = 2:100 3:1 | 2:100 3:1000
+    -- let $row_img_expected_master= 1:1 | 3:1000
+    -- let $row_img_expected_slave = 2:100 3:1 | 3:1000
     -- source include/rpl_row_img_parts_master_slave.inc
 
     -- let $row_img_query= DELETE FROM t WHERE c1=1
@@ -410,6 +410,54 @@ while (`SELECT HEX(@img_types) != HEX('')`)
     --source include/sync_slave_sql_with_master.inc
   }
 
+  if (`SELECT @@binlog_row_image = "MINIMAL" OR (SELECT @@binlog_row_image = "COMPLETE")`)
+  {
+    -- echo ####### MINIMAL AND COMPLETE PARTICULAR SCENARIO SETTING SAME VALUE ######
+  
+    # ASSERTIONS: 
+    #    1. columns that are set to the same value are not in AI
+    #    2. columns that are set to the same value by INSERT ... DUPLICATE KEY
+    #       are not in AI
+    #
+
+    -- connection master
+    CREATE TABLE t (c1 int, c2 int, c3 blob, primary key(c1));
+    INSERT INTO t VALUES (1,2,"a");
+
+
+    --source include/sync_slave_sql_with_master.inc
+    -- connection master
+
+    # Issue some statements
+    -- let $row_img_query= UPDATE t SET c1 = 1, c2 = 4 WHERE c3 = "a";
+    if (`SELECT @@binlog_row_image = "MINIMAL"`)
+    {
+      -- let $row_img_expected_master= 1:1 | 2:4
+    }
+    if (`SELECT @@binlog_row_image = "COMPLETE"`)
+    {
+      -- let $row_img_expected_master= 1:1 2:2 3:'a' | 2:4
+    }
+    -- let $row_img_expected_slave = $row_img_expected_master
+    -- source include/rpl_row_img_parts_master_slave.inc
+
+    -- let $row_img_query= INSERT INTO t VALUES (1, 4, "a") ON DUPLICATE KEY UPDATE c2 = c2 + 2, c3 = values(c3);
+    if (`SELECT @@binlog_row_image = "MINIMAL"`)
+    {
+      -- let $row_img_expected_master= 1:1 | 2:6
+    }
+    if (`SELECT @@binlog_row_image = "COMPLETE"`)
+    {
+      -- let $row_img_expected_master= 1:1 2:4 3:'a' | 2:6
+    }
+    -- let $row_img_expected_slave = $row_img_expected_master
+    -- source include/rpl_row_img_parts_master_slave.inc
+
+    -- connection master
+    DROP TABLE t;
+    --source include/sync_slave_sql_with_master.inc
+  }
+
   if (`SELECT @@binlog_row_image = "NOBLOB"`)
   {
     -- echo ####### NOBLOB PARTICULAR SCENARIO ######
@@ -419,19 +467,17 @@ while (`SELECT HEX(@img_types) != HEX('')`)
     #      1 non-blob columns that are set to default value are in AI
     #    
     #    UPDATE/NOBLOB:
-    #      2 non-blob columns that are set to default value are in AI
-    #      3 blob columns that are set to default value are in AI
-    #      4 blob columns that are in WHERE clause but not in PKE are not in BI
-    #      5 blob columns that are in NON-NULL UK but not in PK are not in BI
-    #      6 on slave, blob columns that are in master's BI clause but not in
+    #      2 blob columns that are in WHERE clause but not in PKE are not in BI
+    #      3 blob columns that are in NON-NULL UK but not in PK are not in BI
+    #      4 on slave, blob columns that are in master's BI clause but not in
     #        slave's PKE are not in BI
     #
     #    DELETE
-    #      7 non-blob columns that are not in WHERE clause but in PKE are in BI
-    #      8 blob columns that are used in WHERE clause but not in PKE are not
+    #      5 non-blob columns that are not in WHERE clause but in PKE are in BI
+    #      6 blob columns that are used in WHERE clause but not in PKE are not
     #        in BI
-    #      9 blob columns that are NOT NULL UK but not in PK are not in BI
-    #     10 on slave, blob columns that are in master's BI but not in slave's PKE
+    #      7 blob columns that are NOT NULL UK but not in PK are not in BI
+    #      8 on slave, blob columns that are in master's BI but not in slave's PKE
     #        are not in slave's BI
 
     -- connection master
@@ -454,23 +500,21 @@ while (`SELECT HEX(@img_types) != HEX('')`)
     -- connection master
     -- let $row_img_query= UPDATE t SET c2='aaa', c3=1000, c5=10000 WHERE c1=1 AND c4='bbb'
     # asserts that:
-    #  2. non-blob columns that are set to default value are in AI
-    #  3. blob columns that are set to default value are in AI
-    #  4. blob columns that are in WHERE clause but not in PKE are not in BI
-    #  5. blob columns that are in NON-NULL UK but not in PK are not in BI
-    #  6. on slave, blob columns that are in master's BI clause but not in
+    #  2. blob columns that are in WHERE clause but not in PKE are not in BI
+    #  3. blob columns that are in NON-NULL UK but not in PK are not in BI
+    #  4. on slave, blob columns that are in master's BI clause but not in
     #     slave's PKE are not in BI
-    -- let $row_img_expected_master= 1:1 2:'aaa' 3:1000 5:1 | 1:1 2:'aaa' 3:1000 5:10000
-    -- let $row_img_expected_slave = 1:1 3:1000 5:1 | 1:1 2:'aaa' 3:1000 5:10000
+    -- let $row_img_expected_master= 1:1 2:'aaa' 3:1000 5:1 | 1:1 3:1000 5:10000
+    -- let $row_img_expected_slave = 1:1 3:1000 5:1 | 1:1 3:1000 5:10000
     -- source include/rpl_row_img_parts_master_slave.inc
 
     -- connection master
     -- let $row_img_query= DELETE FROM t WHERE c1=1 AND c4='bbb'
     # asserts that:
-    #   7. non-blob columns that are not in WHERE clause but in PKE are in BI
-    #   8. blob columns that are used in WHERE clause but not in PKE are not n BI
-    #   9. blob columns that are NOT NULL UK but not in PK are not in BI
-    #  10. on slave, blob columns that are in master's BI but not in slave's PKE are not in slave's BI
+    #   5. non-blob columns that are not in WHERE clause but in PKE are in BI
+    #   6. blob columns that are used in WHERE clause but not in PKE are not n BI
+    #   7. blob columns that are NOT NULL UK but not in PK are not in BI
+    #   8. on slave, blob columns that are in master's BI but not in slave's PKE are not in slave's BI
     -- let $row_img_expected_master= 1:1 2:'aaa' 3:1000 5:10000 | 
     -- let $row_img_expected_slave = 1:1 3:1000 5:10000 | 
     -- source include/rpl_row_img_parts_master_slave.inc

sql/binlog.cc

@@ -9158,7 +9158,7 @@ int THD::binlog_update_row(TABLE* table, bool is_trans,
      not needed for binlogging. This is done according to the:
      binlog-row-image option.
    */
-  binlog_prepare_row_images(table);
+  binlog_prepare_row_images(table, true);
 
   size_t const before_maxlen = max_row_length(table, before_record);
   size_t const after_maxlen  = max_row_length(table, after_record);
@@ -9224,7 +9224,7 @@ int THD::binlog_delete_row(TABLE* table, bool is_trans,
      not needed for binlogging. This is done according to the:
      binlog-row-image option.
    */
-  binlog_prepare_row_images(table);
+  binlog_prepare_row_images(table, false);
 
   /* 
      Pack records into format for transfer. We are allocating more
@@ -9256,17 +9256,18 @@ int THD::binlog_delete_row(TABLE* table, bool is_trans,
   return error;
 }
 
-void THD::binlog_prepare_row_images(TABLE *table) 
+void THD::binlog_prepare_row_images(TABLE *table, bool is_update)
 {
   DBUG_ENTER("THD::binlog_prepare_row_images");
   /** 
-    Remove from read_set spurious columns. The write_set has been
+    Remove spurious columns. The write_set has been partially
     handled before in table->mark_columns_needed_for_update. 
    */
 
   DBUG_PRINT_BITSET("debug", "table->read_set (before preparing): %s", table->read_set);
   THD *thd= table->in_use;
 
+  /* Handle the read set */
   /** 
     if there is a primary key in the table (ie, user declared PK or a
     non-null unique index) and we dont want to ship the entire image,
@@ -9314,6 +9315,44 @@ void THD::binlog_prepare_row_images(TABLE *table)
                                         table->write_set);
   }
 
+  /* Now, handle the write set */
+  if (is_update &&
+      thd->variables.binlog_row_image != BINLOG_ROW_IMAGE_FULL &&
+      !ha_check_storage_engine_flag(table->s->db_type(),
+                                    HTON_NO_BINLOG_ROW_OPT))
+  {
+    /**
+      Just to be sure that tmp_write_set is currently not in use as
+      the write_set already.
+    */
+    DBUG_ASSERT(table->write_set != &table->tmp_write_set);
+
+    bitmap_copy(&table->tmp_write_set, table->write_set);
+
+    for (Field **ptr=table->field ; *ptr ; ptr++)
+    {
+      Field *field= (*ptr);
+      if (bitmap_is_set(&table->tmp_write_set, field->field_index))
+      {
+        /* When image type is NOBLOB, we prune only BLOB fields */
+        if (thd->variables.binlog_row_image == BINLOG_ROW_IMAGE_NOBLOB &&
+            field->type() != MYSQL_TYPE_BLOB)
+          continue;
+
+        /* compare null bit */
+        if (field->is_null() && field->is_null_in_record(table->record[1]))
+            bitmap_clear_bit(&table->tmp_write_set, field->field_index);
+
+        /* compare content, only if fields are not set to NULL */
+        else if (!field->is_null() &&
+                 !field->cmp_binary_offset(table->s->rec_buff_length))
+          bitmap_clear_bit(&table->tmp_write_set, field->field_index);
+      }
+    }
+    table->column_bitmaps_set_no_signal(table->read_set,
+                                        &table->tmp_write_set);
+  }
+
   DBUG_PRINT_BITSET("debug", "table->read_set (after preparing): %s", table->read_set);
   DBUG_VOID_RETURN;
 }

sql/sql_class.h

@@ -2623,7 +2623,7 @@ class THD :public MDL_context_owner,
   int binlog_update_row(TABLE* table, bool is_transactional,
                         const uchar *old_data, const uchar *new_data,
                         const uchar* extra_row_info);
-  void binlog_prepare_row_images(TABLE* table);
+  void binlog_prepare_row_images(TABLE* table, bool is_update);
 
   void set_server_id(uint32 sid) { server_id = sid; }
 

sql/table.cc

@@ -2657,14 +2657,16 @@ int open_table_from_share(THD *thd, TABLE_SHARE *share, const char *alias,
   /* Allocate bitmaps */
 
   bitmap_size= share->column_bitmap_size;
-  if (!(bitmaps= (uchar*) alloc_root(&outparam->mem_root, bitmap_size*3)))
+  if (!(bitmaps= (uchar*) alloc_root(&outparam->mem_root, bitmap_size*4)))
     goto err;
   bitmap_init(&outparam->def_read_set,
               (my_bitmap_map*) bitmaps, share->fields, FALSE);
   bitmap_init(&outparam->def_write_set,
               (my_bitmap_map*) (bitmaps+bitmap_size), share->fields, FALSE);
   bitmap_init(&outparam->tmp_set,
               (my_bitmap_map*) (bitmaps+bitmap_size*2), share->fields, FALSE);
+  bitmap_init(&outparam->tmp_write_set,
+              (my_bitmap_map*) (bitmaps+bitmap_size*3), share->fields, FALSE);
   outparam->default_column_bitmaps();
 
   /* The table struct is now initialized;  Open the table */

sql/table.h

@@ -1170,7 +1170,8 @@ struct TABLE
   const char	*alias;            	  /* alias or table name */
   uchar		*null_flags;
   my_bitmap_map	*bitmap_init_value;
-  MY_BITMAP     def_read_set, def_write_set, tmp_set; /* containers */
+  MY_BITMAP     def_read_set, def_write_set; /* containers */
+  MY_BITMAP     tmp_set, tmp_write_set; /* containers */
   MY_BITMAP     *read_set, *write_set;          /* Active column sets */
   /*
    The ID of the query that opened and is using this table. Has different