Bug#1634932/83648: Assertion failure in thread x in file fts0que.cc

The bug was caused by three problems:
1) query->intersection was not freed in case of error caused by
exceeding innodb_ft_result_cache_limit.
2) errors from init_ftfuncs were not propagated - this was fixed
in 5.7 by bug fix 21140111. This was ported into 5.6
3) bug fix 21140111 was causing assertion failure when
innodb_ft_result_cache was exceeded in DELETE command. This was
also fixed.

mysql-test/suite/innodb_fts/r/bug83648.result

@@ -0,0 +1,37 @@
+#
+# Bug 1634932: Assertion failure in thread x in
+# file fts0que.cc
+#
+SET @saved_innodb_ft_result_cache_limit= @@innodb_ft_result_cache_limit;
+CREATE TABLE `t1` (
+`FTS_DOC_ID` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+`text_content` MEDIUMTEXT, PRIMARY KEY (`FTS_DOC_ID`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+CREATE UNIQUE INDEX FTS_DOC_ID_INDEX ON t1(FTS_DOC_ID);
+SET autocommit=0;
+CREATE PROCEDURE populate_t1()
+BEGIN
+DECLARE i INT DEFAULT 1;
+WHILE (i <= 250) DO
+INSERT INTO t1 (text_content) VALUES ("some_text_1234 aaa");
+SET i = i + 1;
+END WHILE;
+END//
+CALL populate_t1;
+SET autocommit=1;
+SET SESSION debug="+d,fts_instrument_result_cache_limit";
+ALTER TABLE t1 ADD FULLTEXT INDEX `text_content_idx` (`text_content`);
+SELECT FTS_DOC_ID, text_content
+FROM t1
+WHERE MATCH text_content AGAINST ('+some_text_1234' IN BOOLEAN MODE);
+ERROR HY000: FTS query exceeds result cache limit
+UPDATE t1
+SET text_content='some_text_12345'
+where MATCH text_content AGAINST ('+some_text_1234' IN BOOLEAN MODE);
+ERROR HY000: FTS query exceeds result cache limit
+DELETE FROM t1
+WHERE MATCH text_content AGAINST ('+some_text_1234' IN BOOLEAN MODE);
+ERROR HY000: FTS query exceeds result cache limit
+SET GLOBAL innodb_ft_result_cache_limit = @saved_innodb_ft_result_cache_limit;
+DROP TABLE t1;
+DROP PROCEDURE populate_t1;

mysql-test/suite/innodb_fts/t/bug83648.test

@@ -0,0 +1,57 @@
+--echo #
+--echo # Bug 1634932: Assertion failure in thread x in
+--echo # file fts0que.cc
+--echo #
+
+--source include/have_innodb.inc
+--source include/have_debug.inc
+
+SET @saved_innodb_ft_result_cache_limit= @@innodb_ft_result_cache_limit;
+
+CREATE TABLE `t1` (
+	`FTS_DOC_ID` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+	`text_content` MEDIUMTEXT, PRIMARY KEY (`FTS_DOC_ID`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+CREATE UNIQUE INDEX FTS_DOC_ID_INDEX ON t1(FTS_DOC_ID);
+
+SET autocommit=0;
+
+DELIMITER //;
+CREATE PROCEDURE populate_t1()
+BEGIN
+        DECLARE i INT DEFAULT 1;
+        WHILE (i <= 250) DO
+          INSERT INTO t1 (text_content) VALUES ("some_text_1234 aaa");
+          SET i = i + 1;
+        END WHILE;
+END//
+
+DELIMITER ;//
+
+CALL populate_t1;
+SET autocommit=1;
+
+SET SESSION debug="+d,fts_instrument_result_cache_limit";
+
+ALTER TABLE t1 ADD FULLTEXT INDEX `text_content_idx` (`text_content`);
+
+# HA_ERR_FTS_EXCEED_RESULT_CACHE_LIMIT = 188
+--error 188
+SELECT FTS_DOC_ID, text_content
+FROM t1
+WHERE MATCH text_content AGAINST ('+some_text_1234' IN BOOLEAN MODE);
+
+--error 188
+UPDATE t1
+SET text_content='some_text_12345'
+where MATCH text_content AGAINST ('+some_text_1234' IN BOOLEAN MODE);
+
+--error 188
+DELETE FROM t1
+WHERE MATCH text_content AGAINST ('+some_text_1234' IN BOOLEAN MODE);
+
+SET GLOBAL innodb_ft_result_cache_limit = @saved_innodb_ft_result_cache_limit;
+
+DROP TABLE t1;
+DROP PROCEDURE populate_t1;

sql/sql_delete.cc

@@ -322,12 +322,16 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, Item *conds,
     table->file->print_error(error, MYF(0));
     goto exit_without_my_ok;
   }
+
+  if (select_lex->has_ft_funcs() && init_ftfuncs(thd, select_lex, 1))
+    goto exit_without_my_ok;
+
   if (usable_index==MAX_KEY || (select && select->quick))
     error= init_read_record(&info, thd, table, select, 1, 1, FALSE);
   else
     error= init_read_record_idx(&info, thd, table, 1, usable_index, reverse);
 
-  if (error || (select_lex->has_ft_funcs() && init_ftfuncs(thd, select_lex, 1)))
+  if (error)
     goto exit_without_my_ok;
 
   THD_STAGE_INFO(thd, stage_updating);

sql/sql_executor.cc

@@ -116,7 +116,7 @@ JOIN::exec()
   if (prepare_result(&columns_list))
     DBUG_VOID_RETURN;
 
-  if (select_lex->materialized_table_count)
+  if (select_lex->materialized_table_count && select_lex->has_ft_funcs())
     init_ftfuncs(thd, select_lex, order);
 
   if (!tables_list && (tables || !select_lex->with_sum_func))

storage/innobase/fts/fts0que.cc

@@ -953,6 +953,18 @@ fts_query_free_doc_ids(
 	query->total_size -= SIZEOF_RBT_CREATE;
 }
 
+/**
+Free the query intersection
+@param[in] query        query instance */
+static
+void
+fts_query_free_intersection(
+	fts_query_t*	query)
+{
+        fts_query_free_doc_ids(query, query->intersection);
+        query->intersection = NULL;
+}
+
 /*******************************************************************//**
 Add the word to the documents "list" of matching words from
 the query. We make a copy of the word from the query heap. */
@@ -1311,6 +1323,7 @@ fts_query_intersect(
 		/* error is passed by 'query->error' */
 		if (query->error != DB_SUCCESS) {
 			ut_ad(query->error == DB_FTS_EXCEED_RESULT_CACHE_LIMIT);
+			fts_query_free_intersection(query);
 			return(query->error);
 		}
 
@@ -1339,7 +1352,9 @@ fts_query_intersect(
 
 			ut_a(!query->multi_exist || (query->multi_exist
 			     && rbt_size(query->doc_ids) <= n_doc_ids));
-		}
+		} else if (query->intersection != NULL) {
+            fts_query_free_intersection(query);
+        }
 	}
 
 	return(query->error);
@@ -1557,6 +1572,11 @@ fts_merge_doc_ids(
 				query, ranking->doc_id, ranking->rank);
 
 		if (query->error != DB_SUCCESS) {
+			if (query->intersection != NULL)
+			{
+				ut_a(query->oper == FTS_EXIST);
+				fts_query_free_intersection(query);
+			}
 			DBUG_RETURN(query->error);
 		}