MDEV-32294 2nd execution problem with inconsistent outer context paths

WORK IN PROGRESS

context {select_lex, outer_context} elements made private, accessor
function will now skip merged selects.

SELECT_LEX::resolved_here contains a list of Item_fields that is resolved
in that select.  It is used to calculate new_used_tables_cache in
Item_subselect::recalc_used_tables.  We can compare new/old calculations
with (in the correct directory and with the trace file enabled)
grep used_tables_cache mysqld.trace

It currently works, except when it doesn't, i.e.

  select * from t1
    where t1c in
    (
      select * from
      (
        select t2a from (select * from t2) table2
          where t2a in
          (
            select t3a from t3
              where t3a in
              (
                 select t4a from t4
                   where t4a < any(select t5c from t5 where t5a > t2a)
              )
          )
      ) dt
    );

resolved_here isn't being correctly set in select #1 after merges.
Working on it.

sql/item.cc

@@ -5358,8 +5358,8 @@ void mark_select_range_as_dependent(THD *thd, SELECT_LEX *last_select,
     resolving)
   */
   SELECT_LEX *previous_select= current_sel;
-  for (; previous_select->context.outer_select() != last_select;
-       previous_select= previous_select->context.outer_select())
+  for (; previous_select->outer_select() != last_select;
+       previous_select= previous_select->outer_select())
   {
     Item_subselect *prev_subselect_item=
       previous_select->master_unit()->item;
@@ -5732,6 +5732,7 @@ Item_field::fix_outer_field(THD *thd, Field **from_field, Item **reference)
   SELECT_LEX *current_sel= thd->lex->current_select;
   Name_resolution_context *outer_context= 0;
   SELECT_LEX *select= 0;
+
   /* Currently derived tables cannot be correlated */
   if ((current_sel->master_unit()->first_select()->get_linkage() !=
        DERIVED_TABLE_TYPE) &&
@@ -5809,7 +5810,28 @@ Item_field::fix_outer_field(THD *thd, Field **from_field, Item **reference)
         }
         if (*from_field != view_ref_found)
         {
+          /*
+            Populate the resolved_here list in statement memory
+            only on first execution
+          */
+          if (thd->stmt_arena->is_stmt_prepare_or_first_stmt_execute() ||
+                  thd->stmt_arena->is_conventional())
+          {
+            Query_arena *arena, backup;
+            arena= thd->activate_stmt_arena_if_needed(&backup);
+
+            if (!select->resolved_here)
+            {
+              if (!(select->resolved_here= new (thd->mem_root)List<Item_field>))
+                return -1;
+            }
+            select->resolved_here->push_back( this, thd->mem_root);
+            if (arena)
+              thd->restore_active_arena(arena, &backup);
+          }
+
           prev_subselect_item->used_tables_cache|= (*from_field)->table->map;
+          prev_subselect_item->new_used_tables_cache|= (*from_field)->table->map;
           prev_subselect_item->const_item_cache= 0;
           set_field(*from_field);
           if (!last_checked_context->get_select_lex()->having_fix_field &&
@@ -5904,6 +5926,7 @@ Item_field::fix_outer_field(THD *thd, Field **from_field, Item **reference)
       case it does not matter which used tables bits we set)
     */
     prev_subselect_item->used_tables_cache|= OUTER_REF_TABLE_BIT;
+    prev_subselect_item->new_used_tables_cache|= OUTER_REF_TABLE_BIT;
     prev_subselect_item->const_item_cache= 0;
   }
 
@@ -8230,6 +8253,7 @@ bool Item_ref::fix_fields(THD *thd, Item **reference)
                    cached_table->select_lex != outer_context->get_select_lex());
             }
             prev_subselect_item->used_tables_cache|= from_field->table->map;
+            prev_subselect_item->new_used_tables_cache|= from_field->table->map;
             prev_subselect_item->const_item_cache= 0;
             break;
           }
@@ -8238,6 +8262,7 @@ bool Item_ref::fix_fields(THD *thd, Item **reference)
 
         /* Reference is not found => depend on outer (or just error). */
         prev_subselect_item->used_tables_cache|= OUTER_REF_TABLE_BIT;
+        prev_subselect_item->new_used_tables_cache|= OUTER_REF_TABLE_BIT;
         prev_subselect_item->const_item_cache= 0;
 
         outer_context= outer_context->get_outer_context();

sql/item.h

@@ -2204,6 +2204,8 @@ class Item: public Value_source,
     If there is some, sets a bit for this key in the proper key map.
   */
   virtual bool check_index_dependence(void *arg) { return 0; }
+  virtual bool update_resolved_here_processor(void *arg) { return 0; }
+  virtual bool clear_used_tables_bit_processor(void *arg) { return 0; }
   /*============== End of Item processor list ======================*/
 
   /*
@@ -5286,24 +5288,29 @@ class Used_tables_and_const_cache
     (even internally in Item_func_* code).
   */
   table_map used_tables_cache;
+  table_map new_used_tables_cache;
   bool const_item_cache;
 
   Used_tables_and_const_cache()
    :used_tables_cache(0),
+    new_used_tables_cache(0),
     const_item_cache(true)
   { }
   Used_tables_and_const_cache(const Used_tables_and_const_cache *other)
    :used_tables_cache(other->used_tables_cache),
+    new_used_tables_cache(other->new_used_tables_cache),
     const_item_cache(other->const_item_cache)
   { }
   void used_tables_and_const_cache_init()
   {
     used_tables_cache= 0;
+    new_used_tables_cache= 0;
     const_item_cache= true;
   }
   void used_tables_and_const_cache_join(const Item *item)
   {
     used_tables_cache|= item->used_tables();
+    new_used_tables_cache|= item->used_tables();
     const_item_cache&= item->const_item();
   }
   void used_tables_and_const_cache_update_and_join(Item *item)

sql/item_cmpfunc.h

@@ -2644,12 +2644,14 @@ class Item_func_isnull :public Item_func_null_predicate
     if (!args[0]->maybe_null && !arg_is_datetime_notnull_field())
     {
       used_tables_cache= 0;			/* is always false */
+      new_used_tables_cache= 0;			/* is always false */
       const_item_cache= 1;
     }
     else
     {
       args[0]->update_used_tables();
       used_tables_cache= args[0]->used_tables();
+      new_used_tables_cache= args[0]->used_tables();
       const_item_cache= args[0]->const_item();
     }
   }

sql/item_subselect.cc

@@ -344,7 +344,10 @@ bool Item_subselect::fix_fields(THD *thd_param, Item **ref)
   {
     const_item_cache= 0;
     if (uncacheable & UNCACHEABLE_RAND)
+    {
       used_tables_cache|= RAND_TABLE_BIT;
+      new_used_tables_cache|= RAND_TABLE_BIT;
+    }
   }
   fixed= 1;
 
@@ -553,7 +556,26 @@ void Item_subselect::recalc_used_tables(st_select_lex *new_parent,
     he has done const table detection, and that will be our chance to update
     const_tables_cache.
   */
-  DBUG_PRINT("exit", ("used_tables_cache: %llx", used_tables_cache));
+  DBUG_PRINT("info", ("used_tables_cache: %llx", used_tables_cache));
+
+  // New implementation (MDEV-32294)
+  new_used_tables_cache= (min_resolved_nest_level < new_parent->nest_level) ?
+                            OUTER_REF_TABLE_BIT : 0;
+
+  if (new_parent->resolved_here)
+  {
+    List_iterator<Item_field>  it(*new_parent->resolved_here);
+    Item_field *item;
+    while ((item= it++))
+    {
+      if (item->field->table->pos_in_table_list->select_lex ==
+          this->unit->first_select()->context.outer_select())
+        this->new_used_tables_cache |= item->field->table->map;
+    }
+  }
+
+  DBUG_PRINT("info", ("new_used_tables_cache: %llx", new_used_tables_cache));
+
   DBUG_VOID_RETURN;
 }
 
@@ -1060,6 +1082,54 @@ void Item_subselect::print(String *str, enum_query_type query_type)
 }
 
 
+/*
+  Update Item_subselect::used_tables_cache attached to this SELECT_LEX (if any)
+
+  SYNOPSIS
+    update_resolved_items_used_tables()
+
+  DESCRIPTION
+*/
+
+bool Item_subselect::update_resolved_here_processor( void *arg )
+{
+  List<Item_field> *refs = (List<Item_field> *)arg;
+  DBUG_ENTER("Item_subselect::update_resolved_items_used_tables");
+  if (!refs ||
+      !refs->elements)
+    DBUG_RETURN(FALSE);
+
+  List_iterator_fast<Item_field>  it( *refs );
+  Item_field *item;
+
+  // for each item resolved in this SELECT_LEX
+  while ((item= it++))
+    set_used_tables( item->field->table->map );
+  DBUG_PRINT("info", ("%s", dbug_print_item(this)));
+
+  DBUG_PRINT("info", ("new used_tables_cache: %llx",
+        new_get_used_tables_cache()));
+
+  DBUG_PRINT("info", ("old used_tables_cache: %llx",
+        get_used_tables_cache()));
+
+  DBUG_RETURN(FALSE);
+}
+
+bool Item_subselect::clear_used_tables_bit_processor( void *arg )
+{
+  table_map bits_to_clear= (table_map)arg;
+  clear_used_tables(bits_to_clear);
+  DBUG_PRINT("info", ("%s", dbug_print_item(this)));
+
+  DBUG_PRINT("info", ("new used_tables_cache: %llx",
+        new_get_used_tables_cache()));
+
+  DBUG_PRINT("info", ("old used_tables_cache: %llx",
+        get_used_tables_cache()));
+  return FALSE;
+}
+
 Item_singlerow_subselect::Item_singlerow_subselect(THD *thd, st_select_lex *select_lex):
   Item_subselect(thd), value(0)
 {
@@ -3539,6 +3609,7 @@ void Item_in_subselect::fix_after_pullout(st_select_lex *new_parent,
   left_expr->fix_after_pullout(new_parent, &left_expr, merge);
   Item_subselect::fix_after_pullout(new_parent, ref, merge);
   used_tables_cache |= left_expr->used_tables();
+  new_used_tables_cache |= left_expr->used_tables();
 }
 
 void Item_in_subselect::update_used_tables()
@@ -3547,6 +3618,7 @@ void Item_in_subselect::update_used_tables()
   left_expr->update_used_tables();
   //used_tables_cache |= left_expr->used_tables();
   used_tables_cache= Item_subselect::used_tables() | left_expr->used_tables();
+  new_used_tables_cache= Item_subselect::used_tables() | left_expr->used_tables();
 }
 
 

sql/item_subselect.h

@@ -178,6 +178,7 @@ class Item_subselect :public Item_result_field,
   virtual void reset()
   {
     eliminated= FALSE;
+    min_resolved_nest_level= INT_MAX;
     null_value= 1;
   }
   /**
@@ -210,14 +211,20 @@ class Item_subselect :public Item_result_field,
   void make_const()
   { 
     used_tables_cache= 0;
+    new_used_tables_cache= 0;
     const_item_cache= 0;
     forced_const= TRUE; 
   }
+  void set_used_tables(table_map newmap) { new_used_tables_cache|= newmap; }
+  void clear_used_tables(table_map newmap) { new_used_tables_cache&= ~newmap; }
+  void update_resolved_items_used_tables( List<Item_field> *refs );
   virtual bool fix_length_and_dec();
   table_map used_tables() const override;
   table_map not_null_tables() const override { return 0; }
   bool const_item() const override;
   inline table_map get_used_tables_cache() { return used_tables_cache; }
+  inline table_map new_get_used_tables_cache() { return new_used_tables_cache; }
+  int min_resolved_nest_level;  // ML
   Item *get_tmp_table_item(THD *thd) override;
   void update_used_tables() override;
   void print(String *str, enum_query_type query_type) override;
@@ -252,6 +259,8 @@ class Item_subselect :public Item_result_field,
   {
     return mark_unsupported_function("select ...", arg, VCOL_IMPOSSIBLE);
   }
+  bool update_resolved_here_processor(void *arg) override;
+  bool clear_used_tables_bit_processor(void *arg) override;
   /**
     Callback to test if an IN predicate is expensive.
 
@@ -286,6 +295,8 @@ class Item_subselect :public Item_result_field,
 
   st_select_lex *wrap_tvc_into_select(THD *thd, st_select_lex *tvc_sl);
 
+  bool update_resolved_items_used_tables();
+
   friend class select_result_interceptor;
   friend class Item_in_optimizer;
   friend bool Item_field::fix_fields(THD *, Item **);
@@ -410,6 +421,7 @@ class Item_exists_subselect :public Item_subselect
   void reset() override
   {
     eliminated= FALSE;
+    min_resolved_nest_level= INT_MAX;
     value= 0;
   }
   void no_rows_in_result() override;
@@ -641,6 +653,7 @@ class Item_in_subselect :public Item_exists_subselect
   void reset() override
   {
     eliminated= FALSE;
+    min_resolved_nest_level= INT_MAX;
     value= 0;
     null_value= 0;
     was_null= 0;

sql/item_sum.h

@@ -496,6 +496,7 @@ class Item_sum :public Item_func_or_sum
   void make_const () 
   { 
     used_tables_cache= 0;
+    new_used_tables_cache= 0;
     const_item_cache= true;
   }
   void reset_forced_const() { const_item_cache= false; }

sql/opt_subselect.cc

@@ -1596,6 +1596,24 @@ static void reset_equality_number_for_subq_conds(Item * cond)
   return;
 }
 
+
+/**
+  Remove Item_field elements of list that have been resolved to
+  remove_me
+*/
+
+void remove_references_to(List<Item_field> *list, SELECT_LEX *remove_me)
+{
+  List_iterator<Item_field>  it( *list );
+  Item_field *item;
+
+  while ((item= it++))
+  {
+    if (item->context->get_select_lex() == remove_me)
+      it.remove();
+  }
+}
+
 /*
   Convert a subquery predicate into a TABLE_LIST semi-join nest
 
@@ -1639,6 +1657,9 @@ static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred)
   uint ncols;
   DBUG_ENTER("convert_subq_to_sj");
 
+  DBUG_PRINT("info", ("parent_lex select #%d, subq_pred select #%d",
+          parent_lex->select_number,
+          subq_pred->unit->first_select()->select_number) );
   /*
     1. Find out where to put the predicate into.
      Note: for "t1 LEFT JOIN t2" this will be t2, a leaf.
@@ -1947,6 +1968,26 @@ static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred)
     item_eq->in_equality_no= 0;
     sj_nest->sj_on_expr= and_items(thd, sj_nest->sj_on_expr, item_eq);
   }
+
+  if (subq_lex->resolved_here)
+  {
+    DBUG_PRINT("info", ("shifting resolved_here from select #%d to #%d",
+          subq_lex->select_number,
+          parent_lex->select_number) );
+    if (!parent_lex->resolved_here)
+      parent_lex->resolved_here= subq_lex->resolved_here;
+    else
+    {
+      /*
+        References in parent_lex->resolved_here that are resolved in
+        subq_lex need to be removed as they are no longer outer references
+      */
+      remove_references_to(parent_lex->resolved_here, subq_lex);
+      parent_lex->resolved_here->append(subq_lex->resolved_here);
+      subq_lex->resolved_here= nullptr;
+    }
+  }
+
   /*
     Fix the created equality and AND