bindinfo: sync concurrent ops on mysql.bind_info from multiple tidb instances

[eurekaka]

What problem does this PR solve?

Issue Number: close #21516

Problem Summary:

Inconsistency between binding cache and mysql.bind_info if there are concurrent CREATE BINDING / DROP BINDING on multiple tidb instances.

What is changed and how it works?

What's Changed:

• Physically remove old bindings in CREATE BINDING no matter if oldRecord from binding cache is empty.
• Combine removeBindRecord and appendBindRecord into a single setBindRecord.
• Insert a builtin row into mysql.bind_info to simulate table lock.
• Lock mysql.bind_info before manipulating records in that table by updating the specific builtin row. Since we have no gap lock in TiDB, and the LOCK TABLE mysql.bind_info WRITE is not generally available now, we use UPDATE on a specific row to simulate LOCK TABLE.
• Get the real update_time after acquiring the "simulated table lock", instead of the StartTS of the transaction.
• Prevent interruption of Update() during the interval of transaction commit and binding cache update by h.bindInfo.Lock().

How it Works:

The core idea of this PR is to make the operations on mysql.bind_info atomic, i.e, the steps should not be interrupted by another transaction from another tidb instance.

Related changes

• Need to cherry-pick to the release branch

Check List

Tests

• Integration test: UTF contains a test for this PR.
• Manual test: I started a tidb cluster with 2 tidb instances, and run the below test program for more than 2 hours, it didn't fail.

package main

import (
	"context"
	"database/sql"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	"strconv"
	"sync"
	"time"
)

var (
	dsns = []string{
		"root:@tcp(127.0.0.1:4000)/test?charset=utf8&parseTime=True",
		"root:@tcp(127.0.0.1:4001)/test?charset=utf8&parseTime=True",
	}
)

func main() {
	for {
		ctx, cancel := context.WithCancel(context.Background())
		wg := sync.WaitGroup{}
		for i := range dsns {
			wg.Add(1)
			go func(name string) {
				defer wg.Done()
				run(ctx, name, i)
			}(dsns[i])
		}

		time.Sleep(3 * time.Second)
		cancel()
		wg.Wait()
		for _, item := range dsns {
			check(item)
		}
		fmt.Println("success\n")
	}
}

func check(dsn string) {
	db, err := sql.Open("mysql", dsn)
	if err != nil {
		panic(err)
	}
	defer db.Close()

	c, err := db.Conn(context.Background())
	if err != nil {
		panic(err)
	}
	defer c.Close()

	if _, err = c.ExecContext(context.Background(), "admin flush bindings"); err != nil {
		panic(err)
	}

	rows, err := c.QueryContext(context.Background(), "select count(*) from mysql.bind_info where status != 'deleted' and status != 'builtin'")
	if err != nil {
		panic(err)
	}
	count := 0
	rows.Next()
	if err := rows.Scan(&count); err != nil {
		panic(err)
	}
	if err := rows.Err(); err != nil {
		panic(err)
	}
	rows.Close()

	rows, err = c.QueryContext(context.Background(), "show global bindings")
	if err != nil {
		panic(err)
	}
	n := 0
	for rows.Next() {
		n++
	}
	if err := rows.Err(); err != nil {
		panic(err)
	}

	if count != n {
		panic(fmt.Sprintf("unexpected count, %d, %d, %s\n", count, n, dsn))
	}
}

func run(ctx context.Context, dsn string, index int) {
	db, err := sql.Open("mysql", dsn)
	if err != nil {
		panic(err)
	}
	defer db.Close()

	wg := sync.WaitGroup{}
	for i := 0; i < 1; i++ {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()

			c, err := db.Conn(context.Background())
			if err != nil {
				panic(err)
			}
			defer c.Close()

			name := "t" + strconv.Itoa(index*1000+i)
			if _, err := c.ExecContext(context.Background(), fmt.Sprintf("create table if not exists %s(a int, b int, index idx(a))", name)); err != nil {
				panic(err)
			}

			for {

				ss := []string{
					fmt.Sprintf("create global binding for select * from %s using select * from %s;", name, name),
					fmt.Sprintf("drop global binding for select * from %s", name),
				}
				for _, s := range ss {
					select {
					case <-ctx.Done():
						return
					default:
						if _, err := c.ExecContext(context.Background(), s); err != nil {
							panic(err)
						}
					}
				}
			}

		}(i)
	}
	wg.Wait()
}

Side effects

• DBA must NOT delete the builtin row in mysql.bind_info, otherwise the locking mechanism would be broken.

Release note

• Synchronize concurrent operations on mysql.bind_info from multiple tidb instances

[zz-jason]

Breaking backward compatibility: the mysql.bind_info is redesigned, the upgrade utility brutally drops the previous table and creates a new table, i.e, the previous bindings would be lost if without backup.

@eurekaka This side effect is unacceptable. It brings a huge impact on the stability of TiDB applications which rely on SQL binding. Could you transport the old bindings to the new table?

[eurekaka]

@zz-jason

An alternative is that we can back up this table internally, i.e, save the old records before dropping mysql.bind_info, and insert them into the new table in this upgrade utility, but there may be potential errors if the previous original_sql is longer than VARCHAR(512), so I don't implement this logic, at least for now.

How to handle the possible insert error?

[eurekaka]

I re-implemented the locking mechanism by introducing a builtin row into mysql.bind_info(thanks for suggestion from @qw4990), PTAL.

[qw4990]

LGTM

[Reminiscent]

LGTM

[ti-srebot]

@Reminiscent, Thanks for your review. The bot only counts LGTMs from Reviewers and higher roles, but you're still welcome to leave your comments. See the corresponding SIG page for more information. Related SIG: planner(slack).

[Reminiscent]

@eurekaka Please help merge. Thanks!

[winoros]

lgtm

[winoros]

/merge

[ti-srebot]

/run-all-tests

[ti-srebot]

cherry pick to release-4.0 in PR #21868