From 6ed5e77ef1f0603fbcba59a3771e999a28954aa2 Mon Sep 17 00:00:00 2001 From: fengou1 Date: Mon, 12 Dec 2022 13:17:03 +0530 Subject: [PATCH] fix: restore failure on batch create table when size of tables schema large than 6 mb --- br/pkg/gluetidb/glue.go | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/br/pkg/gluetidb/glue.go b/br/pkg/gluetidb/glue.go index 3b1675921afd3..2c8726310153b 100644 --- a/br/pkg/gluetidb/glue.go +++ b/br/pkg/gluetidb/glue.go @@ -203,13 +203,33 @@ func (gs *tidbSession) CreatePlacementPolicy(ctx context.Context, policy *model. return d.CreatePlacementPolicyWithInfo(gs.se, policy, ddl.OnExistIgnore) } +// SplitBatchCreateTable provide a way to split batch into small batch when batch size is large than 6 MB. +// The raft entry has limit size of 6 MB, a batch of CreateTables may hit this limitation +// TODO: shall query string be set for each split batch create, it looks does not matter if we set once for all. +func (gs *tidbSession) SplitBatchCreateTable(schema model.CIStr, info []*model.TableInfo, l int, r int, cs ...ddl.CreateTableWithInfoConfigurier) error { + var err error + d := domain.GetDomain(gs.se).DDL() + if err = d.BatchCreateTableWithInfo(gs.se, schema, info[l:r], append(cs, ddl.OnExistIgnore)...); kv.ErrEntryTooLarge.Equal(err) { + if r-l == 1 { + return err + } + err = gs.SplitBatchCreateTable(schema, info, l, (l+r)/2) + if err != nil { + return err + } + err = gs.SplitBatchCreateTable(schema, info, (l+r)/2, r) + if err != nil { + return err + } + return nil + } + return err +} + // CreateTables implements glue.BatchCreateTableSession. func (gs *tidbSession) CreateTables(ctx context.Context, tables map[string][]*model.TableInfo, cs ...ddl.CreateTableWithInfoConfigurier) error { - d := domain.GetDomain(gs.se).DDL() var dbName model.CIStr - // Disable foreign key check when batch create tables. - gs.se.GetSessionVars().ForeignKeyChecks = false for db, tablesInDB := range tables { dbName = model.NewCIStr(db) queryBuilder := strings.Builder{} @@ -233,8 +253,7 @@ func (gs *tidbSession) CreateTables(ctx context.Context, tables map[string][]*mo cloneTables = append(cloneTables, table) } gs.se.SetValue(sessionctx.QueryString, queryBuilder.String()) - err := d.BatchCreateTableWithInfo(gs.se, dbName, cloneTables, append(cs, ddl.OnExistIgnore)...) - if err != nil { + if err := gs.SplitBatchCreateTable(dbName, cloneTables, 0, len(cloneTables)); err != nil { //It is possible to failure when TiDB does not support model.ActionCreateTables. //In this circumstance, BatchCreateTableWithInfo returns errno.ErrInvalidDDLJob, //we fall back to old way that creating table one by one