Skip to content

Commit

Permalink
This is an automated cherry-pick of pingcap#57144
Browse files Browse the repository at this point in the history
Signed-off-by: ti-chi-bot <ti-community-prow-bot@tidb.io>
  • Loading branch information
hawkingrei authored and ti-chi-bot committed Nov 22, 2024
1 parent 719f68e commit bc7ffdc
Show file tree
Hide file tree
Showing 3 changed files with 116 additions and 0 deletions.
46 changes: 46 additions & 0 deletions pkg/statistics/handle/handle_hist.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/sessionctx/stmtctx"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/statistics"
"github.com/pingcap/tidb/pkg/statistics/handle/storage"
utilstats "github.com/pingcap/tidb/pkg/statistics/handle/util"
Expand Down Expand Up @@ -280,6 +281,19 @@ func (h *Handle) handleOneItemTask(task *NeededItemTask) (err error) {
h.SPool().Put(se)
}
}()
<<<<<<< HEAD:pkg/statistics/handle/handle_hist.go
=======
var skipTypes map[string]struct{}
val, err := sctx.GetSessionVars().GlobalVarsAccessor.GetGlobalSysVar(variable.TiDBAnalyzeSkipColumnTypes)
if err != nil {
logutil.BgLogger().Warn("failed to get global variable", zap.Error(err))
} else {
skipTypes = variable.ParseAnalyzeSkipColumnTypes(val)
}

item := task.Item.TableItemID
tbl, ok := s.statsHandle.Get(item.TableID)
>>>>>>> bfec7325a12 (statsitstics: avoid sync load column which is skiped type to analyze (#57144)):pkg/statistics/handle/syncload/stats_syncload.go

item := task.TableItemID
tbl, ok := h.Get(item.TableID)
Expand All @@ -294,12 +308,44 @@ func (h *Handle) handleOneItemTask(task *NeededItemTask) (err error) {
}
wrapper.idx = index
} else {
<<<<<<< HEAD:pkg/statistics/handle/handle_hist.go
col, ok := tbl.Columns[item.ID]
if !ok || col.IsFullLoad() {
=======
col, loadNeeded, analyzed := tbl.ColumnIsLoadNeeded(item.ID, task.Item.FullLoad)
if !loadNeeded {
return nil
}
if col != nil {
wrapper.colInfo = col.Info
} else {
// Now, we cannot init the column info in the ColAndIdxExistenceMap when to disable lite-init-stats.
// so we have to get the column info from the domain.
wrapper.colInfo = tblInfo.Meta().GetColumnByID(item.ID)
}
if skipTypes != nil {
_, skip := skipTypes[types.TypeToStr(wrapper.colInfo.FieldType.GetType(), wrapper.colInfo.FieldType.GetCharset())]
if skip {
return nil
}
}

// If this column is not analyzed yet and we don't have it in memory.
// We create a fake one for the pseudo estimation.
if loadNeeded && !analyzed {
wrapper.col = &statistics.Column{
PhysicalID: item.TableID,
Info: wrapper.colInfo,
Histogram: *statistics.NewHistogram(item.ID, 0, 0, 0, &wrapper.colInfo.FieldType, 0, 0),
IsHandle: isPkIsHandle && mysql.HasPriKeyFlag(wrapper.colInfo.GetFlag()),
}
s.updateCachedItem(tblInfo, item, wrapper.col, wrapper.idx, task.Item.FullLoad)
>>>>>>> bfec7325a12 (statsitstics: avoid sync load column which is skiped type to analyze (#57144)):pkg/statistics/handle/syncload/stats_syncload.go
return nil
}
wrapper.col = col
}
failpoint.Inject("handleOneItemTaskPanic", nil)
t := time.Now()
needUpdate := false
wrapper, err = h.readStatsForOneItem(sctx, item, wrapper)
Expand Down
16 changes: 16 additions & 0 deletions pkg/statistics/handle/handle_hist_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,22 @@ func TestSyncLoadSkipUnAnalyzedItems(t *testing.T) {
failpoint.Disable("github.com/pingcap/tidb/pkg/statistics/handle/assertSyncLoadItems")
}

func TestSyncLoadSkipAnalyzSkipColumnItems(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
tk.MustExec("create table t(`id` bigint(20) NOT NULL AUTO_INCREMENT,content text,PRIMARY KEY (`id`))")
h := dom.StatsHandle()
h.SetLease(1)

tk.MustExec("analyze table t")
tk.MustExec("set @@session.tidb_analyze_skip_column_types = 'json, text, blob'") // text is not default.
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/statistics/handle/syncload/handleOneItemTaskPanic", `panic`))
tk.MustQuery("trace plan select * from t where content ='ab'")
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/statistics/handle/syncload/handleOneItemTaskPanic"))
}

func TestConcurrentLoadHist(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)

Expand Down
54 changes: 54 additions & 0 deletions pkg/statistics/handle/syncload/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")

go_library(
name = "syncload",
srcs = ["stats_syncload.go"],
importpath = "github.com/pingcap/tidb/pkg/statistics/handle/syncload",
visibility = ["//visibility:public"],
deps = [
"//pkg/config",
"//pkg/infoschema",
"//pkg/kv",
"//pkg/meta/model",
"//pkg/metrics",
"//pkg/parser/mysql",
"//pkg/sessionctx",
"//pkg/sessionctx/stmtctx",
"//pkg/sessionctx/variable",
"//pkg/statistics",
"//pkg/statistics/handle/storage",
"//pkg/statistics/handle/types",
"//pkg/table",
"//pkg/types",
"//pkg/util",
"//pkg/util/intest",
"//pkg/util/logutil",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
"@org_golang_x_sync//singleflight",
"@org_uber_go_zap//:zap",
],
)

go_test(
name = "syncload_test",
timeout = "short",
srcs = ["stats_syncload_test.go"],
flaky = True,
race = "on",
shard_count = 6,
deps = [
":syncload",
"//pkg/config",
"//pkg/meta/model",
"//pkg/parser/model",
"//pkg/sessionctx",
"//pkg/sessionctx/stmtctx",
"//pkg/statistics/handle/types",
"//pkg/testkit",
"//pkg/testkit/analyzehelper",
"//pkg/util/mathutil",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_stretchr_testify//require",
],
)

0 comments on commit bc7ffdc

Please sign in to comment.