Skip to content

Commit

Permalink
statistics: move JSON* to statistics/util (#57950)
Browse files Browse the repository at this point in the history 
ref #57949
  • Loading branch information
@winoros
winoros authored Dec 4, 2024
1 parent b4c719a commit da965d1
Show file tree
Hide file tree
Showing 35 changed files with 203 additions and 163 deletions.
2 changes: 1 addition & 1 deletion br/pkg/backup/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ go_library(
"//pkg/meta",
"//pkg/meta/model",
"//pkg/statistics/handle",
"//pkg/statistics/handle/util",
"//pkg/statistics/util",
"//pkg/util",
"//pkg/util/table-filter",
"@com_github_google_btree//:btree",
Expand Down
2 changes: 1 addition & 1 deletion br/pkg/backup/schema.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/meta/model"
"github.com/pingcap/tidb/pkg/statistics/handle"
"github.com/pingcap/tidb/pkg/statistics/handle/util"
"github.com/pingcap/tidb/pkg/statistics/util"
tidbutil "github.com/pingcap/tidb/pkg/util"
kvutil "github.com/tikv/client-go/v2/util"
"go.uber.org/zap"
Expand Down
4 changes: 2 additions & 2 deletions br/pkg/metautil/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ go_library(
"//pkg/meta/model",
"//pkg/statistics/handle",
"//pkg/statistics/handle/types",
"//pkg/statistics/handle/util",
"//pkg/statistics/util",
"//pkg/tablecodec",
"//pkg/util",
"//pkg/util/encrypt",
Expand Down Expand Up @@ -54,7 +54,7 @@ go_test(
"//pkg/meta/model",
"//pkg/parser/model",
"//pkg/statistics/handle/types",
"//pkg/statistics/handle/util",
"//pkg/statistics/util",
"//pkg/tablecodec",
"//pkg/testkit/testsetup",
"//pkg/util",
Expand Down
2 changes: 1 addition & 1 deletion br/pkg/metautil/load_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import (
"github.com/pingcap/tidb/br/pkg/storage"
"github.com/pingcap/tidb/pkg/meta/model"
pmodel "github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/statistics/handle/util"
"github.com/pingcap/tidb/pkg/statistics/util"
"github.com/pingcap/tidb/pkg/tablecodec"
"github.com/stretchr/testify/require"
)
Expand Down
2 changes: 1 addition & 1 deletion br/pkg/metautil/metafile.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import (
"github.com/pingcap/tidb/br/pkg/summary"
"github.com/pingcap/tidb/br/pkg/utils"
"github.com/pingcap/tidb/pkg/meta/model"
"github.com/pingcap/tidb/pkg/statistics/handle/util"
"github.com/pingcap/tidb/pkg/statistics/util"
"github.com/pingcap/tidb/pkg/tablecodec"
tidbutil "github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/encrypt"
Expand Down
2 changes: 1 addition & 1 deletion br/pkg/metautil/statsfile.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import (
"github.com/pingcap/tidb/pkg/meta/model"
"github.com/pingcap/tidb/pkg/statistics/handle"
statstypes "github.com/pingcap/tidb/pkg/statistics/handle/types"
statsutil "github.com/pingcap/tidb/pkg/statistics/handle/util"
statsutil "github.com/pingcap/tidb/pkg/statistics/util"
"github.com/pingcap/tidb/pkg/util"
"golang.org/x/sync/errgroup"
)
Expand Down
2 changes: 1 addition & 1 deletion br/pkg/metautil/statsfile_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import (
"github.com/pingcap/kvproto/pkg/encryptionpb"
"github.com/pingcap/tidb/br/pkg/storage"
"github.com/pingcap/tidb/pkg/statistics/handle/types"
"github.com/pingcap/tidb/pkg/statistics/handle/util"
"github.com/pingcap/tidb/pkg/statistics/util"
tidbutil "github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tipb/go-tipb"
"github.com/stretchr/testify/require"
Expand Down
2 changes: 1 addition & 1 deletion br/pkg/task/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ go_test(
"//pkg/meta/model",
"//pkg/parser/model",
"//pkg/parser/mysql",
"//pkg/statistics/handle/util",
"//pkg/statistics/util",
"//pkg/tablecodec",
"//pkg/testkit",
"//pkg/types",
Expand Down
2 changes: 1 addition & 1 deletion br/pkg/task/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import (
"github.com/pingcap/tidb/br/pkg/utils"
"github.com/pingcap/tidb/pkg/meta/model"
pmodel "github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/statistics/handle/util"
"github.com/pingcap/tidb/pkg/statistics/util"
"github.com/pingcap/tidb/pkg/tablecodec"
"github.com/stretchr/testify/require"
pd "github.com/tikv/pd/client"
Expand Down
2 changes: 1 addition & 1 deletion cmd/importer/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ go_library(
"//pkg/planner/core",
"//pkg/statistics",
"//pkg/statistics/handle/storage",
"//pkg/statistics/handle/util",
"//pkg/statistics/util",
"//pkg/types",
"@com_github_burntsushi_toml//:toml",
"@com_github_go_sql_driver_mysql//:mysql",
Expand Down
2 changes: 1 addition & 1 deletion cmd/importer/stats.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import (
"github.com/pingcap/tidb/pkg/meta/model"
stats "github.com/pingcap/tidb/pkg/statistics"
"github.com/pingcap/tidb/pkg/statistics/handle/storage"
"github.com/pingcap/tidb/pkg/statistics/handle/util"
"github.com/pingcap/tidb/pkg/statistics/util"
"github.com/pingcap/tidb/pkg/types"
"go.uber.org/zap"
)
Expand Down
1 change: 1 addition & 0 deletions pkg/domain/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ go_library(
"//pkg/statistics/handle/initstats",
"//pkg/statistics/handle/logutil",
"//pkg/statistics/handle/util",
"//pkg/statistics/util",
"//pkg/store",
"//pkg/store/helper",
"//pkg/ttl/ttlworker",
Expand Down
2 changes: 1 addition & 1 deletion pkg/domain/plan_replayer_dump.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ import (
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/statistics"
"github.com/pingcap/tidb/pkg/statistics/handle/util"
"github.com/pingcap/tidb/pkg/statistics/util"
"github.com/pingcap/tidb/pkg/util/chunk"
"github.com/pingcap/tidb/pkg/util/logutil"
"github.com/pingcap/tidb/pkg/util/printer"
Expand Down
3 changes: 2 additions & 1 deletion pkg/executor/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,7 @@ go_library(
"//pkg/statistics/handle/storage",
"//pkg/statistics/handle/types",
"//pkg/statistics/handle/util",
"//pkg/statistics/util",
"//pkg/store/driver/backoff",
"//pkg/store/driver/txn",
"//pkg/store/helper",
Expand Down Expand Up @@ -436,7 +437,7 @@ go_test(
"//pkg/sessiontxn/staleread",
"//pkg/statistics",
"//pkg/statistics/handle/storage",
"//pkg/statistics/handle/util",
"//pkg/statistics/util",
"//pkg/store/copr",
"//pkg/store/driver/error",
"//pkg/store/helper",
Expand Down
2 changes: 1 addition & 1 deletion pkg/executor/historical_stats_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/statistics/handle/storage"
"github.com/pingcap/tidb/pkg/statistics/handle/util"
"github.com/pingcap/tidb/pkg/statistics/util"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/stretchr/testify/require"
"github.com/tikv/client-go/v2/oracle"
Expand Down
2 changes: 1 addition & 1 deletion pkg/executor/load_stats.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import (
"github.com/pingcap/tidb/pkg/executor/internal/exec"
"github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/statistics/handle/util"
"github.com/pingcap/tidb/pkg/statistics/util"
"github.com/pingcap/tidb/pkg/util/chunk"
)

Expand Down
2 changes: 1 addition & 1 deletion pkg/executor/plan_replayer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import (
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/sessiontxn"
"github.com/pingcap/tidb/pkg/statistics/handle/util"
"github.com/pingcap/tidb/pkg/statistics/util"
"github.com/pingcap/tidb/pkg/util/chunk"
"github.com/pingcap/tidb/pkg/util/logutil"
"github.com/pingcap/tidb/pkg/util/replayer"
Expand Down
2 changes: 1 addition & 1 deletion pkg/planner/core/casetest/cbotest/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ go_test(
"//pkg/session",
"//pkg/sessionctx/variable",
"//pkg/statistics",
"//pkg/statistics/handle/util",
"//pkg/statistics/util",
"//pkg/testkit",
"//pkg/testkit/testdata",
"//pkg/testkit/testmain",
Expand Down
2 changes: 1 addition & 1 deletion pkg/planner/core/casetest/cbotest/cbo_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import (
"github.com/pingcap/tidb/pkg/session"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/statistics"
"github.com/pingcap/tidb/pkg/statistics/handle/util"
"github.com/pingcap/tidb/pkg/statistics/util"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/testkit/testdata"
"github.com/stretchr/testify/require"
Expand Down
4 changes: 2 additions & 2 deletions pkg/server/handler/optimizor/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ go_library(
"//pkg/server/handler",
"//pkg/sessionctx/variable",
"//pkg/statistics/handle",
"//pkg/statistics/handle/util",
"//pkg/statistics/util",
"//pkg/table",
"//pkg/types",
"//pkg/util",
Expand Down Expand Up @@ -57,7 +57,7 @@ go_test(
"//pkg/server/internal/util",
"//pkg/session",
"//pkg/statistics/handle/types",
"//pkg/statistics/handle/util",
"//pkg/statistics/util",
"//pkg/store/mockstore/unistore",
"//pkg/testkit",
"//pkg/testkit/testsetup",
Expand Down
2 changes: 1 addition & 1 deletion pkg/server/handler/optimizor/plan_replayer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ import (
pmodel "github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/server/handler"
"github.com/pingcap/tidb/pkg/statistics/handle"
util2 "github.com/pingcap/tidb/pkg/statistics/handle/util"
util2 "github.com/pingcap/tidb/pkg/statistics/util"
"github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/logutil"
"github.com/pingcap/tidb/pkg/util/replayer"
Expand Down
2 changes: 1 addition & 1 deletion pkg/server/handler/optimizor/plan_replayer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ import (
"github.com/pingcap/tidb/pkg/server/internal/testutil"
"github.com/pingcap/tidb/pkg/server/internal/util"
"github.com/pingcap/tidb/pkg/session"
util2 "github.com/pingcap/tidb/pkg/statistics/handle/util"
util2 "github.com/pingcap/tidb/pkg/statistics/util"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/util/replayer"
"github.com/stretchr/testify/require"
Expand Down
6 changes: 3 additions & 3 deletions pkg/server/handler/optimizor/statistics_handler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import (
"github.com/pingcap/tidb/pkg/server/internal/util"
"github.com/pingcap/tidb/pkg/session"
"github.com/pingcap/tidb/pkg/statistics/handle/types"
util2 "github.com/pingcap/tidb/pkg/statistics/handle/util"
statsutil "github.com/pingcap/tidb/pkg/statistics/util"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/stretchr/testify/require"
)
Expand Down Expand Up @@ -179,10 +179,10 @@ func testDumpPartitionTableStats(t *testing.T, client *testserverclient.TestServ
}()
b, err := io.ReadAll(resp0.Body)
require.NoError(t, err)
jsonTable := &util2.JSONTable{}
jsonTable := &statsutil.JSONTable{}
err = json.Unmarshal(b, jsonTable)
require.NoError(t, err)
require.NotNil(t, jsonTable.Partitions[util2.TiDBGlobalStats])
require.NotNil(t, jsonTable.Partitions[statsutil.TiDBGlobalStats])
require.Len(t, jsonTable.Partitions, expectedLen)
}
check(false)
Expand Down
1 change: 1 addition & 0 deletions pkg/statistics/handle/history/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ go_library(
"//pkg/statistics/handle/storage",
"//pkg/statistics/handle/types",
"//pkg/statistics/handle/util",
"//pkg/statistics/util",
"//pkg/util/logutil",
"@com_github_pingcap_errors//:errors",
"@org_uber_go_zap//:zap",
Expand Down
25 changes: 13 additions & 12 deletions pkg/statistics/handle/history/history_stats.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ import (
"github.com/pingcap/tidb/pkg/statistics/handle/cache"
"github.com/pingcap/tidb/pkg/statistics/handle/storage"
"github.com/pingcap/tidb/pkg/statistics/handle/types"
"github.com/pingcap/tidb/pkg/statistics/handle/util"
handleutil "github.com/pingcap/tidb/pkg/statistics/handle/util"
statsutil "github.com/pingcap/tidb/pkg/statistics/util"
"github.com/pingcap/tidb/pkg/util/logutil"
"go.uber.org/zap"
)
Expand All @@ -44,7 +45,7 @@ func NewStatsHistory(statsHandle types.StatsHandle,

// RecordHistoricalStatsToStorage records the given table's stats data to mysql.stats_history
func (sh *statsHistoryImpl) RecordHistoricalStatsToStorage(dbName string, tableInfo *model.TableInfo, physicalID int64, isPartition bool) (uint64, error) {
var js *util.JSONTable
var js *statsutil.JSONTable
var err error
if isPartition {
js, err = sh.statsHandle.TableStatsToJSON(dbName, tableInfo, physicalID, 0)
Expand All @@ -59,10 +60,10 @@ func (sh *statsHistoryImpl) RecordHistoricalStatsToStorage(dbName string, tableI
return 0, nil
}
var version uint64
err = util.CallWithSCtx(sh.statsHandle.SPool(), func(sctx sessionctx.Context) error {
err = handleutil.CallWithSCtx(sh.statsHandle.SPool(), func(sctx sessionctx.Context) error {
version, err = RecordHistoricalStatsToStorage(sctx, physicalID, js)
return err
}, util.FlagWrapTxn)
}, handleutil.FlagWrapTxn)
return version, err
}

Expand All @@ -80,12 +81,12 @@ func (sh *statsHistoryImpl) RecordHistoricalStatsMeta(tableID int64, version uin
return
}
}
err := util.CallWithSCtx(sh.statsHandle.SPool(), func(sctx sessionctx.Context) error {
err := handleutil.CallWithSCtx(sh.statsHandle.SPool(), func(sctx sessionctx.Context) error {
if !sctx.GetSessionVars().EnableHistoricalStats {
return nil
}
return RecordHistoricalStatsMeta(util.StatsCtx, sctx, tableID, version, source)
}, util.FlagWrapTxn)
return RecordHistoricalStatsMeta(handleutil.StatsCtx, sctx, tableID, version, source)
}, handleutil.FlagWrapTxn)
if err != nil { // just log the error, hide the error from the outside caller.
logutil.BgLogger().Error("record historical stats meta failed",
zap.Int64("table-id", tableID),
Expand All @@ -97,7 +98,7 @@ func (sh *statsHistoryImpl) RecordHistoricalStatsMeta(tableID int64, version uin

// CheckHistoricalStatsEnable checks whether historical stats is enabled.
func (sh *statsHistoryImpl) CheckHistoricalStatsEnable() (enable bool, err error) {
err = util.CallWithSCtx(sh.statsHandle.SPool(), func(sctx sessionctx.Context) error {
err = handleutil.CallWithSCtx(sh.statsHandle.SPool(), func(sctx sessionctx.Context) error {
enable = sctx.GetSessionVars().EnableHistoricalStats
return nil
})
Expand All @@ -115,7 +116,7 @@ func RecordHistoricalStatsMeta(
if tableID == 0 || version == 0 {
return errors.Errorf("tableID %d, version %d are invalid", tableID, version)
}
rows, _, err := util.ExecRowsWithCtx(
rows, _, err := handleutil.ExecRowsWithCtx(
ctx,
sctx,
"select modify_count, count from mysql.stats_meta where table_id = %? and version = %?",
Expand All @@ -131,7 +132,7 @@ func RecordHistoricalStatsMeta(
modifyCount, count := rows[0].GetInt64(0), rows[0].GetInt64(1)

const sql = "REPLACE INTO mysql.stats_meta_history(table_id, modify_count, count, version, source, create_time) VALUES (%?, %?, %?, %?, %?, NOW())"
if _, err := util.ExecWithCtx(
if _, err := handleutil.ExecWithCtx(
ctx,
sctx,
sql,
Expand All @@ -152,7 +153,7 @@ func RecordHistoricalStatsMeta(
const maxColumnSize = 5 << 20

// RecordHistoricalStatsToStorage records the given table's stats data to mysql.stats_history
func RecordHistoricalStatsToStorage(sctx sessionctx.Context, physicalID int64, js *util.JSONTable) (uint64, error) {
func RecordHistoricalStatsToStorage(sctx sessionctx.Context, physicalID int64, js *statsutil.JSONTable) (uint64, error) {
version := uint64(0)
if len(js.Partitions) == 0 {
version = js.Version
Expand All @@ -170,7 +171,7 @@ func RecordHistoricalStatsToStorage(sctx sessionctx.Context, physicalID int64, j
const sql = "INSERT INTO mysql.stats_history(table_id, stats_data, seq_no, version, create_time) VALUES (%?, %?, %?, %?, %?)" +
"ON DUPLICATE KEY UPDATE stats_data=%?, create_time=%?"
for i := 0; i < len(blocks); i++ {
if _, err = util.Exec(sctx, sql, physicalID, blocks[i], i, version, ts, blocks[i], ts); err != nil {
if _, err = handleutil.Exec(sctx, sql, physicalID, blocks[i], i, version, ts, blocks[i], ts); err != nil {
return 0, errors.Trace(err)
}
}
Expand Down
2 changes: 2 additions & 0 deletions pkg/statistics/handle/storage/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ go_library(
"//pkg/statistics/handle/types",
"//pkg/statistics/handle/usage/predicatecolumn",
"//pkg/statistics/handle/util",
"//pkg/statistics/util",
"//pkg/types",
"//pkg/util/chunk",
"//pkg/util/compress",
Expand Down Expand Up @@ -70,6 +71,7 @@ go_test(
"//pkg/statistics/handle/internal",
"//pkg/statistics/handle/types",
"//pkg/statistics/handle/util",
"//pkg/statistics/util",
"//pkg/testkit",
"//pkg/testkit/analyzehelper",
"//pkg/types",
Expand Down
Loading

0 comments on commit da965d1

Please sign in to comment.