Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

planner: move more methods from StatsHandle to its sub-packages (#47749) #47763

Closed
wants to merge 1 commit into from
Closed

planner: move more methods from StatsHandle to its sub-packages (#47749) #47763

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pkg/executor/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ go_library(
"//pkg/statistics",
"//pkg/statistics/handle",
"//pkg/statistics/handle/cache",
"//pkg/statistics/handle/globalstats",
"//pkg/statistics/handle/util",
"//pkg/store/driver/backoff",
"//pkg/store/driver/txn",
Expand Down
3 changes: 1 addition & 2 deletions pkg/executor/analyze_col.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -280,8 +280,7 @@ func (e *AnalyzeColumnsExec) buildStats(ranges []*ranger.Range, needExtStats boo
fms = append(fms, collectors[i].FMSketch)
}
if needExtStats {
statsHandle := domain.GetDomain(e.ctx).StatsHandle()
extStats, err = statsHandle.BuildExtendedStats(e.TableID.GetStatisticsID(), e.colsInfo, collectors)
extStats, err = statistics.BuildExtendedStats(e.ctx, e.TableID.GetStatisticsID(), e.colsInfo, collectors)
if err != nil {
return nil, nil, nil, nil, nil, err
}
Expand Down
3 changes: 1 addition & 2 deletions pkg/executor/analyze_col_v2.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -465,8 +465,7 @@ func (e *AnalyzeColumnsExecV2) buildSamplingStats(

count = rootRowCollector.Base().Count
if needExtStats {
statsHandle := domain.GetDomain(e.ctx).StatsHandle()
extStats, err = statsHandle.BuildExtendedStats(e.TableID.GetStatisticsID(), e.colsInfo, sampleCollectors)
extStats, err = statistics.BuildExtendedStats(e.ctx, e.TableID.GetStatisticsID(), e.colsInfo, sampleCollectors)
if err != nil {
return 0, nil, nil, nil, nil, err
}
Expand Down
4 changes: 3 additions & 1 deletion pkg/executor/analyze_global_stats.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/statistics"
"github.com/pingcap/tidb/pkg/statistics/handle/globalstats"
"github.com/pingcap/tidb/pkg/statistics/handle/util"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/logutil"
Expand Down Expand Up @@ -84,7 +85,7 @@ func (e *AnalyzeExec) handleGlobalStats(ctx context.Context, globalStatsMap glob
cache = nil
}

globalStats, err := statsHandle.MergePartitionStats2GlobalStatsByTableID(
globalStatsI, err := statsHandle.MergePartitionStats2GlobalStatsByTableID(
e.Ctx(),
globalOpts, e.Ctx().GetInfoSchema().(infoschema.InfoSchema),
globalStatsID.tableID,
Expand All @@ -101,6 +102,7 @@ func (e *AnalyzeExec) handleGlobalStats(ctx context.Context, globalStatsMap glob
}
return err
}
globalStats := globalStatsI.(*globalstats.GlobalStats)
// Dump global-level stats to kv.
for i := 0; i < globalStats.Num; i++ {
hg, cms, topN := globalStats.Hg[i], globalStats.Cms[i], globalStats.TopN[i]
Expand Down
1 change: 1 addition & 0 deletions pkg/statistics/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ go_library(
"analyze.go",
"analyze_jobs.go",
"builder.go",
"builder_ext_stats.go",
"cmsketch.go",
"cmsketch_util.go",
"column.go",
Expand Down
39 changes: 26 additions & 13 deletions ...tistics/handle/extstats/extended_stats.go → pkg/statistics/builder_ext_stats.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,38 +12,46 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package extstats
package statistics

import (
"context"
"encoding/json"

"github.com/pingcap/errors"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/statistics"
"github.com/pingcap/tidb/pkg/statistics/handle/util"
"github.com/pingcap/tidb/pkg/util/logutil"
<<<<<<< HEAD:pkg/statistics/handle/extstats/extended_stats.go
"github.com/pingcap/tidb/pkg/util/mathutil"
=======
"github.com/pingcap/tidb/pkg/util/sqlexec"
>>>>>>> ac1f0d92a6e (planner: move more methods from StatsHandle to its sub-packages (#47749)):pkg/statistics/builder_ext_stats.go
"go.uber.org/zap"
)

// BuildExtendedStats build extended stats for column groups if needed based on the column samples.
// TODO: move this function to statistics/builder.go.
func BuildExtendedStats(sctx sessionctx.Context,
tableID int64, cols []*model.ColumnInfo, collectors []*statistics.SampleCollector) (*statistics.ExtendedStatsColl, error) {
tableID int64, cols []*model.ColumnInfo, collectors []*SampleCollector) (*ExtendedStatsColl, error) {
const sql = "SELECT name, type, column_ids FROM mysql.stats_extended WHERE table_id = %? and status in (%?, %?)"
rows, _, err := util.ExecRows(sctx, sql, tableID, statistics.ExtendedStatsAnalyzed, statistics.ExtendedStatsInited)

sqlExec, ok := sctx.(sqlexec.RestrictedSQLExecutor)
if !ok {
return nil, errors.Errorf("invalid sql executor")
}
rows, _, err := sqlExec.ExecRestrictedSQL(kv.WithInternalSourceType(context.Background(), kv.InternalTxnStats), nil, sql, tableID, ExtendedStatsAnalyzed, ExtendedStatsInited)
if err != nil {
return nil, errors.Trace(err)
}
if len(rows) == 0 {
return nil, nil
}
statsColl := statistics.NewExtendedStatsColl()
statsColl := NewExtendedStatsColl()
for _, row := range rows {
name := row.GetString(0)
item := &statistics.ExtendedStatsItem{Tp: uint8(row.GetInt64(1))}
item := &ExtendedStatsItem{Tp: uint8(row.GetInt64(1))}
colIDs := row.GetString(2)
err := json.Unmarshal([]byte(colIDs), &item.ColIDs)
if err != nil {
Expand All @@ -61,7 +69,7 @@ func BuildExtendedStats(sctx sessionctx.Context,
return statsColl, nil
}

func fillExtendedStatsItemVals(sctx sessionctx.Context, item *statistics.ExtendedStatsItem, cols []*model.ColumnInfo, collectors []*statistics.SampleCollector) *statistics.ExtendedStatsItem {
func fillExtendedStatsItemVals(sctx sessionctx.Context, item *ExtendedStatsItem, cols []*model.ColumnInfo, collectors []*SampleCollector) *ExtendedStatsItem {
switch item.Tp {
case ast.StatsTypeCardinality, ast.StatsTypeDependency:
return nil
Expand All @@ -71,7 +79,7 @@ func fillExtendedStatsItemVals(sctx sessionctx.Context, item *statistics.Extende
return nil
}

func fillExtStatsCorrVals(sctx sessionctx.Context, item *statistics.ExtendedStatsItem, cols []*model.ColumnInfo, collectors []*statistics.SampleCollector) *statistics.ExtendedStatsItem {
func fillExtStatsCorrVals(sctx sessionctx.Context, item *ExtendedStatsItem, cols []*model.ColumnInfo, collectors []*SampleCollector) *ExtendedStatsItem {
colOffsets := make([]int, 0, 2)
for _, id := range item.ColIDs {
for i, col := range cols {
Expand All @@ -87,8 +95,13 @@ func fillExtStatsCorrVals(sctx sessionctx.Context, item *statistics.ExtendedStat
// samplesX and samplesY are in order of handle, i.e, their SampleItem.Ordinals are in order.
samplesX := collectors[colOffsets[0]].Samples
// We would modify Ordinal of samplesY, so we make a deep copy.
<<<<<<< HEAD:pkg/statistics/handle/extstats/extended_stats.go
samplesY := statistics.CopySampleItems(collectors[colOffsets[1]].Samples)
sampleNum := mathutil.Min(len(samplesX), len(samplesY))
=======
samplesY := CopySampleItems(collectors[colOffsets[1]].Samples)
sampleNum := min(len(samplesX), len(samplesY))
>>>>>>> ac1f0d92a6e (planner: move more methods from StatsHandle to its sub-packages (#47749)):pkg/statistics/builder_ext_stats.go
if sampleNum == 1 {
item.ScalarVals = 1
return item
Expand All @@ -101,11 +114,11 @@ func fillExtStatsCorrVals(sctx sessionctx.Context, item *statistics.ExtendedStat
sc := sctx.GetSessionVars().StmtCtx

var err error
samplesX, err = statistics.SortSampleItems(sc, samplesX)
samplesX, err = SortSampleItems(sc, samplesX)
if err != nil {
return nil
}
samplesYInXOrder := make([]*statistics.SampleItem, 0, sampleNum)
samplesYInXOrder := make([]*SampleItem, 0, sampleNum)
for i, itemX := range samplesX {
if itemX.Ordinal >= len(samplesY) {
continue
Expand All @@ -114,7 +127,7 @@ func fillExtStatsCorrVals(sctx sessionctx.Context, item *statistics.ExtendedStat
itemY.Ordinal = i
samplesYInXOrder = append(samplesYInXOrder, itemY)
}
samplesYInYOrder, err := statistics.SortSampleItems(sc, samplesYInXOrder)
samplesYInYOrder, err := SortSampleItems(sc, samplesYInXOrder)
if err != nil {
return nil
}
Expand Down
6 changes: 1 addition & 5 deletions pkg/statistics/handle/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ go_library(
"//pkg/infoschema",
"//pkg/kv",
"//pkg/metrics",
"//pkg/parser/ast",
"//pkg/parser/model",
"//pkg/parser/mysql",
"//pkg/parser/terror",
Expand All @@ -26,7 +25,6 @@ go_library(
"//pkg/statistics",
"//pkg/statistics/handle/autoanalyze",
"//pkg/statistics/handle/cache",
"//pkg/statistics/handle/extstats",
"//pkg/statistics/handle/globalstats",
"//pkg/statistics/handle/history",
"//pkg/statistics/handle/lockstats",
Expand All @@ -51,21 +49,19 @@ go_test(
timeout = "short",
srcs = [
"ddl_test.go",
"gc_test.go",
"handle_hist_test.go",
"main_test.go",
],
embed = [":handle"],
flaky = True,
race = "on",
shard_count = 13,
shard_count = 8,
deps = [
"//pkg/config",
"//pkg/parser/model",
"//pkg/planner/cardinality",
"//pkg/sessionctx",
"//pkg/sessionctx/stmtctx",
"//pkg/sessionctx/variable",
"//pkg/testkit",
"//pkg/testkit/testsetup",
"//pkg/types",
Expand Down
9 changes: 8 additions & 1 deletion pkg/statistics/handle/ddl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/tidb/pkg/ddl/util"
"github.com/pingcap/tidb/pkg/parser/model"
<<<<<<< HEAD
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/parser/terror"
"github.com/pingcap/tidb/pkg/sessionctx"
Expand All @@ -29,6 +30,9 @@ import (
statsutil "github.com/pingcap/tidb/pkg/statistics/handle/util"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/sqlexec"
=======
"github.com/pingcap/tidb/pkg/sessionctx/variable"
>>>>>>> ac1f0d92a6e (planner: move more methods from StatsHandle to its sub-packages (#47749))
)

// HandleDDLEvent begins to process a ddl task.
Expand Down Expand Up @@ -76,7 +80,7 @@ func (h *Handle) HandleDDLEvent(t *util.Event) error {
return err
}
if variable.PartitionPruneMode(pruneMode) == variable.Dynamic && t.PartInfo != nil {
if err := h.updateGlobalStats(t.TableInfo); err != nil {
if err := h.UpdateGlobalStats(t.TableInfo); err != nil {
return err
}
}
Expand Down Expand Up @@ -114,6 +118,7 @@ func (h *Handle) HandleDDLEvent(t *util.Event) error {
return nil
}

<<<<<<< HEAD
// updateStatsVersion will set statistics version to the newest TS,
// then tidb-server will reload automatic.
func (h *Handle) updateStatsVersion() error {
Expand Down Expand Up @@ -142,6 +147,8 @@ func (h *Handle) changeGlobalStatsID(from, to int64) (err error) {
}, statsutil.FlagWrapTxn)
}

=======
>>>>>>> ac1f0d92a6e (planner: move more methods from StatsHandle to its sub-packages (#47749))
func (h *Handle) getInitStateTableIDs(tblInfo *model.TableInfo) (ids []int64, err error) {
pi := tblInfo.GetPartitionInfo()
if pi == nil {
Expand Down
1 change: 0 additions & 1 deletion pkg/statistics/handle/globalstats/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ go_library(
"//pkg/statistics",
"//pkg/statistics/handle/storage",
"//pkg/statistics/handle/util",
"//pkg/table",
"//pkg/types",
"//pkg/util/hack",
"//pkg/util/logutil",
Expand Down
Loading