Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

br: replace exp/slices with std #45986

Merged
merged 6 commits into from
Aug 11, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion br/pkg/lightning/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@ go_library(
"@com_github_prometheus_client_golang//prometheus/collectors",
"@com_github_prometheus_client_golang//prometheus/promhttp",
"@com_github_shurcool_httpgzip//:httpgzip",
"@org_golang_x_exp//slices",
"@org_uber_go_atomic//:atomic",
"@org_uber_go_zap//:zap",
"@org_uber_go_zap//zapcore",
Expand Down
1 change: 0 additions & 1 deletion br/pkg/lightning/backend/kv/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ go_library(
"//util/topsql/stmtstats",
"@com_github_docker_go_units//:go-units",
"@com_github_pingcap_errors//:errors",
"@org_golang_x_exp//slices",
"@org_uber_go_zap//:zap",
"@org_uber_go_zap//zapcore",
],
Expand Down
7 changes: 4 additions & 3 deletions br/pkg/lightning/backend/kv/sql2kv.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@
package kv

import (
"cmp"
"context"
"fmt"
"math"
"slices"

"github.com/pingcap/errors"
"github.com/pingcap/tidb/br/pkg/lightning/backend/encode"
Expand All @@ -35,7 +37,6 @@ import (
"github.com/pingcap/tidb/table"
"github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/types"
"golang.org/x/exp/slices"
)

type tableKVEncoder struct {
Expand Down Expand Up @@ -126,8 +127,8 @@ func CollectGeneratedColumns(se *Session, meta *model.TableInfo, cols []*table.C
}

// order the result by column offset so they match the evaluation order.
slices.SortFunc(genCols, func(i, j GeneratedCol) bool {
return cols[i.Index].Offset < cols[j.Index].Offset
slices.SortFunc(genCols, func(i, j GeneratedCol) int {
return cmp.Compare(cols[i.Index].Offset, cols[j.Index].Offset)
})
return genCols, nil
}
Expand Down
1 change: 0 additions & 1 deletion br/pkg/lightning/duplicate/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,5 @@ go_test(
"//br/pkg/lightning/log",
"//util/extsort",
"@com_github_stretchr_testify//require",
"@org_golang_x_exp//slices",
],
)
13 changes: 5 additions & 8 deletions br/pkg/lightning/duplicate/detector_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,14 @@ import (
"encoding/binary"
"errors"
"math/rand"
"slices"
"sync"
"testing"

"github.com/pingcap/tidb/br/pkg/lightning/duplicate"
"github.com/pingcap/tidb/br/pkg/lightning/log"
"github.com/pingcap/tidb/util/extsort"
"github.com/stretchr/testify/require"
"golang.org/x/exp/slices"
)

func TestDetector(t *testing.T) {
Expand Down Expand Up @@ -92,20 +92,17 @@ func TestDetector(t *testing.T) {
}

func verifyResults(t *testing.T, keys [][]byte, results []result) {
less := func(a, b []byte) bool {
return bytes.Compare(a, b) < 0
}
for _, r := range results {
require.GreaterOrEqual(t, len(r.keyIDs), 2, "keyIDs should have at least 2 elements")
require.True(t, slices.IsSortedFunc(r.keyIDs, less), "keyIDs should be sorted")
require.True(t, slices.IsSortedFunc(r.keyIDs, bytes.Compare), "keyIDs should be sorted")
}
slices.SortFunc(results, func(a, b result) bool {
return bytes.Compare(a.key, b.key) < 0
slices.SortFunc(results, func(a, b result) int {
return bytes.Compare(a.key, b.key)
})

sortedKeys := make([][]byte, len(keys))
copy(sortedKeys, keys)
slices.SortFunc(sortedKeys, less)
slices.SortFunc(sortedKeys, bytes.Compare)

for i := 0; i < len(sortedKeys); {
j := i + 1
Expand Down
2 changes: 1 addition & 1 deletion br/pkg/lightning/importer/dup_detect.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"context"
"fmt"
"io"
"slices"

"github.com/pingcap/errors"
"github.com/pingcap/tidb/br/pkg/lightning/backend/encode"
Expand All @@ -34,7 +35,6 @@ import (
"github.com/pingcap/tidb/util/codec"
"github.com/pingcap/tidb/util/extsort"
"go.uber.org/zap"
"golang.org/x/exp/slices"
"golang.org/x/sync/errgroup"
)

Expand Down
7 changes: 4 additions & 3 deletions br/pkg/lightning/importer/precheck_impl.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,12 @@
package importer

import (
"cmp"
"context"
"fmt"
"path/filepath"
"reflect"
"slices"
"strconv"
"strings"
"sync"
Expand Down Expand Up @@ -47,7 +49,6 @@ import (
"github.com/pingcap/tidb/util/set"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
"golang.org/x/exp/slices"
"golang.org/x/sync/errgroup"
"google.golang.org/grpc"
)
Expand Down Expand Up @@ -361,8 +362,8 @@ func (ci *regionDistributionCheckItem) Check(ctx context.Context) (*precheck.Che
if len(stores) <= 1 {
return theResult, nil
}
slices.SortFunc(stores, func(i, j *pdtypes.StoreInfo) bool {
return i.Status.RegionCount < j.Status.RegionCount
slices.SortFunc(stores, func(i, j *pdtypes.StoreInfo) int {
return cmp.Compare(i.Status.RegionCount, j.Status.RegionCount)
})
minStore := stores[0]
maxStore := stores[len(stores)-1]
Expand Down
7 changes: 4 additions & 3 deletions br/pkg/lightning/lightning.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package lightning

import (
"cmp"
"compress/gzip"
"context"
"crypto/ecdsa"
Expand All @@ -29,6 +30,7 @@ import (
"net/http"
"net/http/pprof"
"os"
"slices"
"strconv"
"strings"
"sync"
Expand Down Expand Up @@ -65,7 +67,6 @@ import (
"go.uber.org/atomic"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"golang.org/x/exp/slices"
)

// Lightning is the main struct of the lightning package.
Expand Down Expand Up @@ -960,8 +961,8 @@ func checkSystemRequirement(cfg *config.Config, dbsMeta []*mydump.MDDatabaseMeta
tableTotalSizes = append(tableTotalSizes, tb.TotalSize)
}
}
slices.SortFunc(tableTotalSizes, func(i, j int64) bool {
return i > j
slices.SortFunc(tableTotalSizes, func(i, j int64) int {
return cmp.Compare(j, i)
})
topNTotalSize := int64(0)
for i := 0; i < len(tableTotalSizes) && i < cfg.App.TableConcurrency; i++ {
Expand Down
7 changes: 4 additions & 3 deletions br/pkg/restore/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@
package restore

import (
"cmp"
"context"
"fmt"
"slices"
"sync"

"github.com/pingcap/errors"
Expand All @@ -21,7 +23,6 @@ import (
"github.com/pingcap/tidb/sessionctx/variable"
tidbutil "github.com/pingcap/tidb/util"
"go.uber.org/zap"
"golang.org/x/exp/slices"
)

// DB is a TiDB instance, not thread-safe.
Expand Down Expand Up @@ -403,8 +404,8 @@ func (db *DB) ensureTablePlacementPolicies(ctx context.Context, tableInfo *model
// FilterDDLJobs filters ddl jobs.
func FilterDDLJobs(allDDLJobs []*model.Job, tables []*metautil.Table) (ddlJobs []*model.Job) {
// Sort the ddl jobs by schema version in descending order.
slices.SortFunc(allDDLJobs, func(i, j *model.Job) bool {
return i.BinlogInfo.SchemaVersion > j.BinlogInfo.SchemaVersion
slices.SortFunc(allDDLJobs, func(i, j *model.Job) int {
return cmp.Compare(j.BinlogInfo.SchemaVersion, i.BinlogInfo.SchemaVersion)
})
dbs := getDatabases(tables)
for _, db := range dbs {
Expand Down
1 change: 0 additions & 1 deletion br/pkg/task/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,6 @@ go_library(
"@org_golang_google_grpc//:grpc",
"@org_golang_google_grpc//backoff",
"@org_golang_google_grpc//keepalive",
"@org_golang_x_exp//slices",
"@org_golang_x_sync//errgroup",
"@org_uber_go_multierr//:multierr",
"@org_uber_go_zap//:zap",
Expand Down
6 changes: 3 additions & 3 deletions br/pkg/task/stream.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"encoding/binary"
"fmt"
"net/http"
"slices"
"strings"
"sync"
"time"
Expand Down Expand Up @@ -58,7 +59,6 @@ import (
"github.com/tikv/client-go/v2/oracle"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
"golang.org/x/exp/slices"
)

const (
Expand Down Expand Up @@ -414,8 +414,8 @@ func (s *streamMgr) buildObserveRanges(ctx context.Context) ([]kv.KeyRange, erro

mRange := stream.BuildObserveMetaRange()
rs := append([]kv.KeyRange{*mRange}, dRanges...)
slices.SortFunc(rs, func(i, j kv.KeyRange) bool {
return bytes.Compare(i.StartKey, j.StartKey) < 0
slices.SortFunc(rs, func(i, j kv.KeyRange) int {
return bytes.Compare(i.StartKey, j.StartKey)
})

return rs, nil
Expand Down
1 change: 0 additions & 1 deletion br/pkg/trace/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ go_library(
"@com_github_pingcap_log//:log",
"@com_sourcegraph_sourcegraph_appdash//:appdash",
"@com_sourcegraph_sourcegraph_appdash//opentracing",
"@org_golang_x_exp//slices",
"@org_uber_go_zap//:zap",
],
)
Expand Down
6 changes: 3 additions & 3 deletions br/pkg/trace/tracing.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,14 @@ import (
"fmt"
"os"
"path/filepath"
"slices"
"text/tabwriter"
"time"

"github.com/cheynewallace/tabby"
"github.com/opentracing/opentracing-go"
"github.com/pingcap/log"
"go.uber.org/zap"
"golang.org/x/exp/slices"
"sourcegraph.com/sourcegraph/appdash"
traceImpl "sourcegraph.com/sourcegraph/appdash/opentracing"
)
Expand Down Expand Up @@ -88,15 +88,15 @@ func dfsTree(t *appdash.Trace, prefix string, isLast bool, tub *tabby.Tabby) {
tub.AddLine(prefix+suffix+t.Span.Name(), start.Format("15:04:05.000000"), duration.String())

// Sort events by their start time
slices.SortFunc(t.Sub, func(i, j *appdash.Trace) bool {
slices.SortFunc(t.Sub, func(i, j *appdash.Trace) int {
var istart, jstart time.Time
if ievent, err := i.TimespanEvent(); err == nil {
istart = ievent.Start()
}
if jevent, err := j.TimespanEvent(); err == nil {
jstart = jevent.Start()
}
return istart.Before(jstart)
return istart.Compare(jstart)
})

for i, sp := range t.Sub {
Expand Down
1 change: 0 additions & 1 deletion executor/infoschema_reader_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@ import (
)

func TestInspectionTables(t *testing.T) {
t.Skip("unstable, skip it and fix it before 20210624")
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
instances := []string{
Expand Down
4 changes: 1 addition & 3 deletions executor/merge_join_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -281,9 +281,7 @@ func TestShuffleMergeJoinInDisk(t *testing.T) {
}

func TestMergeJoinInDisk(t *testing.T) {
t.Skip("unstable, skip it and fix it before 20210618")
restore := config.RestoreFunc()
defer restore()
defer config.RestoreFunc()()
config.UpdateGlobal(func(conf *config.Config) {
conf.TempStoragePath = t.TempDir()
})
Expand Down
1 change: 0 additions & 1 deletion executor/test/analyzetest/analyze_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -551,7 +551,6 @@ func TestDefaultValForAnalyze(t *testing.T) {
}

func TestAnalyzeFullSamplingOnIndexWithVirtualColumnOrPrefixColumn(t *testing.T) {
t.Skip("unstable, skip it and fix it before 20210624")
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)

Expand Down
10 changes: 3 additions & 7 deletions planner/core/prepare_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,14 @@ import (
"testing"
"time"

"github.com/pingcap/tidb/errno"
"github.com/pingcap/tidb/executor"
"github.com/pingcap/tidb/expression"
"github.com/pingcap/tidb/infoschema"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/metrics"
"github.com/pingcap/tidb/parser"
"github.com/pingcap/tidb/parser/auth"
"github.com/pingcap/tidb/parser/terror"
"github.com/pingcap/tidb/planner/core"
"github.com/pingcap/tidb/session"
"github.com/pingcap/tidb/sessionctx/variable"
Expand Down Expand Up @@ -720,7 +720,6 @@ func TestPrepareCacheNow(t *testing.T) {
}

func TestPrepareOverMaxPreparedStmtCount(t *testing.T) {
t.Skip("unstable, skip it and fix it before 20210705")
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
Expand Down Expand Up @@ -752,18 +751,15 @@ func TestPrepareOverMaxPreparedStmtCount(t *testing.T) {
for i := 1; ; i++ {
prePrepared = readGaugeInt(metrics.PreparedStmtGauge)
if prePrepared >= 2 {
_, err := tk.Exec(`prepare stmt` + strconv.Itoa(i) + ` from "select 1"`)
require.True(t, terror.ErrorEqual(err, variable.ErrMaxPreparedStmtCountReached))
tk.MustGetErrCode(`prepare stmt`+strconv.Itoa(i)+` from "select 1"`, errno.ErrMaxPreparedStmtCountReached)
break
}
_, err := tk.Exec(`prepare stmt` + strconv.Itoa(i) + ` from "select 1"`)
require.NoError(t, err)
tk.MustExec(`prepare stmt` + strconv.Itoa(i) + ` from "select 1"`)
}
}

// unit test for issue https://github.com/pingcap/tidb/issues/8518
func TestPrepareTableAsNameOnGroupByWithCache(t *testing.T) {
t.Skip("unstable, skip it and fix it before 20210702")
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec(`set tidb_enable_prepared_plan_cache=1`)
Expand Down