Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

store/tikv:Ignore error and do gc anyway #5797

Merged
merged 18 commits into from
Feb 7, 2018
Merged

store/tikv:Ignore error and do gc anyway #5797

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
d1c6181
ignore error when gc, and do gc on next region
wentaoxu Feb 5, 2018
774980e
add ignored regions
wentaoxu Feb 5, 2018
3a09833
add log ticker
wentaoxu Feb 6, 2018
1f724ac
fix comments
wentaoxu Feb 6, 2018
68c8f4b
fix ctx
wentaoxu Feb 6, 2018
65ef1bf
fix comments, add log and metrics
wentaoxu Feb 6, 2018
971928b
fix comments
wentaoxu Feb 6, 2018
f54618f
fix comments
wentaoxu Feb 6, 2018
68b5a60
Merge branch 'master' into ignore_error_and_do_gc_anyway
wentaoxu Feb 6, 2018
6702d83
fix comments
wentaoxu Feb 6, 2018
4112b67
Merge branch 'ignore_error_and_do_gc_anyway' of github.com:wentaoxu/t…
wentaoxu Feb 6, 2018
30516f1
Merge branch 'master' into ignore_error_and_do_gc_anyway
wentaoxu Feb 6, 2018
9e0c930
add test
wentaoxu Feb 7, 2018
e02f351
add test and active gc worker test
wentaoxu Feb 7, 2018
8360650
fix fmt
wentaoxu Feb 7, 2018
afd9c75
Merge branch 'master' into ignore_error_and_do_gc_anyway
wentaoxu Feb 7, 2018
d24eb68
fix comments
wentaoxu Feb 7, 2018
3d0155a
fix comments
wentaoxu Feb 7, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion store/tikv/backoff.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ const (
getMaxBackoff = 20000
prewriteMaxBackoff = 20000
cleanupMaxBackoff = 20000
GcMaxBackoff = 100000
GcOneRegionMaxBackoff = 20000
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why change this value?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think gc command can definitely returns in 20 secs.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok, I will change it for one request

GcResolveLockMaxBackoff = 100000
GcDeleteRangeMaxBackoff = 100000
rawkvMaxBackoff = 20000
Expand Down
1 change: 1 addition & 0 deletions store/tikv/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ const (
readTimeoutShort = 20 * time.Second // For requests that read/write several key-values.
ReadTimeoutMedium = 60 * time.Second // For requests that may need scan region.
ReadTimeoutLong = 150 * time.Second // For requests that may need scan region multiple times.
GCTimeout = 5 * time.Minute

grpcInitialWindowSize = 1 << 30
grpcInitialConnWindowSize = 1 << 30
Expand Down
86 changes: 59 additions & 27 deletions store/tikv/gcworker/gc_worker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"time"

"github.com/juju/errors"
"github.com/pingcap/kvproto/pkg/errorpb"
"github.com/pingcap/kvproto/pkg/kvrpcpb"
"github.com/pingcap/tidb"
"github.com/pingcap/tidb/ddl/util"
Expand Down Expand Up @@ -90,6 +91,7 @@ func (w *GCWorker) Close() {
const (
gcTimeFormat = "20060102-15:04:05 -0700 MST"
gcWorkerTickInterval = time.Minute
gcJobLogTickInterval = time.Minute * 10
gcWorkerLease = time.Minute * 2
gcLeaderUUIDKey = "tikv_gc_leader_uuid"
gcLeaderDescKey = "tikv_gc_leader_desc"
Expand Down Expand Up @@ -327,7 +329,6 @@ func (w *GCWorker) runGCJob(ctx goctx.Context, safePoint uint64) {
}
err = doGC(ctx, w.store, safePoint, w.uuid)
if err != nil {
gcFailureCounter.WithLabelValues("gc").Inc()
log.Error("do GC returns an error", err)
w.gcIsRunning = false
w.done <- errors.Trace(err)
Expand All @@ -346,7 +347,7 @@ func (w *GCWorker) deleteRanges(ctx goctx.Context, safePoint uint64) error {
return errors.Trace(err)
}

bo := tikv.NewBackoffer(tikv.GcDeleteRangeMaxBackoff, goctx.Background())
bo := tikv.NewBackoffer(tikv.GcDeleteRangeMaxBackoff, ctx)
log.Infof("[gc worker] %s start delete %v ranges", w.uuid, len(ranges))
startTime := time.Now()
regions := 0
Expand Down Expand Up @@ -430,7 +431,7 @@ func resolveLocks(ctx goctx.Context, store tikv.Storage, safePoint uint64, ident
Limit: gcScanLockLimit,
},
}
bo := tikv.NewBackoffer(tikv.GcResolveLockMaxBackoff, goctx.Background())
bo := tikv.NewBackoffer(tikv.GcResolveLockMaxBackoff, ctx)

log.Infof("[gc worker] %s start resolve locks, safePoint: %v.", identifier, safePoint)
startTime := time.Now()
Expand Down Expand Up @@ -515,63 +516,94 @@ func doGC(ctx goctx.Context, store tikv.Storage, safePoint uint64, identifier st
// Sleep to wait for all other tidb instances update their safepoint cache.
time.Sleep(gcSafePointCacheInterval)

log.Infof("[gc worker] %s start gc, safePoint: %v.", identifier, safePoint)
startTime := time.Now()
regions := 0
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

s/regions/successRegions
s/errRegions/failedRegions

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok

errRegions := 0

ticker := time.NewTicker(gcJobLogTickInterval)
defer ticker.Stop()

req := &tikvrpc.Request{
Type: tikvrpc.CmdGC,
GC: &kvrpcpb.GCRequest{
SafePoint: safePoint,
},
}
bo := tikv.NewBackoffer(tikv.GcMaxBackoff, goctx.Background())

log.Infof("[gc worker] %s start gc, safePoint: %v.", identifier, safePoint)
startTime := time.Now()
regions := 0

bo := tikv.NewBackoffer(tikv.GcOneRegionMaxBackoff, ctx)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Initialize it in following for loop seems better.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is hard to do because It should not be initialized when we meet region error.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Got it.

var key []byte
for {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please remove the loop.

select {
case <-ctx.Done():
return errors.New("[gc worker] gc job canceled")
case <-ticker.C:
log.Infof("[gc worker] %s gc in process, safePoint: %v, regions: %v, ignored regions: %v, cost time: %s",
identifier, safePoint, regions, errRegions, time.Since(startTime))
default:
}

loc, err := store.GetRegionCache().LocateKey(bo, key)
if err != nil {
return errors.Trace(err)
}
resp, err := store.SendReq(bo, req, loc.Region, tikv.ReadTimeoutLong)
if err != nil {
return errors.Trace(err)
}
regionErr, err := resp.GetRegionError()

var regionErr *errorpb.Error
regionErr, err = doGCForOneRegion(bo, store, req, loc.Region)

if err != nil {
return errors.Trace(err)
}
if regionErr != nil {
errRegions++
log.Warnf("[gc worker] %s failed to do gc on region(%s, %s), ignore it", identifier, string(loc.StartKey), string(loc.EndKey))
gcFailureCounter.WithLabelValues("gc").Inc()
} else if regionErr != nil {
Copy link
Contributor

@zhangjinpeng87 zhangjinpeng87 Feb 6, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if regionErr != nil {
backoff
continue
}
if err != nil {
} else {
}

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it is better to check err first, because if regionErr and err both exist, err should have higher priority.

err = bo.Backoff(tikv.BoRegionMiss, errors.New(regionErr.String()))
if err != nil {
return errors.Trace(err)
errRegions++
log.Warnf("[gc worker] %s failed to do gc on region(%s, %s), ignore it", identifier, string(loc.StartKey), string(loc.EndKey))
gcFailureCounter.WithLabelValues("gc").Inc()
} else {
continue
}
continue
}
gcResp := resp.GC
if gcResp == nil {
return errors.Trace(tikv.ErrBodyMissing)
}
if gcResp.GetError() != nil {
return errors.Errorf("unexpected gc error: %s", gcResp.GetError())
} else {
regions++
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if err != nil {
err++
}
if regionError != nil {
backoff
continue
}
if err == nil {
++}

}
regions++

key = loc.EndKey
if len(key) == 0 {
break
}
bo = tikv.NewBackoffer(tikv.GcOneRegionMaxBackoff, ctx)
}
log.Infof("[gc worker] %s finish gc, safePoint: %v, regions: %v, cost time: %s", identifier, safePoint, regions, time.Since(startTime))
log.Infof("[gc worker] %s finish gc, safePoint: %v, regions: %v, ignored regions: %v, cost time: %s",
identifier, safePoint, regions, errRegions, time.Since(startTime))
gcHistogram.WithLabelValues("do_gc").Observe(time.Since(startTime).Seconds())
return nil
}

func doGCForOneRegion(bo *tikv.Backoffer, store tikv.Storage, req *tikvrpc.Request, region tikv.RegionVerID) (*errorpb.Error, error) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems we should move the construction of req into this function?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think for the completion, it should be move in here, but if we have 1M regions, this cost very large extra memory.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The cost is acceptable. If i pass an request not gc, the behavior is not consistent with the function name.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok, I will change it

resp, err := store.SendReq(bo, req, region, tikv.GCTimeout)
if err != nil {
return nil, errors.Trace(err)
}
regionErr, err := resp.GetRegionError()
if err != nil {
return nil, errors.Trace(err)
}
if regionErr != nil {
return regionErr, nil
}

gcResp := resp.GC
if gcResp == nil {
return nil, errors.Trace(tikv.ErrBodyMissing)
}
if gcResp.GetError() != nil {
return nil, errors.Errorf("unexpected gc error: %s", gcResp.GetError())
}

return nil, nil
}

func (w *GCWorker) checkLeader() (bool, error) {
gcWorkerCounter.WithLabelValues("check_leader").Inc()
session := createSession(w.store)
Expand Down