From 20d199bdb26a58473e5bc7ac57e65db7ed51c66a Mon Sep 17 00:00:00 2001 From: Naman Jain Date: Wed, 5 May 2021 13:32:56 +0530 Subject: [PATCH] opt(dropPrefix): allow logical drop for deleting predicates and indexing (#7764) Issue: DropPrefix is Stop The World operation. It blocks the incoming writes. Dgraph uses this API at a couple of places like: - drop data - reindexing - dropping predicate The last 2 usages can lead to bad user impact in a multi-tenant cluster. Solution: This PR allows DropPrefix to use the blocking/non-blocking DropPrefix based on DB option `AllowStopTheWorld` that can be set using `--badger` superflag. (cherry picked from commit c86b85536dd2b0337599b04bef7ce7b2457f4c39) --- go.mod | 5 ++--- go.sum | 16 ++++++++-------- posting/index.go | 13 ++++++++++++- posting/list_test.go | 2 +- worker/snapshot.go | 5 ++++- 5 files changed, 27 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 2f3d0b462ea..8c11ea7acb4 100644 --- a/go.mod +++ b/go.mod @@ -7,19 +7,18 @@ require ( contrib.go.opencensus.io/exporter/prometheus v0.1.0 github.com/DataDog/datadog-go v0.0.0-20190425163447-40bafcb5f6c1 // indirect github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20190503082300-0f32ad59ab08 - github.com/DataDog/zstd v1.4.5 // indirect github.com/Masterminds/semver/v3 v3.1.0 github.com/Microsoft/go-winio v0.4.15 // indirect github.com/OneOfOne/xxhash v1.2.5 // indirect github.com/Shopify/sarama v1.27.2 github.com/blevesearch/bleve v1.0.13 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd - github.com/dgraph-io/badger/v3 v3.0.0-20210405181011-d918b9904b2a + github.com/dgraph-io/badger/v3 v3.0.0-20210504192519-da5f789ecb38 github.com/dgraph-io/dgo/v210 v210.0.0-20210407152819-261d1c2a6987 github.com/dgraph-io/gqlgen v0.13.2 github.com/dgraph-io/gqlparser/v2 v2.2.0 github.com/dgraph-io/graphql-transport-ws v0.0.0-20210223074046-e5b8b80bb4ed - github.com/dgraph-io/ristretto v0.0.4-0.20210310100713-a4346e5d1f90 + github.com/dgraph-io/ristretto v0.0.4-0.20210504190834-0bf2acd73aa3 github.com/dgraph-io/simdjson-go v0.3.0 github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1 diff --git a/go.sum b/go.sum index e043e8f4ba5..63ae51b7bbc 100644 --- a/go.sum +++ b/go.sum @@ -28,9 +28,8 @@ github.com/DataDog/datadog-go v0.0.0-20190425163447-40bafcb5f6c1 h1:fSu93OUqfEko github.com/DataDog/datadog-go v0.0.0-20190425163447-40bafcb5f6c1/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20190503082300-0f32ad59ab08 h1:5btKvK+N+FpW0EEgvxq7LWcUEwIRLsL4IwIo0u+Qlhs= github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20190503082300-0f32ad59ab08/go.mod h1:gMGUEe16aZh0QN941HgDjwrdjU4iTthPoz2/AtDRADE= -github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= -github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/DataDog/zstd v1.4.6-0.20210216161059-8cb8bacba7ba h1:3qB2yylqW3kVPr9QoPZtTJOXsJOUdNWT2CrZcifhs5g= +github.com/DataDog/zstd v1.4.6-0.20210216161059-8cb8bacba7ba/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Masterminds/semver/v3 v3.1.0 h1:Y2lUDsFKVRSYGojLJ1yLxSXdMmMYTYls0rCvoqmMUQk= @@ -89,6 +88,8 @@ github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QH github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= @@ -117,8 +118,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/badger v1.6.0 h1:DshxFxZWXUcO0xX476VJC07Xsr6ZCBVRHKZ93Oh7Evo= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= -github.com/dgraph-io/badger/v3 v3.0.0-20210405181011-d918b9904b2a h1:KUJzMbhVSuSDkXXkV0yI1Uj/hGNOGTUEc0dbusDixas= -github.com/dgraph-io/badger/v3 v3.0.0-20210405181011-d918b9904b2a/go.mod h1:GHMCYxuDWyzbHkh4k3yyg4PM61tJPFfEGSMbE3Vd5QE= +github.com/dgraph-io/badger/v3 v3.0.0-20210504192519-da5f789ecb38 h1:xbd21QOUIeIXI0dQtj2U+68PKo8x8CrK7abJyUt8dkU= +github.com/dgraph-io/badger/v3 v3.0.0-20210504192519-da5f789ecb38/go.mod h1:NiMHhAd/Elhx154lGQ4oJennK/f4ADMqhbggxs/mAew= github.com/dgraph-io/dgo/v210 v210.0.0-20210407152819-261d1c2a6987 h1:5aN6H88a2q3HkO8vSZxDlgjEpJf4Fz8rfy+/Wzx2uAc= github.com/dgraph-io/dgo/v210 v210.0.0-20210407152819-261d1c2a6987/go.mod h1:dCzdThGGTPYOAuNtrM6BiXj/86voHn7ZzkPL6noXR3s= github.com/dgraph-io/gqlgen v0.13.2 h1:TNhndk+eHKj5qE7BenKKSYdSIdOGhLqxR1rCiMso9KM= @@ -128,9 +129,8 @@ github.com/dgraph-io/gqlparser/v2 v2.2.0 h1:fKSCW8OxoMogjDwUhO9OrFvrgIA0UZspTDbc github.com/dgraph-io/gqlparser/v2 v2.2.0/go.mod h1:MYS4jppjyx8b9tuUtjV7jU1UFZK6P9fvO8TsIsQtRKU= github.com/dgraph-io/graphql-transport-ws v0.0.0-20210223074046-e5b8b80bb4ed h1:pgGMBoTtFhR+xkyzINaToLYRurHn+6pxMYffIGmmEPc= github.com/dgraph-io/graphql-transport-ws v0.0.0-20210223074046-e5b8b80bb4ed/go.mod h1:7z3c/5w0sMYYZF5bHsrh8IH4fKwG5O5Y70cPH1ZLLRQ= -github.com/dgraph-io/ristretto v0.0.4-0.20210309073149-3836124cdc5a/go.mod h1:MIonLggsKgZLUSt414ExgwNtlOL5MuEoAJP514mwGe8= -github.com/dgraph-io/ristretto v0.0.4-0.20210310100713-a4346e5d1f90 h1:arWVlUO9NhZ/2vWprIqpe825GISUPpgJhU/b0ep3j/M= -github.com/dgraph-io/ristretto v0.0.4-0.20210310100713-a4346e5d1f90/go.mod h1:MIonLggsKgZLUSt414ExgwNtlOL5MuEoAJP514mwGe8= +github.com/dgraph-io/ristretto v0.0.4-0.20210504190834-0bf2acd73aa3 h1:jU/wpYsEL+8JPLf/QcjkQKI5g0dOjSuwcMjkThxt5x0= +github.com/dgraph-io/ristretto v0.0.4-0.20210504190834-0bf2acd73aa3/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgraph-io/simdjson-go v0.3.0 h1:h71LO7vR4LHMPUhuoGN8bqGm1VNfGOlAG8BI6iDUKw0= github.com/dgraph-io/simdjson-go v0.3.0/go.mod h1:Otpysdjaxj9OGaJusn4pgQV7OFh2bELuHANq0I78uvY= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= diff --git a/posting/index.go b/posting/index.go index 73a7f99748c..84ff1023f2e 100644 --- a/posting/index.go +++ b/posting/index.go @@ -1241,14 +1241,25 @@ func DeleteData() error { return pstore.DropPrefix([]byte{x.DefaultPrefix}) } -// DeletePredicate deletes all entries and indices for a given predicate. +// DeletePredicate deletes all entries and indices for a given predicate. The delete may be logical +// based on DB options set. func DeletePredicate(ctx context.Context, attr string) error { glog.Infof("Dropping predicate: [%s]", attr) prefix := x.PredicatePrefix(attr) if err := pstore.DropPrefix(prefix); err != nil { return err } + return schema.State().Delete(attr) +} +// DeletePredicateBlocking deletes all entries and indices for a given predicate. It also blocks the +// writes. +func DeletePredicateBlocking(ctx context.Context, attr string) error { + glog.Infof("Dropping predicate: [%s]", attr) + prefix := x.PredicatePrefix(attr) + if err := pstore.DropPrefixBlocking(prefix); err != nil { + return err + } return schema.State().Delete(attr) } diff --git a/posting/list_test.go b/posting/list_test.go index 6e4d8f8f096..776ced1e0ae 100644 --- a/posting/list_test.go +++ b/posting/list_test.go @@ -1411,7 +1411,7 @@ func TestMain(m *testing.M) { dir, err := ioutil.TempDir("", "storetest_") x.Check(err) - ps, err = badger.OpenManaged(badger.DefaultOptions(dir)) + ps, err = badger.OpenManaged(badger.DefaultOptions(dir).WithAllowStopTheWorld(false)) x.Check(err) // Not using posting list cache Init(ps, 0) diff --git a/worker/snapshot.go b/worker/snapshot.go index e7fde2339de..b39eaf340cd 100644 --- a/worker/snapshot.go +++ b/worker/snapshot.go @@ -144,7 +144,10 @@ func deleteStalePreds(ctx context.Context, kvs *pb.KVS) error { if _, ok := snapshotPreds[pred]; !ok { LOOP: for { - err := posting.DeletePredicate(ctx, pred) + // While retrieving the snapshot, we mark the node as unhealthy. So it is better to + // a blocking delete of predicate as we know that no new writes will arrive at + // this alpha. + err := posting.DeletePredicateBlocking(ctx, pred) switch err { case badger.ErrBlockedWrites: time.Sleep(1 * time.Second)