From d713f878eb42aafac4bc043dec0049d708bef8d2 Mon Sep 17 00:00:00 2001 From: Ibrahim Jarif Date: Sat, 5 Dec 2020 00:50:55 +0530 Subject: [PATCH 1/5] feat(debug): Support rollup in debug tool --- dgraph/cmd/debug/run.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/dgraph/cmd/debug/run.go b/dgraph/cmd/debug/run.go index 8960bb194db..fb13fab78fe 100644 --- a/dgraph/cmd/debug/run.go +++ b/dgraph/cmd/debug/run.go @@ -56,6 +56,7 @@ var ( type flagOptions struct { vals bool keyLookup string + rollupKey string keyHistory bool predicate string readOnly bool @@ -93,6 +94,7 @@ func init() { flag.BoolVarP(&opt.readOnly, "readonly", "o", true, "Open in read only mode.") flag.StringVarP(&opt.predicate, "pred", "r", "", "Only output specified predicate.") flag.StringVarP(&opt.keyLookup, "lookup", "l", "", "Hex of key to lookup.") + flag.StringVar(&opt.rollupKey, "rollup", "", "Hex of key to rollup.") flag.BoolVarP(&opt.keyHistory, "history", "y", false, "Show all versions of a key.") flag.StringVarP(&opt.pdir, "postings", "p", "", "Directory where posting lists are stored.") flag.BoolVar(&opt.sizeHistogram, "histogram", false, @@ -441,6 +443,37 @@ func appendPosting(w io.Writer, o *pb.Posting) { } fmt.Fprintln(w, "") } +func rollupKey(db *badger.DB) { + txn := db.NewTransactionAt(opt.readTs, false) + defer txn.Discard() + + iopts := badger.DefaultIteratorOptions + iopts.AllVersions = true + iopts.PrefetchValues = false + itr := txn.NewIterator(iopts) + defer itr.Close() + + key, err := hex.DecodeString(opt.keyLookup) + x.Check(err) + itr.Seek(key) + if !itr.Valid() { + log.Fatalf("Unable to seek to key: %s", hex.Dump(key)) + } + + item := itr.Item() + pl, err := posting.ReadPostingList(item.KeyCopy(nil), itr) + x.Check(err) + + alloc := z.NewAllocator(32 << 20) + defer alloc.Release() + + kvs, err := pl.Rollup(alloc) + x.Check(err) + + wb := db.NewManagedWriteBatch() + x.Check(wb.WriteList(&bpb.KVList{Kv: kvs})) + x.Check(wb.Flush()) +} func lookup(db *badger.DB) { txn := db.NewTransactionAt(opt.readTs, false) @@ -896,6 +929,8 @@ func run() { // fmt.Printf("Min commit: %d. Max commit: %d, w.r.t %d\n", min, max, opt.readTs) switch { + case len(opt.rollupKey) > 0: + rollupKey(db) case len(opt.keyLookup) > 0: lookup(db) case len(opt.jepsen) > 0: From c6381f732b339bf5fc8d484391d4ced0042c43ce Mon Sep 17 00:00:00 2001 From: Ibrahim Jarif Date: Sat, 5 Dec 2020 00:52:09 +0530 Subject: [PATCH 2/5] fixup --- dgraph/cmd/debug/run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dgraph/cmd/debug/run.go b/dgraph/cmd/debug/run.go index fb13fab78fe..8fdd8b7a2be 100644 --- a/dgraph/cmd/debug/run.go +++ b/dgraph/cmd/debug/run.go @@ -453,7 +453,7 @@ func rollupKey(db *badger.DB) { itr := txn.NewIterator(iopts) defer itr.Close() - key, err := hex.DecodeString(opt.keyLookup) + key, err := hex.DecodeString(opt.rollupKey) x.Check(err) itr.Seek(key) if !itr.Valid() { From fe6f435493e9282536e74d5c62d8121d94f5e41d Mon Sep 17 00:00:00 2001 From: Ibrahim Jarif Date: Sat, 5 Dec 2020 01:08:43 +0530 Subject: [PATCH 3/5] add review comments --- dgraph/cmd/debug/run.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/dgraph/cmd/debug/run.go b/dgraph/cmd/debug/run.go index 8fdd8b7a2be..fae1f337067 100644 --- a/dgraph/cmd/debug/run.go +++ b/dgraph/cmd/debug/run.go @@ -447,20 +447,26 @@ func rollupKey(db *badger.DB) { txn := db.NewTransactionAt(opt.readTs, false) defer txn.Discard() + key, err := hex.DecodeString(opt.rollupKey) + x.Check(err) + iopts := badger.DefaultIteratorOptions iopts.AllVersions = true iopts.PrefetchValues = false - itr := txn.NewIterator(iopts) + itr := txn.NewKeyIterator(key, iopts) defer itr.Close() - key, err := hex.DecodeString(opt.rollupKey) - x.Check(err) - itr.Seek(key) + itr.Rewind() if !itr.Valid() { log.Fatalf("Unable to seek to key: %s", hex.Dump(key)) } item := itr.Item() + // Don't need to do anything if the bitdelta is not set. + if item.UserMeta()&posting.BitDeltaPosting == 0 { + fmt.Println("First item doesn't have bitdelta. Nothing to do") + return + } pl, err := posting.ReadPostingList(item.KeyCopy(nil), itr) x.Check(err) From ef629c26da01b8ecbe2a405997838733030069ae Mon Sep 17 00:00:00 2001 From: Ibrahim Jarif Date: Sat, 5 Dec 2020 01:29:28 +0530 Subject: [PATCH 4/5] break from for loop --- dgraph/cmd/debug/run.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dgraph/cmd/debug/run.go b/dgraph/cmd/debug/run.go index fae1f337067..fd7dc9992d4 100644 --- a/dgraph/cmd/debug/run.go +++ b/dgraph/cmd/debug/run.go @@ -582,6 +582,7 @@ func printKeys(db *badger.DB) { } var sz, deltaCount int64 + LOOP: for ; itr.ValidForPrefix(prefix); itr.Next() { item := itr.Item() if !bytes.Equal(item.Key(), key) { @@ -595,7 +596,7 @@ func printKeys(db *badger.DB) { // This is rather a default case as one of the 4 bit must be set. case posting.BitCompletePosting, posting.BitEmptyPosting, posting.BitSchemaPosting: sz += item.EstimatedSize() - break + break LOOP case posting.BitDeltaPosting: sz += item.EstimatedSize() deltaCount++ From 62d86300a23fc79ee349b11e5d75460e6f119a3d Mon Sep 17 00:00:00 2001 From: Ibrahim Jarif Date: Sat, 5 Dec 2020 01:31:58 +0530 Subject: [PATCH 5/5] Address review comments --- dgraph/cmd/debug/run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dgraph/cmd/debug/run.go b/dgraph/cmd/debug/run.go index fd7dc9992d4..765d47543a3 100644 --- a/dgraph/cmd/debug/run.go +++ b/dgraph/cmd/debug/run.go @@ -464,7 +464,7 @@ func rollupKey(db *badger.DB) { item := itr.Item() // Don't need to do anything if the bitdelta is not set. if item.UserMeta()&posting.BitDeltaPosting == 0 { - fmt.Println("First item doesn't have bitdelta. Nothing to do") + fmt.Printf("First item has UserMeta:[b%04b]. Nothing to do\n", item.UserMeta()) return } pl, err := posting.ReadPostingList(item.KeyCopy(nil), itr)