From b4f11a0d9d5c0c5cc4478c0923926ca5fb1fdd04 Mon Sep 17 00:00:00 2001 From: Thiago Figueiredo Date: Wed, 22 Sep 2021 13:03:48 -0300 Subject: [PATCH 1/5] tstorebe: Add record inventory fsck. --- politeiad/backendv2/tstorebe/tstorebe.go | 87 ++++++++++++++++++++---- 1 file changed, 72 insertions(+), 15 deletions(-) diff --git a/politeiad/backendv2/tstorebe/tstorebe.go b/politeiad/backendv2/tstorebe/tstorebe.go index 08587dbd9..e54c0c282 100644 --- a/politeiad/backendv2/tstorebe/tstorebe.go +++ b/politeiad/backendv2/tstorebe/tstorebe.go @@ -10,7 +10,9 @@ import ( "encoding/hex" "encoding/json" "fmt" + "os" "path/filepath" + "sort" "strconv" "sync" "time" @@ -1017,21 +1019,76 @@ func (t *tstoreBackend) Fsck() error { return err } - // Rebuild the inventory cache. This is a multi-step process: - // - // 1. Sort the tokens into two groups; unvetted and vetted. Each - // group is additionally sorted by the timestamp of their most - // recent status change. - // - // 2. Add the vetted tokens to the inventory cache. They MUST be - // added starting with the oldest timestamp. The vetted token - // is first added to the unvetted cache then moved to the vetted - // cache. This is a limitation of the inventory API and mimicks - // how records are added and updated when using the politeiad - // API. - // - // 3. Add the unvetted tokens to the inventory cache. They MUST be - // added starting with the oldest timestamp. + // Get the partial record for all tokens. This also guarantees that all + // tokens being manipulated actually correspond to a record on the backend. + records := make(map[string]*backend.Record, len(allTokens)) + for _, token := range allTokens { + r, err := t.tstore.RecordPartial(token, 0, nil, true) + if err != nil { + return err + } + records[r.RecordMetadata.Token] = r + } + + // Sort records into vetted and unvetted groups. + vetted := make([]*backend.Record, len(allTokens)) + unvetted := make([]*backend.Record, len(allTokens)) + for _, token := range allTokens { + record := records[hex.EncodeToString(token)] + if record.RecordMetadata.State == backend.StateVetted { + vetted = append(vetted, record) + } + if record.RecordMetadata.State == backend.StateUnvetted { + unvetted = append(unvetted, record) + } + } + + // Sort records from both groups by the timestamp of their record metadata, + // from oldest to newest. The order of the record inventory will be + // slightly different. On runtime, the timestamp order is through the most + // recent status change md. On this fsck rebuild, the order is through the + // record metadata's timestamp from their last update. + sort.Slice(vetted, func(i, j int) bool { + return vetted[i].RecordMetadata.Timestamp < + vetted[j].RecordMetadata.Timestamp + }) + sort.Slice(unvetted, func(i, j int) bool { + return unvetted[i].RecordMetadata.Timestamp < + unvetted[j].RecordMetadata.Timestamp + }) + + // Now that data is sorted, delete inventory cache before building the new, + // updated one. + err = os.RemoveAll(t.invPathVetted()) + if err != nil { + return err + } + err = os.RemoveAll(t.invPathUnvetted()) + if err != nil { + return err + } + + // Add vetted tokens to inventory cache. First add to inventory as unvetted, + // then move to vetted. This is a temporary limitation of the inventory API, + // which was done this way to mimick the way records are added and updated + // on the politeiad API. + for _, record := range vetted { + bToken, err := hex.DecodeString(record.RecordMetadata.Token) + if err != nil { + return err + } + t.inventoryAdd(backend.StateUnvetted, bToken, backend.StatusUnreviewed) + t.inventoryMoveToVetted(bToken, record.RecordMetadata.Status) + } + + // Add unvetted tokens to inventory cache. + for _, record := range unvetted { + bToken, err := hex.DecodeString(record.RecordMetadata.Token) + if err != nil { + return err + } + t.inventoryAdd(record.RecordMetadata.State, bToken, record.RecordMetadata.Status) + } // Update all plugin caches return t.tstore.Fsck(allTokens) From 1ecb2ec4ead5ce266479013ad48762031da35083 Mon Sep 17 00:00:00 2001 From: Thiago Figueiredo Date: Sat, 25 Sep 2021 14:40:30 -0300 Subject: [PATCH 2/5] address review comments --- politeiad/backendv2/tstorebe/inventory.go | 10 ++++++++++ politeiad/backendv2/tstorebe/tstorebe.go | 22 ++++++++++++---------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/politeiad/backendv2/tstorebe/inventory.go b/politeiad/backendv2/tstorebe/inventory.go index a4c287248..667b48eac 100644 --- a/politeiad/backendv2/tstorebe/inventory.go +++ b/politeiad/backendv2/tstorebe/inventory.go @@ -43,6 +43,16 @@ func (t *tstoreBackend) invPathVetted() string { return filepath.Join(t.dataDir, filenameInvVetted) } +// invRemoveUnvetted removes the unvetted inventory from its respective path. +func (t *tstoreBackend) invRemoveUnvetted() error { + return os.RemoveAll(t.invPathUnvetted()) +} + +// invRemoveVetted removes the vetted inventory from its respective path. +func (t *tstoreBackend) invRemoveVetted() error { + return os.RemoveAll(t.invPathVetted()) +} + // invGetLocked retrieves the inventory from disk. A new inventory is returned // if one does not exist yet. // diff --git a/politeiad/backendv2/tstorebe/tstorebe.go b/politeiad/backendv2/tstorebe/tstorebe.go index e54c0c282..a8353ffad 100644 --- a/politeiad/backendv2/tstorebe/tstorebe.go +++ b/politeiad/backendv2/tstorebe/tstorebe.go @@ -10,7 +10,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "os" "path/filepath" "sort" "strconv" @@ -1046,8 +1045,10 @@ func (t *tstoreBackend) Fsck() error { // Sort records from both groups by the timestamp of their record metadata, // from oldest to newest. The order of the record inventory will be // slightly different. On runtime, the timestamp order is through the most - // recent status change md. On this fsck rebuild, the order is through the - // record metadata's timestamp from their last update. + // recent status change metadata. On this fsck rebuild, the order is + // through the record timestamp from their last edit. This happens because + // the record timestamp gets updated on both status changes and edits, so + // the status change timestamp gets lost when the record is edited. sort.Slice(vetted, func(i, j int) bool { return vetted[i].RecordMetadata.Timestamp < vetted[j].RecordMetadata.Timestamp @@ -1059,19 +1060,19 @@ func (t *tstoreBackend) Fsck() error { // Now that data is sorted, delete inventory cache before building the new, // updated one. - err = os.RemoveAll(t.invPathVetted()) + err = t.invRemoveVetted() if err != nil { return err } - err = os.RemoveAll(t.invPathUnvetted()) + err = t.invRemoveUnvetted() if err != nil { return err } - // Add vetted tokens to inventory cache. First add to inventory as unvetted, - // then move to vetted. This is a temporary limitation of the inventory API, - // which was done this way to mimick the way records are added and updated - // on the politeiad API. + // Add vetted tokens to inventory cache. First add to inventory as + // unvetted, then move to vetted. This is a temporary limitation of the + // inventory API, which was done this way to mimick the way records are + // added and updated on the politeiad API. for _, record := range vetted { bToken, err := hex.DecodeString(record.RecordMetadata.Token) if err != nil { @@ -1087,7 +1088,8 @@ func (t *tstoreBackend) Fsck() error { if err != nil { return err } - t.inventoryAdd(record.RecordMetadata.State, bToken, record.RecordMetadata.Status) + t.inventoryAdd(record.RecordMetadata.State, bToken, + record.RecordMetadata.Status) } // Update all plugin caches From ba098f1e905a43b8a4c599ff62b01c7bdb95e824 Mon Sep 17 00:00:00 2001 From: Thiago Figueiredo Date: Mon, 27 Sep 2021 15:44:08 -0300 Subject: [PATCH 3/5] slice sort fix --- politeiad/backendv2/tstorebe/tstorebe.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/politeiad/backendv2/tstorebe/tstorebe.go b/politeiad/backendv2/tstorebe/tstorebe.go index a8353ffad..d6f8dc33e 100644 --- a/politeiad/backendv2/tstorebe/tstorebe.go +++ b/politeiad/backendv2/tstorebe/tstorebe.go @@ -1030,8 +1030,10 @@ func (t *tstoreBackend) Fsck() error { } // Sort records into vetted and unvetted groups. - vetted := make([]*backend.Record, len(allTokens)) - unvetted := make([]*backend.Record, len(allTokens)) + var ( + vetted []*backend.Record + unvetted []*backend.Record + ) for _, token := range allTokens { record := records[hex.EncodeToString(token)] if record.RecordMetadata.State == backend.StateVetted { From 3a5dfef831c8123deb50b03ab177b6832b711862 Mon Sep 17 00:00:00 2001 From: Thiago Figueiredo Date: Tue, 28 Sep 2021 12:27:05 -0300 Subject: [PATCH 4/5] Set vetted/unvetted slice capacity --- politeiad/backendv2/tstorebe/tstorebe.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/politeiad/backendv2/tstorebe/tstorebe.go b/politeiad/backendv2/tstorebe/tstorebe.go index d6f8dc33e..b320d54bf 100644 --- a/politeiad/backendv2/tstorebe/tstorebe.go +++ b/politeiad/backendv2/tstorebe/tstorebe.go @@ -1030,10 +1030,8 @@ func (t *tstoreBackend) Fsck() error { } // Sort records into vetted and unvetted groups. - var ( - vetted []*backend.Record - unvetted []*backend.Record - ) + vetted := make([]*backend.Record, 0, len(allTokens)) + unvetted := make([]*backend.Record, 0, len(allTokens)) for _, token := range allTokens { record := records[hex.EncodeToString(token)] if record.RecordMetadata.State == backend.StateVetted { From a32c3386519267ea5ee09293832f73d12b24c7dc Mon Sep 17 00:00:00 2001 From: Thiago Figueiredo Date: Thu, 30 Sep 2021 11:29:57 -0300 Subject: [PATCH 5/5] review comments --- politeiad/backendv2/tstorebe/tstorebe.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/politeiad/backendv2/tstorebe/tstorebe.go b/politeiad/backendv2/tstorebe/tstorebe.go index b320d54bf..cc0572432 100644 --- a/politeiad/backendv2/tstorebe/tstorebe.go +++ b/politeiad/backendv2/tstorebe/tstorebe.go @@ -1030,8 +1030,10 @@ func (t *tstoreBackend) Fsck() error { } // Sort records into vetted and unvetted groups. - vetted := make([]*backend.Record, 0, len(allTokens)) - unvetted := make([]*backend.Record, 0, len(allTokens)) + var ( + vetted = make([]*backend.Record, 0, len(allTokens)) + unvetted = make([]*backend.Record, 0, len(allTokens)) + ) for _, token := range allTokens { record := records[hex.EncodeToString(token)] if record.RecordMetadata.State == backend.StateVetted { @@ -1092,6 +1094,8 @@ func (t *tstoreBackend) Fsck() error { record.RecordMetadata.Status) } + log.Infof("%v records added to the inventory", len(allTokens)) + // Update all plugin caches return t.tstore.Fsck(allTokens) }