diff --git a/politeiad/backendv2/tstorebe/inventory.go b/politeiad/backendv2/tstorebe/inventory.go index a4c287248..667b48eac 100644 --- a/politeiad/backendv2/tstorebe/inventory.go +++ b/politeiad/backendv2/tstorebe/inventory.go @@ -43,6 +43,16 @@ func (t *tstoreBackend) invPathVetted() string { return filepath.Join(t.dataDir, filenameInvVetted) } +// invRemoveUnvetted removes the unvetted inventory from its respective path. +func (t *tstoreBackend) invRemoveUnvetted() error { + return os.RemoveAll(t.invPathUnvetted()) +} + +// invRemoveVetted removes the vetted inventory from its respective path. +func (t *tstoreBackend) invRemoveVetted() error { + return os.RemoveAll(t.invPathVetted()) +} + // invGetLocked retrieves the inventory from disk. A new inventory is returned // if one does not exist yet. // diff --git a/politeiad/backendv2/tstorebe/tstorebe.go b/politeiad/backendv2/tstorebe/tstorebe.go index 08587dbd9..cc0572432 100644 --- a/politeiad/backendv2/tstorebe/tstorebe.go +++ b/politeiad/backendv2/tstorebe/tstorebe.go @@ -11,6 +11,7 @@ import ( "encoding/json" "fmt" "path/filepath" + "sort" "strconv" "sync" "time" @@ -1017,21 +1018,83 @@ func (t *tstoreBackend) Fsck() error { return err } - // Rebuild the inventory cache. This is a multi-step process: - // - // 1. Sort the tokens into two groups; unvetted and vetted. Each - // group is additionally sorted by the timestamp of their most - // recent status change. - // - // 2. Add the vetted tokens to the inventory cache. They MUST be - // added starting with the oldest timestamp. The vetted token - // is first added to the unvetted cache then moved to the vetted - // cache. This is a limitation of the inventory API and mimicks - // how records are added and updated when using the politeiad - // API. - // - // 3. Add the unvetted tokens to the inventory cache. They MUST be - // added starting with the oldest timestamp. + // Get the partial record for all tokens. This also guarantees that all + // tokens being manipulated actually correspond to a record on the backend. + records := make(map[string]*backend.Record, len(allTokens)) + for _, token := range allTokens { + r, err := t.tstore.RecordPartial(token, 0, nil, true) + if err != nil { + return err + } + records[r.RecordMetadata.Token] = r + } + + // Sort records into vetted and unvetted groups. + var ( + vetted = make([]*backend.Record, 0, len(allTokens)) + unvetted = make([]*backend.Record, 0, len(allTokens)) + ) + for _, token := range allTokens { + record := records[hex.EncodeToString(token)] + if record.RecordMetadata.State == backend.StateVetted { + vetted = append(vetted, record) + } + if record.RecordMetadata.State == backend.StateUnvetted { + unvetted = append(unvetted, record) + } + } + + // Sort records from both groups by the timestamp of their record metadata, + // from oldest to newest. The order of the record inventory will be + // slightly different. On runtime, the timestamp order is through the most + // recent status change metadata. On this fsck rebuild, the order is + // through the record timestamp from their last edit. This happens because + // the record timestamp gets updated on both status changes and edits, so + // the status change timestamp gets lost when the record is edited. + sort.Slice(vetted, func(i, j int) bool { + return vetted[i].RecordMetadata.Timestamp < + vetted[j].RecordMetadata.Timestamp + }) + sort.Slice(unvetted, func(i, j int) bool { + return unvetted[i].RecordMetadata.Timestamp < + unvetted[j].RecordMetadata.Timestamp + }) + + // Now that data is sorted, delete inventory cache before building the new, + // updated one. + err = t.invRemoveVetted() + if err != nil { + return err + } + err = t.invRemoveUnvetted() + if err != nil { + return err + } + + // Add vetted tokens to inventory cache. First add to inventory as + // unvetted, then move to vetted. This is a temporary limitation of the + // inventory API, which was done this way to mimick the way records are + // added and updated on the politeiad API. + for _, record := range vetted { + bToken, err := hex.DecodeString(record.RecordMetadata.Token) + if err != nil { + return err + } + t.inventoryAdd(backend.StateUnvetted, bToken, backend.StatusUnreviewed) + t.inventoryMoveToVetted(bToken, record.RecordMetadata.Status) + } + + // Add unvetted tokens to inventory cache. + for _, record := range unvetted { + bToken, err := hex.DecodeString(record.RecordMetadata.Token) + if err != nil { + return err + } + t.inventoryAdd(record.RecordMetadata.State, bToken, + record.RecordMetadata.Status) + } + + log.Infof("%v records added to the inventory", len(allTokens)) // Update all plugin caches return t.tstore.Fsck(allTokens)