tstorebe: Add record inventory fsck.

politeiad/backendv2/tstorebe/inventory.go

@@ -43,6 +43,16 @@ func (t *tstoreBackend) invPathVetted() string {
 	return filepath.Join(t.dataDir, filenameInvVetted)
 }
 
+// invRemoveUnvetted removes the unvetted inventory from its respective path.
+func (t *tstoreBackend) invRemoveUnvetted() error {
+	return os.RemoveAll(t.invPathUnvetted())
+}
+
+// invRemoveVetted removes the vetted inventory from its respective path.
+func (t *tstoreBackend) invRemoveVetted() error {
+	return os.RemoveAll(t.invPathVetted())
+}
+
 // invGetLocked retrieves the inventory from disk. A new inventory is returned
 // if one does not exist yet.
 //

politeiad/backendv2/tstorebe/tstorebe.go

@@ -11,6 +11,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"path/filepath"
+	"sort"
 	"strconv"
 	"sync"
 	"time"
@@ -1017,21 +1018,83 @@ func (t *tstoreBackend) Fsck() error {
 		return err
 	}
 
-	// Rebuild the inventory cache. This is a multi-step process:
-	//
-	// 1. Sort the tokens into two groups; unvetted and vetted. Each
-	//    group is additionally sorted by the timestamp of their most
-	//    recent status change.
-	//
-	// 2. Add the vetted tokens to the inventory cache. They MUST be
-	//    added starting with the oldest timestamp. The vetted token
-	//    is first added to the unvetted cache then moved to the vetted
-	//    cache. This is a limitation of the inventory API and mimicks
-	//    how records are added and updated when using the politeiad
-	//    API.
-	//
-	// 3. Add the unvetted tokens to the inventory cache. They MUST be
-	//    added starting with the oldest timestamp.
+	// Get the partial record for all tokens. This also guarantees that all
+	// tokens being manipulated actually correspond to a record on the backend.
+	records := make(map[string]*backend.Record, len(allTokens))
+	for _, token := range allTokens {
+		r, err := t.tstore.RecordPartial(token, 0, nil, true)
+		if err != nil {
+			return err
+		}
+		records[r.RecordMetadata.Token] = r
+	}
+
+	// Sort records into vetted and unvetted groups.
+	var (
+		vetted   = make([]*backend.Record, 0, len(allTokens))
+		unvetted = make([]*backend.Record, 0, len(allTokens))
+	)
+	for _, token := range allTokens {
+		record := records[hex.EncodeToString(token)]
+		if record.RecordMetadata.State == backend.StateVetted {
+			vetted = append(vetted, record)
+		}
+		if record.RecordMetadata.State == backend.StateUnvetted {
+			unvetted = append(unvetted, record)
+		}
+	}
+
+	// Sort records from both groups by the timestamp of their record metadata,
+	// from oldest to newest. The order of the record inventory will be
+	// slightly different. On runtime, the timestamp order is through the most
+	// recent status change metadata. On this fsck rebuild, the order is
+	// through the record timestamp from their last edit. This happens because
+	// the record timestamp gets updated on both status changes and edits, so
+	// the status change timestamp gets lost when the record is edited.
+	sort.Slice(vetted, func(i, j int) bool {
+		return vetted[i].RecordMetadata.Timestamp <
+			vetted[j].RecordMetadata.Timestamp
+	})
+	sort.Slice(unvetted, func(i, j int) bool {
+		return unvetted[i].RecordMetadata.Timestamp <
+			unvetted[j].RecordMetadata.Timestamp
+	})
+
+	// Now that data is sorted, delete inventory cache before building the new,
+	// updated one.
+	err = t.invRemoveVetted()
+	if err != nil {
+		return err
+	}
+	err = t.invRemoveUnvetted()
+	if err != nil {
+		return err
+	}
+
+	// Add vetted tokens to inventory cache. First add to inventory as
+	// unvetted, then move to vetted. This is a temporary limitation of the
+	// inventory API, which was done this way to mimick the way records are
+	// added and updated on the politeiad API.
+	for _, record := range vetted {
+		bToken, err := hex.DecodeString(record.RecordMetadata.Token)
+		if err != nil {
+			return err
+		}
+		t.inventoryAdd(backend.StateUnvetted, bToken, backend.StatusUnreviewed)
+		t.inventoryMoveToVetted(bToken, record.RecordMetadata.Status)
+	}
+
+	// Add unvetted tokens to inventory cache.
+	for _, record := range unvetted {
+		bToken, err := hex.DecodeString(record.RecordMetadata.Token)
+		if err != nil {
+			return err
+		}
+		t.inventoryAdd(record.RecordMetadata.State, bToken,
+			record.RecordMetadata.Status)
+	}
+
+	log.Infof("%v records added to the inventory", len(allTokens))
 
 	// Update all plugin caches
 	return t.tstore.Fsck(allTokens)