From 381af10c3e410df3b1919db8b200ed8c6a394d99 Mon Sep 17 00:00:00 2001 From: Thiago Figueiredo Date: Tue, 19 Oct 2021 10:36:54 -0300 Subject: [PATCH] Add startrunoff blobs for RFP parents Concurrent parsing for comments/ballot journals on the same prop thread Verify cast vote blobs / auth vote blobs Cleanups in general --- politeiad/cmd/legacyimport/README.md | 9 +- politeiad/cmd/legacyimport/comments.go | 66 +++-- politeiad/cmd/legacyimport/dcrdata.go | 94 ++++++ politeiad/cmd/legacyimport/legacyimport.go | 157 ++++------ politeiad/cmd/legacyimport/ticketvote.go | 328 +++++++++------------ politeiad/cmd/legacyimport/types.go | 10 +- 6 files changed, 353 insertions(+), 311 deletions(-) create mode 100644 politeiad/cmd/legacyimport/dcrdata.go diff --git a/politeiad/cmd/legacyimport/README.md b/politeiad/cmd/legacyimport/README.md index b749a6739..4ddcb8d8d 100644 --- a/politeiad/cmd/legacyimport/README.md +++ b/politeiad/cmd/legacyimport/README.md @@ -5,12 +5,19 @@ into the new tlog backend. This tool will only be used once. ## Considerations -We decided to import only the latest version of each record into tlog, and save it as +- We decided to import only the latest version of each record into tlog, and save it as a version 1/iteration 1 record. If one wishes to check further versions of a finished legacy record, the git repo will be available. +- cast vote signatures cannot be verified using the current politeia public key. +should use "a70134196c3cdf3f85f8af6abaa38c15feb7bccf5e6d3db6212358363465e502". +- vote details medatada cannot be sig verified with the new tlog backend because +of significant data changes ## Usage `leagcyimport`. + + +## \ No newline at end of file diff --git a/politeiad/cmd/legacyimport/comments.go b/politeiad/cmd/legacyimport/comments.go index 7832527d5..f5412d5f0 100644 --- a/politeiad/cmd/legacyimport/comments.go +++ b/politeiad/cmd/legacyimport/comments.go @@ -15,9 +15,9 @@ import ( "github.com/decred/politeia/politeiad/plugins/comments" ) -// convertCommentsJournal walks through the legacy comments journal converting +// parseCommentsJournal walks through the legacy comments journal converting // them to the appropriate plugin payloads for the tstore backend. -func (l *legacyImport) convertCommentsJournal(path, legacyToken string, newToken []byte) error { +func (l *legacyImport) parseCommentsJournal(path, legacyToken string, newToken []byte) error { fh, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0664) if err != nil { return err @@ -25,7 +25,7 @@ func (l *legacyImport) convertCommentsJournal(path, legacyToken string, newToken s := bufio.NewScanner(fh) - // Initialize comments cache + // Initialize comments cache. l.Lock() l.comments[hex.EncodeToString(newToken)] = make(map[string]decredplugin.Comment) l.Unlock() @@ -47,7 +47,7 @@ func (l *legacyImport) convertCommentsJournal(path, legacyToken string, newToken var c decredplugin.Comment err = d.Decode(&c) if err != nil { - return fmt.Errorf("comment journal add: %v", err) + return err } err = l.blobSaveCommentAdd(c, newToken) if err != nil { @@ -61,7 +61,7 @@ func (l *legacyImport) convertCommentsJournal(path, legacyToken string, newToken var cc decredplugin.CensorComment err = d.Decode(&cc) if err != nil { - return fmt.Errorf("comment journal censor: %v", err) + return err } l.RLock() @@ -70,21 +70,20 @@ func (l *legacyImport) convertCommentsJournal(path, legacyToken string, newToken err = l.blobSaveCommentDel(cc, newToken, parentID) if err != nil { - return fmt.Errorf("comment journal del: %v", err) + return err } case "addlike": var lc likeCommentV1 err = d.Decode(&lc) if err != nil { - return fmt.Errorf("comment journal addlike: %v", err) + return err } err = l.blobSaveCommentLike(lc, newToken) if err != nil { return err } default: - return fmt.Errorf("invalid action: %v", - action.Action) + return err } } @@ -95,7 +94,7 @@ func (l *legacyImport) convertCommentsJournal(path, legacyToken string, newToken func (l *legacyImport) blobSaveCommentAdd(c decredplugin.Comment, newToken []byte) error { // Get user id from pubkey - usr, err := l.fetchUserByPubKey(c.PublicKey) + _, err := l.fetchUserByPubKey(c.PublicKey) if err != nil { return err } @@ -110,11 +109,37 @@ func (l *legacyImport) blobSaveCommentAdd(c decredplugin.Comment, newToken []byt return err } + // fmt.Println("before verify comment") + // // Verify comment blob signature + // cv1 := v1.Comment{ + // UserID: usr.ID, + // Username: "", + // State: v1.RecordStateT(comments.RecordStateVetted), + // Token: c.Token, + // ParentID: uint32(pid), + // Comment: c.Comment, + // PublicKey: c.PublicKey, + // Signature: c.Signature, + // CommentID: uint32(cid), + // Timestamp: c.Timestamp, + // Receipt: c.Receipt, + // Downvotes: 0, + // Upvotes: 0, + // Deleted: false, + // Reason: "", + // } + // err = client.CommentVerify(cv1, serverPubkey) + // if err != nil { + // return err + // } + // Create comment add blob entry cn := &comments.CommentAdd{ - UserID: usr.ID, + // UserID: usr.ID, + // Token: hex.EncodeToString(newToken), + UserID: "810aefda-1e13-4ebc-a9e8-4162435eca7b", State: comments.RecordStateVetted, - Token: hex.EncodeToString(newToken), + Token: c.Token, ParentID: uint32(pid), Comment: c.Comment, PublicKey: c.PublicKey, @@ -139,6 +164,9 @@ func (l *legacyImport) blobSaveCommentAdd(c decredplugin.Comment, newToken []byt be := store.NewBlobEntry(hint, data) err = l.tstore.BlobSave(newToken, be) + if err != nil && err.Error() == "duplicate payload" { + return nil + } if err != nil { return err } @@ -148,7 +176,7 @@ func (l *legacyImport) blobSaveCommentAdd(c decredplugin.Comment, newToken []byt func (l *legacyImport) blobSaveCommentDel(cc decredplugin.CensorComment, newToken []byte, parentID string) error { // Get user ID from pubkey - usr, err := l.fetchUserByPubKey(cc.PublicKey) + _, err := l.fetchUserByPubKey(cc.PublicKey) if err != nil { return err } @@ -167,7 +195,7 @@ func (l *legacyImport) blobSaveCommentDel(cc decredplugin.CensorComment, newToke // Create comment del blob entry cd := &comments.CommentDel{ - Token: hex.EncodeToString(newToken), + Token: cc.Token, State: comments.RecordStateVetted, CommentID: uint32(cid), Reason: cc.Reason, @@ -175,7 +203,7 @@ func (l *legacyImport) blobSaveCommentDel(cc decredplugin.CensorComment, newToke Signature: cc.Signature, ParentID: uint32(pid), - UserID: usr.ID, + UserID: "810aefda-1e13-4ebc-a9e8-4162435eca7b", Timestamp: cc.Timestamp, Receipt: cc.Receipt, } @@ -202,7 +230,7 @@ func (l *legacyImport) blobSaveCommentDel(cc decredplugin.CensorComment, newToke func (l *legacyImport) blobSaveCommentLike(lc likeCommentV1, newToken []byte) error { // Get user ID from pubkey - usr, err := l.fetchUserByPubKey(lc.PublicKey) + _, err := l.fetchUserByPubKey(lc.PublicKey) if err != nil { return err } @@ -226,9 +254,9 @@ func (l *legacyImport) blobSaveCommentLike(lc likeCommentV1, newToken []byte) er // Create comment vote blob entry c := &comments.CommentVote{ - UserID: usr.ID, + UserID: "810aefda-1e13-4ebc-a9e8-4162435eca7b", State: comments.RecordStateVetted, - Token: hex.EncodeToString(newToken), + Token: lc.Token, CommentID: uint32(cid), Vote: vote, PublicKey: lc.PublicKey, @@ -251,7 +279,7 @@ func (l *legacyImport) blobSaveCommentLike(lc likeCommentV1, newToken []byte) er } be := store.NewBlobEntry(hint, data) err = l.tstore.BlobSave(newToken, be) - if err != nil && err.Error() == "duplicate blob" { + if err != nil && err.Error() == "duplicate payload" { return nil } if err != nil { diff --git a/politeiad/cmd/legacyimport/dcrdata.go b/politeiad/cmd/legacyimport/dcrdata.go new file mode 100644 index 000000000..3123d3028 --- /dev/null +++ b/politeiad/cmd/legacyimport/dcrdata.go @@ -0,0 +1,94 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + + dcrdata "github.com/decred/dcrdata/v6/api/types" +) + +// Get largest commitment address from dcrdata +func batchTransactions(hashes []string) ([]dcrdata.TrimmedTx, error) { + // Request body is dcrdataapi.Txns marshalled to JSON + reqBody, err := json.Marshal(dcrdata.Txns{ + Transactions: hashes, + }) + if err != nil { + return nil, err + } + + // Make the POST request + url := "https://dcrdata.decred.org/api/txs/trimmed" + r, err := http.Post(url, "application/json; charset=utf-8", + bytes.NewReader(reqBody)) + if err != nil { + return nil, err + } + defer r.Body.Close() + + if r.StatusCode != http.StatusOK { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + return nil, fmt.Errorf("dcrdata error: %v %v %v", + r.StatusCode, url, err) + } + return nil, fmt.Errorf("dcrdata error: %v %v %s", + r.StatusCode, url, body) + } + + // Unmarshal the response + var ttx []dcrdata.TrimmedTx + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&ttx); err != nil { + return nil, err + } + return ttx, nil +} + +func largestCommitmentAddresses(hashes []string) (map[string]largestCommitmentResult, error) { + // Batch request all of the transaction info from dcrdata. + ttxs, err := batchTransactions(hashes) + if err != nil { + return nil, err + } + + // Find largest commitment address for each transaction. + addrs := make(map[string]largestCommitmentResult, len(hashes)) + + for i := range ttxs { + // Best is address with largest commit amount. + var bestAddr string + var bestAmount float64 + for _, v := range ttxs[i].Vout { + if v.ScriptPubKeyDecoded.CommitAmt == nil { + continue + } + if *v.ScriptPubKeyDecoded.CommitAmt > bestAmount { + if len(v.ScriptPubKeyDecoded.Addresses) == 0 { + // jrick, does this need to be printed? + fmt.Errorf("unexpected addresses "+ + "length: %v", ttxs[i].TxID) + continue + } + bestAddr = v.ScriptPubKeyDecoded.Addresses[0] + bestAmount = *v.ScriptPubKeyDecoded.CommitAmt + } + } + + if bestAddr == "" || bestAmount == 0.0 { + addrs[ttxs[i].TxID] = largestCommitmentResult{ + err: fmt.Errorf("no best commitment address found: %v", + ttxs[i].TxID), + } + continue + } + addrs[ttxs[i].TxID] = largestCommitmentResult{ + bestAddr: bestAddr, + } + } + + return addrs, nil +} diff --git a/politeiad/cmd/legacyimport/legacyimport.go b/politeiad/cmd/legacyimport/legacyimport.go index 19e63f7b9..5f85b59a6 100644 --- a/politeiad/cmd/legacyimport/legacyimport.go +++ b/politeiad/cmd/legacyimport/legacyimport.go @@ -8,7 +8,6 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" - "errors" "flag" "fmt" "io/ioutil" @@ -59,8 +58,6 @@ var ( dbPass = flag.String("dbpass", defaultDBPass, "mysql DB pass") commentsf = flag.Bool("comments", false, "parse comments journal") ballot = flag.Bool("ballot", false, "parse ballot journal") - - errorIsRFPSubmission = errors.New("is rfp submission") ) type legacyImport struct { @@ -77,15 +74,17 @@ type legacyImport struct { // submissions and feed the LinkTo metadata field. tokens map[string]string // [legacyToken]newToken - // queue holds the RFP submission record paths that needs to be parsed - // last, when their respective RFP parent has already been inserted. - queue []string - + // rfpParents holds the startRunoffRecord blob of each RFP parent to be + // saved at a later stage of the parsing. rfpParents map[string]*startRunoffRecord + + // versions holds a cache for the record latest version being parsed. + versions map[string]int // [legacyToken]version } +// newLegacyImport returns an initialized legacyImport with an open tstore +// connection and client http. func newLegacyImport() (*legacyImport, error) { - // Initialize tstore instance. ts, err := tstore.New(defaultHomeDir, defaultDataDir, activeNetParams.Params, *tlogHost, *tlogPass, defaultDBType, *dbHost, *dbPass, v1.DefaultTestnetTimeHost, @@ -94,7 +93,6 @@ func newLegacyImport() (*legacyImport, error) { return nil, err } - // Initialize http client to make pi requests. c, err := util.NewHTTPClient(false, "") if err != nil { return nil, err @@ -106,14 +104,15 @@ func newLegacyImport() (*legacyImport, error) { comments: make(map[string]map[string]decredplugin.Comment), tokens: make(map[string]string), rfpParents: make(map[string]*startRunoffRecord), + versions: make(map[string]int), }, nil } // preParsePaths builds an optimized traversal path for the git record // repository. -func preParsePaths(path string) (map[string]string, error) { - // Pre-parse git records folder and get the path for each record's - // latest version. +func (l *legacyImport) preParsePaths(path string) (map[string]string, error) { + // Walk the repository and parse the location of each record's latest + // version. var ( token string version int @@ -147,8 +146,14 @@ func preParsePaths(path string) (map[string]string, error) { versionPath = path } + // Save version path on the paths slice to be returned. paths[token] = versionPath + // Save cache of legacy record latest version. + l.Lock() + l.versions[token] = version + l.Unlock() + return nil }) if err != nil { @@ -213,11 +218,16 @@ func (l *legacyImport) parseRecordData(rootpath string) (*parsedData, error) { if err != nil { return err } + // Get correct record version from cache. The version in 13.metadata.txt + // is not coherent. This makes the signature verify successfully. + l.RLock() + authDetailsMd.Version = uint32(l.versions[authDetailsMd.Token]) + l.RUnlock() } // Build start vote metadata. if info.Name() == "14.metadata.txt" { - startVoteMd, err = convertStartVoteMetadata(path) + startVoteMd, err = l.convertStartVoteMetadata(path) if err != nil { return err } @@ -225,7 +235,7 @@ func (l *legacyImport) parseRecordData(rootpath string) (*parsedData, error) { // Build vote details metadata. if info.Name() == "15.metadata.txt" { - voteDetailsMd, err = convertVoteDetailsMetadata(path, startVoteMd.Starts) + voteDetailsMd, err = convertVoteDetailsMetadata(path, startVoteMd.Starts[0]) if err != nil { return err } @@ -261,22 +271,10 @@ func (l *legacyImport) parseRecordData(rootpath string) (*parsedData, error) { // Parse vote metadata. if pm.LinkTo != "" { - l.RLock() - linkTo := l.tokens[pm.LinkTo] - l.RUnlock() - - if linkTo == "" { - // RFP Parent has not been inserted yet, put this record - // on queue. - l.Lock() - l.queue = append(l.queue, rootpath) - l.Unlock() - return errorIsRFPSubmission - } // Link to RFP parent's new tlog token. - voteMd.LinkTo = linkTo - parentToken = linkTo + voteMd.LinkTo = pm.LinkTo + parentToken = pm.LinkTo } if pm.LinkBy != 0 { voteMd.LinkBy = pm.LinkBy @@ -342,7 +340,8 @@ func (l *legacyImport) parseRecordData(rootpath string) (*parsedData, error) { if voteMd.LinkBy != 0 { l.Lock() l.rfpParents[proposalMd.LegacyToken] = &startRunoffRecord{ - // Submissions: Will be set when all records have been parsed and inserted. + // Submissions: Will be set when all records have been parsed + // and inserted. Mask: voteDetailsMd.Params.Mask, Duration: voteDetailsMd.Params.Duration, QuorumPercentage: voteDetailsMd.Params.QuorumPercentage, @@ -385,20 +384,10 @@ func (l *legacyImport) saveRecordData(data parsedData) ([]byte, error) { return nil, err } - // Save new token to record metadata. + // Save token to status change metadata. + data.statusChangeMd.Token = data.legacyToken + // Save new tstore token to record metadata. data.recordMd.Token = hex.EncodeToString(newToken) - // Save new token to status change metadata. - data.statusChangeMd.Token = hex.EncodeToString(newToken) - - // Check to see if record is RFP. If so, update the RFP parents cache with - // the new tlog token. - l.Lock() - _, ok := l.rfpParents[data.legacyToken] - if ok { - l.rfpParents[hex.EncodeToString(newToken)] = l.rfpParents[data.legacyToken] - delete(l.rfpParents, data.legacyToken) - } - l.Unlock() // Check if record in question is an RFP submission. If so, add it to the // submissions list of its parent. @@ -436,28 +425,22 @@ func (l *legacyImport) saveRecordData(data parsedData) ([]byte, error) { // Save vote details blob, if any. if data.voteDetailsMd != nil { - if data.parentToken != "" { - data.voteDetailsMd.Params.Parent = data.parentToken - } err = l.blobSaveVoteDetails(*data.voteDetailsMd, newToken) if err != nil { return nil, err } } - // TODO: Concurrently parse comments and votes for each record. - // wait group for both routines. - var wg sync.WaitGroup // Spin routine for parsing comments. wg.Add(1) go func() error { defer wg.Done() - // Navigate and convert comments journal. if data.commentsPath != "" && *commentsf { - err = l.convertCommentsJournal(data.commentsPath, data.legacyToken, newToken) + err = l.parseCommentsJournal(data.commentsPath, data.legacyToken, + newToken) if err != nil { - return err + panic(err) } } return nil @@ -466,20 +449,17 @@ func (l *legacyImport) saveRecordData(data parsedData) ([]byte, error) { wg.Add(1) go func() error { defer wg.Done() - // Navigate and convert vote ballot journal. if data.ballotPath != "" && *ballot { - err = l.convertBallotJournal(data.ballotPath, data.legacyToken, newToken) + err = l.parseBallotJournal(data.ballotPath, data.legacyToken, + newToken) if err != nil { - return err + panic(err) } } return nil }() - wg.Wait() - fmt.Println("DONE PARSING COMMENTS&VOTES CONCURRENTLY") - // Save legacy token to new token mapping in cache. l.Lock() l.tokens[data.legacyToken] = hex.EncodeToString(newToken) @@ -509,27 +489,25 @@ func _main() error { return err } - fmt.Println("legacyimport: Pre parsing record paths...") - - paths, err := preParsePaths(path) + paths, err := l.preParsePaths(path) if err != nil { return err } - fmt.Printf("legacyimport: Pre parsing complete, parsing %v records...\n", len(paths)) + fmt.Printf("legacyimport: Parsing %v records...\n", len(paths)) - // First we parse and add all records that are not RFP submissions to - // tstore. This is done because their respective RFP parent needs to be - // added first in order to link to the new tlog token correctly. + // Parse and add all records that are not RFP submissions to tstore. + // This is done because their respective RFP parent needs to be added + // first in order to link to the new tlog token correctly. i := 0 var wg sync.WaitGroup for _, path := range paths { pData, err := l.parseRecordData(path) - if err == errorIsRFPSubmission { - // This record is an RFP submission and is on queue, will be parsed - // last. - continue - } + // if err == errorIsRFPSubmission { + // // This record is an RFP submission and is on queue, will be parsed + // // last. + // continue + // } if err != nil { return err } @@ -558,45 +536,12 @@ func _main() error { fmt.Println("legacyimport: Done parsing first batch!") - fmt.Printf("legacyimport: Parsing %v on queue records...\n", len(l.queue)) - - // Now we parse the remaining RFP submission proposals. - i = 0 - var qwg sync.WaitGroup - for _, path = range l.queue { - pData, err := l.parseRecordData(path) - if err != nil { - return err - } - - fmt.Printf("legacyimport: Parsing queued record %v on thread %v\n", - pData.recordMd.Token, i) - - i++ - qwg.Add(1) - go func(data parsedData) error { - defer qwg.Done() - - // Save legacy record on tstore. - newToken, err := l.saveRecordData(data) - if err != nil { - return err - } - - fmt.Printf("legacyimport: Parsed record %v. new tlog token: %v\n", - data.legacyToken, hex.EncodeToString(newToken)) - - return nil - }(*pData) - } - - qwg.Wait() - - // Now we add the dataDescriptorStartRunoff blob for each RFP parent while - // building the submissions list. + // Add the dataDescriptorStartRunoff blob for each RFP parent after + // the submissions list has been built. l.RLock() for token, startRunoffRecord := range l.rfpParents { - b, err := hex.DecodeString(token) + tlogToken := l.tokens[token] + b, err := hex.DecodeString(tlogToken) if err != nil { return err } diff --git a/politeiad/cmd/legacyimport/ticketvote.go b/politeiad/cmd/legacyimport/ticketvote.go index 289bd5e29..69021d0d6 100644 --- a/politeiad/cmd/legacyimport/ticketvote.go +++ b/politeiad/cmd/legacyimport/ticketvote.go @@ -7,124 +7,17 @@ import ( "encoding/json" "fmt" "io/ioutil" - "net/http" "os" "strconv" - dcrdata "github.com/decred/dcrdata/v6/api/types" "github.com/decred/politeia/politeiad/backend/gitbe" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" tv "github.com/decred/politeia/politeiad/plugins/ticketvote" + v1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" + "github.com/decred/politeia/politeiawww/client" ) -// convertAuthDetailsMetadata converts the 13.metadata.txt file to the -// auth details structure from tlog backend. -func convertAuthDetailsMetadata(path string) (*tv.AuthDetails, error) { - b, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - var av authorizeVoteV1 - err = json.Unmarshal(b, &av) - if err != nil { - return nil, err - } - - return &tv.AuthDetails{ - Token: av.Token, - Version: uint32(av.Version), - Action: av.Action, - PublicKey: av.PublicKey, - Signature: av.Signature, - Timestamp: av.Timestamp, - Receipt: av.Receipt, - }, nil -} - -// convertStartVoteMetadata converts the 14.metadata.txt file to the start -// details structure from tlog backend. This is used further to populate some -// fields of vote details before saving its blob to the tstore. -func convertStartVoteMetadata(path string) (*tv.Start, error) { - b, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - var av startVoteV1 - err = json.Unmarshal(b, &av) - if err != nil { - return nil, err - } - - var opts []tv.VoteOption - for _, v := range av.Vote.Options { - opts = append(opts, tv.VoteOption{ - ID: v.Id, - Description: v.Description, - Bit: v.Bits, - }) - } - - return &tv.Start{ - Starts: []tv.StartDetails{ - { - Params: tv.VoteParams{ - Token: av.Vote.Token, - Version: uint32(av.Version), - Type: tv.VoteT(av.Vote.Type), - Mask: av.Vote.Mask, - Duration: av.Vote.Duration, - QuorumPercentage: av.Vote.QuorumPercentage, - PassPercentage: av.Vote.PassPercentage, - Options: opts, - Parent: "", // This is set at the end of parsing. - }, - PublicKey: av.PublicKey, - Signature: av.Signature, - }, - }, - }, nil -} - -// convertVoteDetailsMetadata converts the 15.metadata.txt file to the vote -// details structure from tlog backend. -// TODO: review. see how its done on ticketvote/cmds.go. how vote details is saved for RFP subs -func convertVoteDetailsMetadata(path string, startDetails []tv.StartDetails) (*tv.VoteDetails, error) { - b, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - var vd voteDetailsV1 - err = json.Unmarshal(b, &vd) - if err != nil { - return nil, err - } - - // Parse block height - sbh, err := strconv.Atoi(vd.StartBlockHeight) - if err != nil { - return nil, err - } - - ebh, err := strconv.Atoi(vd.EndHeight) - if err != nil { - return nil, err - } - - return &tv.VoteDetails{ - Params: startDetails[0].Params, - PublicKey: startDetails[0].PublicKey, - Signature: startDetails[0].Signature, - - StartBlockHeight: uint32(sbh), - StartBlockHash: vd.StartBlockHash, - EndBlockHeight: uint32(ebh), - EligibleTickets: vd.EligibleTickets, - }, nil -} - -// convertBallotJournal walks the ballot journal, parsing the entries data and -// saving their respective blob on tstore. -func (l *legacyImport) convertBallotJournal(path, legacyToken string, newToken []byte) error { +func (l *legacyImport) parseBallotJournal(path, legacyToken string, newToken []byte) error { fh, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0664) if err != nil { return err @@ -156,7 +49,9 @@ func (l *legacyImport) convertBallotJournal(path, legacyToken string, newToken [ tickets = append(tickets, cvj.CastVote.Ticket) castVoteDetails = append(castVoteDetails, &tv.CastVoteDetails{ - Token: hex.EncodeToString(newToken), + // Keep legacy token on cast vote detail blob so that we can + // verify the signature. + Token: cvj.CastVote.Token, Ticket: cvj.CastVote.Ticket, VoteBit: cvj.CastVote.VoteBit, Signature: cvj.CastVote.Signature, @@ -172,17 +67,18 @@ func (l *legacyImport) convertBallotJournal(path, legacyToken string, newToken [ fmt.Println(" ticketvote: Fetching largest commitment addresses from dcrdata...") - lcr, err := largestCommitmentAddresses(tickets) + addrs, err := largestCommitmentAddresses(tickets) if err != nil { panic(err) } fmt.Printf(" ticketvote: Saving ticketvote blobs to tstore for %v...\n", legacyToken) - for k := range castVoteDetails { - // Save cast vote details blob to tstore. - cv := castVoteDetails[k] // vote details - cv.Address = lcr[k].bestAddr // largest commitment address + for _, details := range castVoteDetails { + cv := details + cv.Address = addrs[cv.Ticket].bestAddr + + // Save cast vote details to tstore. err = l.blobSaveCastVoteDetails(*cv, newToken) if err != nil { return err @@ -204,8 +100,14 @@ func (l *legacyImport) convertBallotJournal(path, legacyToken string, newToken [ return nil } -func (l *legacyImport) blobSaveCastVoteDetails(castVoteDetails tv.CastVoteDetails, newToken []byte) error { - data, err := json.Marshal(castVoteDetails) +func (l *legacyImport) blobSaveCastVoteDetails(cdv tv.CastVoteDetails, newToken []byte) error { + // Verify cast vote details signature. + err := client.CastVoteDetailsVerify(convertCastVoteDetailsToV1(cdv), serverPubkey) + if err != nil { + return err + } + + data, err := json.Marshal(cdv) if err != nil { return err } @@ -220,7 +122,7 @@ func (l *legacyImport) blobSaveCastVoteDetails(castVoteDetails tv.CastVoteDetail be := store.NewBlobEntry(hint, data) err = l.tstore.BlobSave(newToken, be) - if err != nil && err.Error() == "duplicate blob" { + if err != nil && err.Error() == "duplicate payload" { return nil } if err != nil { @@ -231,8 +133,15 @@ func (l *legacyImport) blobSaveCastVoteDetails(castVoteDetails tv.CastVoteDetail } func (l *legacyImport) blobSaveAuthDetails(authDetails tv.AuthDetails, newToken []byte) error { - // Set new tlog token to auth details. - authDetails.Token = hex.EncodeToString(newToken) + // // Set new tlog token to auth details. + // authDetails.Token = hex.EncodeToString(newToken) + + // Verify auth details signature. + err := client.AuthDetailsVerify(convertAuthDetailsToV1(authDetails), + serverPubkey) + if err != nil { + return err + } data, err := json.Marshal(authDetails) if err != nil { @@ -257,8 +166,11 @@ func (l *legacyImport) blobSaveAuthDetails(authDetails tv.AuthDetails, newToken } func (l *legacyImport) blobSaveVoteDetails(voteDetails tv.VoteDetails, newToken []byte) error { - // Set new tlog token to vote details params. - voteDetails.Params.Token = hex.EncodeToString(newToken) + // Vote details blob is a combination of parsing the 14.metadata.txt and + // 15.metadata.txt. Therefore, the tool needs to verify the start vote + // signature, which comes from the 14.metadata.txt file, instead of the + // vote details blob signature. This way, the signature verification + // process is the same as in the git legacy backend. data, err := json.Marshal(voteDetails) if err != nil { @@ -328,86 +240,134 @@ func (l *legacyImport) blobSaveStartRunoff(srr startRunoffRecord, newToken []byt return nil } -// Get largest commitment address from dcrdata -func batchTransactions(hashes []string) ([]dcrdata.TrimmedTx, error) { - // Request body is dcrdataapi.Txns marshalled to JSON - reqBody, err := json.Marshal(dcrdata.Txns{ - Transactions: hashes, - }) +// convertAuthDetailsMetadata converts the 13.metadata.txt file to the +// auth details structure from tlog backend. +func convertAuthDetailsMetadata(path string) (*tv.AuthDetails, error) { + b, err := ioutil.ReadFile(path) if err != nil { return nil, err } + var av authorizeVoteV1 + err = json.Unmarshal(b, &av) + if err != nil { + return nil, err + } + + return &tv.AuthDetails{ + Token: av.Token, + Version: uint32(av.Version), + Action: av.Action, + PublicKey: av.PublicKey, + Signature: av.Signature, + Timestamp: av.Timestamp, + Receipt: av.Receipt, + }, nil +} - // Make the POST request - url := "https://dcrdata.decred.org/api/txs/trimmed" - r, err := http.Post(url, "application/json; charset=utf-8", - bytes.NewReader(reqBody)) +// convertStartVoteMetadata converts the 14.metadata.txt file to the start +// details struct from tstore backend. This is used further to populate data +// of the vote details blob. +func (l *legacyImport) convertStartVoteMetadata(path string) (*tv.Start, error) { + b, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + var av startVoteV1 + err = json.Unmarshal(b, &av) if err != nil { return nil, err } - defer r.Body.Close() - if r.StatusCode != http.StatusOK { - body, err := ioutil.ReadAll(r.Body) - if err != nil { - return nil, fmt.Errorf("dcrdata error: %v %v %v", - r.StatusCode, url, err) - } - return nil, fmt.Errorf("dcrdata error: %v %v %s", - r.StatusCode, url, body) + var opts []tv.VoteOption + for _, v := range av.Vote.Options { + opts = append(opts, tv.VoteOption{ + ID: v.Id, + Description: v.Description, + Bit: v.Bits, + }) } - // Unmarshal the response - var ttx []dcrdata.TrimmedTx - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&ttx); err != nil { - return nil, err + // Some 14.metadata.txt files before RFP feature do not contain the vote + // type information. Define it manually here. + if av.Vote.Type == 0 { + av.Vote.Type = int(tv.VoteTypeStandard) } - return ttx, nil -} -// largestCommitmentResult returns the largest commitment address or an error. -type largestCommitmentResult struct { - bestAddr string - err error + return &tv.Start{ + Starts: []tv.StartDetails{ + { + Params: tv.VoteParams{ + Token: av.Vote.Token, + Version: uint32(av.Version), + Type: tv.VoteT(av.Vote.Type), + Mask: av.Vote.Mask, + Duration: av.Vote.Duration, + QuorumPercentage: av.Vote.QuorumPercentage, + PassPercentage: av.Vote.PassPercentage, + Options: opts, + }, + PublicKey: av.PublicKey, + Signature: av.Signature, + }, + }, + }, nil } -func largestCommitmentAddresses(hashes []string) ([]largestCommitmentResult, error) { - // Batch request all of the transaction info from dcrdata. - ttxs, err := batchTransactions(hashes) +// convertVoteDetailsMetadata converts the 15.metadata.txt file to the vote +// details structure from tlog backend. +func convertVoteDetailsMetadata(path string, startDetails tv.StartDetails) (*tv.VoteDetails, error) { + b, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + var vd voteDetailsV1 + err = json.Unmarshal(b, &vd) if err != nil { return nil, err } - // Find largest commitment address for each transaction. - r := make([]largestCommitmentResult, len(hashes)) - for i := range ttxs { - // Best is address with largest commit amount. - var bestAddr string - var bestAmount float64 - for _, v := range ttxs[i].Vout { - if v.ScriptPubKeyDecoded.CommitAmt == nil { - continue - } - if *v.ScriptPubKeyDecoded.CommitAmt > bestAmount { - if len(v.ScriptPubKeyDecoded.Addresses) == 0 { - // jrick, does this need to be printed? - fmt.Errorf("unexpected addresses "+ - "length: %v", ttxs[i].TxID) - continue - } - bestAddr = v.ScriptPubKeyDecoded.Addresses[0] - bestAmount = *v.ScriptPubKeyDecoded.CommitAmt - } - } + // Parse start height and end height. + sbh, err := strconv.Atoi(vd.StartBlockHeight) + if err != nil { + return nil, err + } + ebh, err := strconv.Atoi(vd.EndHeight) + if err != nil { + return nil, err + } - if bestAddr == "" || bestAmount == 0.0 { - r[i].err = fmt.Errorf("no best commitment address found: %v", - ttxs[i].TxID) - continue - } - r[i].bestAddr = bestAddr + return &tv.VoteDetails{ + Params: startDetails.Params, + PublicKey: startDetails.PublicKey, + Signature: startDetails.Signature, + + StartBlockHeight: uint32(sbh), + StartBlockHash: vd.StartBlockHash, + EndBlockHeight: uint32(ebh), + EligibleTickets: vd.EligibleTickets, + }, nil +} + +func convertAuthDetailsToV1(auth tv.AuthDetails) v1.AuthDetails { + return v1.AuthDetails{ + Token: auth.Token, + Version: auth.Version, + Action: auth.Action, + PublicKey: auth.PublicKey, + Signature: auth.Signature, + Timestamp: auth.Timestamp, + Receipt: auth.Receipt, } +} - return r, nil +func convertCastVoteDetailsToV1(vote tv.CastVoteDetails) v1.CastVoteDetails { + return v1.CastVoteDetails{ + Token: vote.Token, + Ticket: vote.Ticket, + VoteBit: vote.VoteBit, + Address: vote.Address, + Signature: vote.Signature, + Receipt: vote.Receipt, + Timestamp: vote.Timestamp, + } } diff --git a/politeiad/cmd/legacyimport/types.go b/politeiad/cmd/legacyimport/types.go index f566e7fbb..be438b1a6 100644 --- a/politeiad/cmd/legacyimport/types.go +++ b/politeiad/cmd/legacyimport/types.go @@ -7,6 +7,8 @@ import ( "github.com/decred/politeia/politeiad/plugins/usermd" ) +const serverPubkey = "a70134196c3cdf3f85f8af6abaa38c15feb7bccf5e6d3db6212358363465e502" + // parsedData holds the data needed by tlog to insert the legacy // records on tstore. type parsedData struct { @@ -133,7 +135,7 @@ type params struct { WalletRPCServerPort string } -// Types for pi API interaction +// Types for external API (Pi and Dcrdata) type user struct { ID string `json:"id"` Email string `json:"email,omitempty"` @@ -144,3 +146,9 @@ type usersReply struct { TotalMatches uint64 `json:"totalmatches"` Users []user `json:"users"` } + +// largestCommitmentResult returns the largest commitment address or an error. +type largestCommitmentResult struct { + bestAddr string + err error +}