Skip to content

Commit

Permalink
Remove the GetLeavesByIndex method from the log storage interface.
Browse files Browse the repository at this point in the history
Part of #2245.
  • Loading branch information
pphaneuf committed Mar 23, 2021
1 parent ab61c60 commit bb6b437
Show file tree
Hide file tree
Showing 9 changed files with 12 additions and 516 deletions.
96 changes: 0 additions & 96 deletions storage/cloudspanner/log_storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -808,102 +808,6 @@ func (tx *logTX) populateLeafData(ctx context.Context, byHash leavesByHash) erro
return rows.Do(byHash.addRow)
}

// validateIndices ensures that all indices are between 0 and treeSize-1.
func validateIndices(indices []int64, treeSize int64) error {
maxIndex := treeSize - 1
for _, i := range indices {
if i < 0 {
return status.Errorf(codes.InvalidArgument, "index %d is < 0", i)
}
if i > maxIndex {
return status.Errorf(codes.OutOfRange, "index %d is > highest index in current tree %d", i, maxIndex)
}
}
return nil
}

// GetLeavesByIndex returns the leaves corresponding to the given indices.
func (tx *logTX) GetLeavesByIndex(ctx context.Context, indices []int64) ([]*trillian.LogLeaf, error) {
// We need the latest root to validate the indices are within range.
currentSTH, err := tx.currentSTH(ctx)
if err != nil {
return nil, err
}

if err := validateIndices(indices, currentSTH.TreeSize); err != nil {
return nil, err
}

// TODO: replace with INNER JOIN when spannertest supports JOINs
// https://github.com/googleapis/google-cloud-go/tree/master/spanner/spannertest
stmt := spanner.NewStatement(
`SELECT
TreeID,
SequenceNumber,
LeafIdentityHash,
MerkleLeafHash,
IntegrateTimestampNanos
FROM
SequencedLeafData
WHERE
TreeID = @tree_id AND
SequenceNumber IN UNNEST(@seq_nums)`)
stmt.Params["tree_id"] = tx.treeID
stmt.Params["seq_nums"] = indices
seqLeaves := make(map[string]sequencedLeafDataCols)
if err := tx.stx.Query(ctx, stmt).Do(func(r *spanner.Row) error {
var seqLeaf sequencedLeafDataCols
if err := r.ToStruct(&seqLeaf); err != nil {
return err
}
seqLeaves[string(seqLeaf.LeafIdentityHash)] = seqLeaf
return nil
}); err != nil {
return nil, err
}

idHashes := make([][]byte, 0, len(seqLeaves))
for _, l := range seqLeaves {
idHashes = append(idHashes, l.LeafIdentityHash)
}

stmt = spanner.NewStatement(
`SELECT
TreeID,
LeafIdentityHash,
LeafValue,
ExtraData,
QueueTimestampNanos
FROM
SequencedLeafData
WHERE
TreeID = @tree_id AND
LeafIdentityHash IN UNNEST(@id_hashes)`)
stmt.Params["tree_id"] = tx.treeID
stmt.Params["id_hashes"] = idHashes
leaves := make(leafmap)
if err := tx.stx.Query(ctx, stmt).Do(leaves.addFullRow(seqLeaves)); err != nil {
return nil, err
}

// Sanity check that we got everything we wanted
if got, want := len(leaves), len(indices); got != want {
return nil, fmt.Errorf("inconsistency: got %d leaves, want %d", got, want)
}

// Sort the leaves so they are in the same order as the indices.
ret := make([]*trillian.LogLeaf, 0, len(indices))
for _, i := range indices {
l, ok := leaves[i]
if !ok {
return nil, fmt.Errorf("inconsistency: missing data for index %d", i)
}
ret = append(ret, l)
}

return ret, nil
}

func validateRange(start, count, treeSize int64) error {
if count <= 0 {
return status.Errorf(codes.InvalidArgument, "invalid count %d", count)
Expand Down
2 changes: 0 additions & 2 deletions storage/log_storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,6 @@ type ReadOnlyLogTreeTX interface {
// GetSequencedLeafCount returns the total number of leaves that have been integrated into the
// tree via sequencing.
GetSequencedLeafCount(ctx context.Context) (int64, error)
// GetLeavesByIndex returns leaf metadata and data for a set of specified sequenced leaf indexes.
GetLeavesByIndex(ctx context.Context, leaves []int64) ([]*trillian.LogLeaf, error)
// GetLeavesByRange returns leaf data for a range of indexes. The returned
// slice is a contiguous prefix of leaves in [start, start+count) ordered by
// LeafIndex. It will be shorter than `count` if the requested range has
Expand Down
11 changes: 0 additions & 11 deletions storage/memory/log_storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -315,17 +315,6 @@ func (t *logTreeTX) GetSequencedLeafCount(ctx context.Context) (int64, error) {
return sequencedLeafCount, nil
}

func (t *logTreeTX) GetLeavesByIndex(ctx context.Context, leaves []int64) ([]*trillian.LogLeaf, error) {
ret := make([]*trillian.LogLeaf, 0, len(leaves))
for _, seq := range leaves {
leaf := t.tx.Get(seqLeafKey(t.treeID, seq))
if leaf != nil {
ret = append(ret, leaf.(*kv).v.(*trillian.LogLeaf))
}
}
return ret, nil
}

func (t *logTreeTX) GetLeavesByRange(ctx context.Context, start, count int64) ([]*trillian.LogLeaf, error) {
ret := make([]*trillian.LogLeaf, 0, count)
for i := int64(0); i < count; i++ {
Expand Down
30 changes: 0 additions & 30 deletions storage/mock_storage.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

76 changes: 0 additions & 76 deletions storage/mysql/log_storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,6 @@ const (
AND s.SequenceNumber >= ? AND s.SequenceNumber < ? AND l.TreeId = ? AND s.TreeId = l.TreeId` + orderBySequenceNumberSQL

// These statements need to be expanded to provide the correct number of parameter placeholders.
selectLeavesByIndexSQL = `SELECT s.MerkleLeafHash,l.LeafIdentityHash,l.LeafValue,s.SequenceNumber,l.ExtraData,l.QueueTimestampNanos,s.IntegrateTimestampNanos
FROM LeafData l,SequencedLeafData s
WHERE l.LeafIdentityHash = s.LeafIdentityHash
AND s.SequenceNumber IN (` + placeholderSQL + `) AND l.TreeId = ? AND s.TreeId = l.TreeId`
selectLeavesByMerkleHashSQL = `SELECT s.MerkleLeafHash,l.LeafIdentityHash,l.LeafValue,s.SequenceNumber,l.ExtraData,l.QueueTimestampNanos,s.IntegrateTimestampNanos
FROM LeafData l,SequencedLeafData s
WHERE l.LeafIdentityHash = s.LeafIdentityHash
Expand Down Expand Up @@ -151,10 +147,6 @@ func (m *mySQLLogStorage) CheckDatabaseAccessible(ctx context.Context) error {
return m.db.PingContext(ctx)
}

func (m *mySQLLogStorage) getLeavesByIndexStmt(ctx context.Context, num int) (*sql.Stmt, error) {
return m.getStmt(ctx, selectLeavesByIndexSQL, num, "?", "?")
}

func (m *mySQLLogStorage) getLeavesByMerkleHashStmt(ctx context.Context, num int, orderBySequence bool) (*sql.Stmt, error) {
if orderBySequence {
return m.getStmt(ctx, selectLeavesByMerkleHashOrderedBySequenceSQL, num, "?", "?")
Expand Down Expand Up @@ -661,74 +653,6 @@ func (t *logTreeTX) GetSequencedLeafCount(ctx context.Context) (int64, error) {
return sequencedLeafCount, err
}

func (t *logTreeTX) GetLeavesByIndex(ctx context.Context, leaves []int64) ([]*trillian.LogLeaf, error) {
if t.treeType == trillian.TreeType_LOG {
treeSize := int64(t.root.TreeSize)
for _, leaf := range leaves {
if leaf < 0 {
return nil, status.Errorf(codes.InvalidArgument, "index %d is < 0", leaf)
}
if leaf >= treeSize {
return nil, status.Errorf(codes.OutOfRange, "invalid leaf index %d, want < TreeSize(%d)", leaf, treeSize)
}
}
}
tmpl, err := t.ls.getLeavesByIndexStmt(ctx, len(leaves))
if err != nil {
return nil, err
}
stx := t.tx.StmtContext(ctx, tmpl)
defer stx.Close()

var args []interface{}
for _, nodeID := range leaves {
args = append(args, int64(nodeID))
}
args = append(args, t.treeID)
rows, err := stx.QueryContext(ctx, args...)
if err != nil {
glog.Warningf("Failed to get leaves by idx: %s", err)
return nil, err
}
defer rows.Close()

ret := make([]*trillian.LogLeaf, 0, len(leaves))
for rows.Next() {
leaf := &trillian.LogLeaf{}
var qTimestamp, iTimestamp int64
if err := rows.Scan(
&leaf.MerkleLeafHash,
&leaf.LeafIdentityHash,
&leaf.LeafValue,
&leaf.LeafIndex,
&leaf.ExtraData,
&qTimestamp,
&iTimestamp); err != nil {
glog.Warningf("Failed to scan merkle leaves: %s", err)
return nil, err
}
var err error
leaf.QueueTimestamp, err = ptypes.TimestampProto(time.Unix(0, qTimestamp))
if err != nil {
return nil, fmt.Errorf("got invalid queue timestamp: %v", err)
}
leaf.IntegrateTimestamp, err = ptypes.TimestampProto(time.Unix(0, iTimestamp))
if err != nil {
return nil, fmt.Errorf("got invalid integrate timestamp: %v", err)
}
ret = append(ret, leaf)
}
if err := rows.Err(); err != nil {
glog.Warningf("Failed to read returned leaves: %s", err)
return nil, err
}

if got, want := len(ret), len(leaves); got != want {
return nil, status.Errorf(codes.Internal, "len(ret): %d, want %d", got, want)
}
return ret, nil
}

func (t *logTreeTX) GetLeavesByRange(ctx context.Context, start, count int64) ([]*trillian.LogLeaf, error) {
t.treeTX.mu.Lock()
defer t.treeTX.mu.Unlock()
Expand Down
Loading

0 comments on commit bb6b437

Please sign in to comment.