Epoch commitment metrics (#2293)

Co-authored-by: Daria Dziubałtowska <daria.dziubaltowska@iota.org>
Co-authored-by: Andrea V <1577639+karimodm@users.noreply.github.com>
Co-authored-by: Daria Dziubałtowska <44535712+daria305@users.noreply.github.com>
Co-authored-by: Ching-Hua (Vivian) Lin <jkrvivian@gmail.com>

packages/jsonmodels/epochs.go

@@ -0,0 +1,48 @@
+package jsonmodels
+
+import (
+	"github.com/iotaledger/goshimmer/packages/epoch"
+)
+
+type EpochInfo struct {
+	EI     uint64 `json:"EI"`
+	ECR    string `json:"ECR"`
+	PrevEC string `json:"prevEC"`
+}
+
+func EpochInfoFromRecord(record *epoch.ECRecord) *EpochInfo {
+	return &EpochInfo{
+		EI:     uint64(record.EI()),
+		ECR:    record.ECR().Base58(),
+		PrevEC: record.PrevEC().Base58(),
+	}
+}
+
+type EpochsResponse struct {
+	Epochs []*EpochInfo `json:"epochs"`
+}
+
+type EpochVotersWeightResponse struct {
+	VotersWeight map[string]*NodeWeight `json:"ecrVoters"`
+}
+
+type NodeWeight struct {
+	Weights map[string]float64 `json:"weights"`
+}
+
+type EpochUTXOsResponse struct {
+	SpentOutputs   []string `json:"spentOutputs"`
+	CreatedOutputs []string `json:"createdOutputs"`
+}
+
+type EpochMessagesResponse struct {
+	Messages []string `json:"messages"`
+}
+
+type EpochTransactionsResponse struct {
+	Transactions []string `json:"transactions"`
+}
+
+type EpochPendingBranchCountResponse struct {
+	PendingBranchCount uint64 `json:"pendingBranchCount"`
+}

packages/jsonmodels/info.go

@@ -32,6 +32,8 @@ type InfoResponse struct {
 	Mana Mana `json:"mana,omitempty"`
 	// Scheduler is the scheduler.
 	Scheduler Scheduler `json:"scheduler"`
+	// LastCommittedEpoch contains information about the last committed epoch.
+	LastCommittedEpoch EpochInfo
 	// RateSetter is the rate setter.
 	RateSetter RateSetter `json:"rateSetter"`
 	// error of the response

packages/ledger/utxo/types.go

@@ -101,6 +101,14 @@ func (o *OutputID) FromRandomness() (err error) {
 	return nil
 }
 
+// FromBytes un-serializes an OutputID from a []byte.
+func (o *OutputID) FromBytes(outputBytes []byte) (err error) {
+	if _, err := serix.DefaultAPI.Decode(context.Background(), outputBytes, o, serix.WithValidation()); err != nil {
+		return errors.Errorf("Fail to parse outputID from bytes: %w", err)
+	}
+	return nil
+}
+
 // RegisterAlias allows to register a human-readable alias for the OutputID which will be used as a replacement for the
 // String method.
 func (o OutputID) RegisterAlias(alias string) {

packages/notarization/manager.go

@@ -52,8 +52,14 @@ func NewManager(epochCommitmentFactory *EpochCommitmentFactory, t *tangle.Tangle
 		log:                      options.Log,
 		options:                  options,
 		Events: &Events{
-			EpochCommittable: event.New[*EpochCommittableEvent](),
-			ManaVectorUpdate: event.New[*ManaVectorUpdateEvent](),
+			TangleTreeInserted:        event.New[*TangleTreeUpdatedEvent](),
+			TangleTreeRemoved:         event.New[*TangleTreeUpdatedEvent](),
+			StateMutationTreeInserted: event.New[*StateMutationTreeUpdatedEvent](),
+			StateMutationTreeRemoved:  event.New[*StateMutationTreeUpdatedEvent](),
+			UTXOTreeInserted:          event.New[*UTXOUpdatedEvent](),
+			UTXOTreeRemoved:           event.New[*UTXOUpdatedEvent](),
+			EpochCommittable:          event.New[*EpochCommittableEvent](),
+			ManaVectorUpdate:          event.New[*ManaVectorUpdateEvent](),
 		},
 	}
 
@@ -193,7 +199,9 @@ func (m *Manager) OnMessageConfirmed(message *tangle.Message) {
 	err := m.epochCommitmentFactory.insertTangleLeaf(ei, message.ID())
 	if err != nil && m.log != nil {
 		m.log.Error(err)
+		return
 	}
+	m.Events.TangleTreeInserted.Trigger(&TangleTreeUpdatedEvent{EI: ei, MessageID: message.ID()})
 }
 
 // OnMessageOrphaned is the handler for message orphaned event.
@@ -210,10 +218,13 @@ func (m *Manager) OnMessageOrphaned(message *tangle.Message) {
 	if err != nil && m.log != nil {
 		m.log.Error(err)
 	}
+	m.Events.TangleTreeRemoved.Trigger(&TangleTreeUpdatedEvent{EI: ei, MessageID: message.ID()})
+
 	transaction, isTransaction := message.Payload().(utxo.Transaction)
 	if isTransaction {
 		spent, created := m.resolveOutputs(transaction)
 		m.epochCommitmentFactory.deleteDiffUTXOs(ei, created, spent)
+		m.Events.UTXOTreeRemoved.Trigger(&UTXOUpdatedEvent{EI: ei, Spent: spent, Created: created})
 	}
 }
 
@@ -281,7 +292,6 @@ func (m *Manager) OnTransactionInclusionUpdated(event *ledger.TransactionInclusi
 func (m *Manager) OnBranchConfirmed(branchID utxo.TransactionID) {
 	m.epochCommitmentFactoryMutex.Lock()
 	defer m.epochCommitmentFactoryMutex.Unlock()
-
 	ei := m.getBranchEI(branchID, true)
 
 	if m.isEpochAlreadyCommitted(ei) {
@@ -340,6 +350,20 @@ func (m *Manager) OnAcceptanceTimeUpdated(newTime time.Time) {
 	}
 }
 
+// PendingConflictsCount returns the current value of pendingConflictsCount.
+func (m *Manager) PendingConflictsCount(ei epoch.Index) (pendingConflictsCount uint64) {
+	return m.pendingConflictsCounters[ei]
+}
+
+// PendingConflictsCountAll returns the current value of pendingConflictsCount per epoch.
+func (m *Manager) PendingConflictsCountAll() (pendingConflicts map[epoch.Index]uint64) {
+	pendingConflicts = make(map[epoch.Index]uint64, len(m.pendingConflictsCounters))
+	for k, v := range m.pendingConflictsCounters {
+		pendingConflicts[k] = v
+	}
+	return pendingConflicts
+}
+
 // Shutdown shuts down the manager's permanent storagee.
 func (m *Manager) Shutdown() {
 	m.epochCommitmentFactoryMutex.Lock()
@@ -363,19 +387,23 @@ func (m *Manager) includeTransactionInEpoch(txID utxo.TransactionID, ei epoch.In
 	if err := m.epochCommitmentFactory.insertStateMutationLeaf(ei, txID); err != nil {
 		return err
 	}
-
 	m.epochCommitmentFactory.storeDiffUTXOs(ei, spent, created)
 
+	m.Events.StateMutationTreeInserted.Trigger(&StateMutationTreeUpdatedEvent{TransactionID: txID})
+	m.Events.UTXOTreeInserted.Trigger(&UTXOUpdatedEvent{EI: ei, Spent: spent, Created: created})
+
 	return nil
 }
 
 func (m *Manager) removeTransactionFromEpoch(txID utxo.TransactionID, ei epoch.Index, spent, created []*ledger.OutputWithMetadata) (err error) {
 	if err := m.epochCommitmentFactory.removeStateMutationLeaf(ei, txID); err != nil {
 		return err
 	}
-
 	m.epochCommitmentFactory.deleteDiffUTXOs(ei, spent, created)
 
+	m.Events.StateMutationTreeRemoved.Trigger(&StateMutationTreeUpdatedEvent{TransactionID: txID})
+	m.Events.UTXOTreeRemoved.Trigger(&UTXOUpdatedEvent{EI: ei, Spent: spent, Created: created})
+
 	return nil
 }
 
@@ -481,7 +509,8 @@ func (m *Manager) moveLatestCommittableEpoch(currentEpoch epoch.Index) {
 
 		// reads the roots and store the ec
 		// rolls the state trees
-		if _, ecRecordErr := m.epochCommitmentFactory.ecRecord(ei); ecRecordErr != nil {
+		ecRecord, ecRecordErr := m.epochCommitmentFactory.ecRecord(ei)
+		if ecRecordErr != nil {
 			m.log.Errorf("could not update commitments for epoch %d: %v", ei, ecRecordErr)
 			return
 		}
@@ -491,7 +520,7 @@ func (m *Manager) moveLatestCommittableEpoch(currentEpoch epoch.Index) {
 			return
 		}
 
-		m.Events.EpochCommittable.Trigger(&EpochCommittableEvent{EI: ei})
+		m.Events.EpochCommittable.Trigger(&EpochCommittableEvent{EI: ei, ECRecord: ecRecord})
 		m.triggerManaVectorUpdate(ei)
 	}
 }
@@ -540,12 +569,52 @@ type Events struct {
 	// EpochCommittable is an event that gets triggered whenever an epoch commitment is committable.
 	EpochCommittable *event.Event[*EpochCommittableEvent]
 	ManaVectorUpdate *event.Event[*ManaVectorUpdateEvent]
+	// TangleTreeInserted is an event that gets triggered when a Message is inserted into the Tangle smt.
+	TangleTreeInserted *event.Event[*TangleTreeUpdatedEvent]
+	// TangleTreeRemoved is an event that gets triggered when a Message is removed from Tangle smt.
+	TangleTreeRemoved *event.Event[*TangleTreeUpdatedEvent]
+	// StateMutationTreeInserted is an event that gets triggered when a transaction is inserted into the state mutation smt.
+	StateMutationTreeInserted *event.Event[*StateMutationTreeUpdatedEvent]
+	// StateMutationTreeRemoved is an event that gets triggered when a transaction is removed from state mutation smt.
+	StateMutationTreeRemoved *event.Event[*StateMutationTreeUpdatedEvent]
+	// UTXOTreeInserted is an event that gets triggered when UTXOs are stored into the UTXO smt.
+	UTXOTreeInserted *event.Event[*UTXOUpdatedEvent]
+	// UTXOTreeRemoved is an event that gets triggered when UTXOs are removed from the UTXO smt.
+	UTXOTreeRemoved *event.Event[*UTXOUpdatedEvent]
+}
+
+// TangleTreeUpdatedEvent is a container that acts as a dictionary for the TangleTree inserted/removed event related parameters.
+type TangleTreeUpdatedEvent struct {
+	// EI is the index of the message.
+	EI epoch.Index
+	// MessageID is the messageID that inserted/removed to/from the tangle smt.
+	MessageID tangle.MessageID
+}
+
+// StateMutationTreeUpdatedEvent is a container that acts as a dictionary for the State mutation tree inserted/removed event related parameters.
+type StateMutationTreeUpdatedEvent struct {
+	// EI is the index of the transaction.
+	EI epoch.Index
+	// TransactionID is the transaction ID that inserted/removed to/from the state mutation smt.
+	TransactionID utxo.TransactionID
+}
+
+// UTXOUpdatedEvent is a container that acts as a dictionary for the UTXO update event related parameters.
+type UTXOUpdatedEvent struct {
+	// EI is the index of updated UTXO.
+	EI epoch.Index
+	// Created are the outputs created in a transaction.
+	Created []*ledger.OutputWithMetadata
+	// Spent are outputs that is spent in a transaction.
+	Spent []*ledger.OutputWithMetadata
 }
 
 // EpochCommittableEvent is a container that acts as a dictionary for the EpochCommittable event related parameters.
 type EpochCommittableEvent struct {
 	// EI is the index of committable epoch.
 	EI epoch.Index
+	// ECRecord is the ec root of committable epoch.
+	ECRecord *epoch.ECRecord
 }
 
 // ManaVectorUpdateEvent is a container that acts as a dictionary for the EpochCommittable event related parameters.

packages/notarization/proofs.go

@@ -106,4 +106,4 @@ func (f *EpochCommitmentFactory) VerifyStateMutationRoot(proof CommitmentProof,
 	return f.verifyRoot(proof, key, key)
 }
 
-// endregion ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+// endregion ///////////////////////////////////////////////////////////////////////////////////////////////////////////

packages/notarization/testutils.go

@@ -1,13 +1,11 @@
 package notarization
 
 import (
-	"reflect"
 	"sync/atomic"
 	"testing"
 
 	"github.com/iotaledger/hive.go/generics/event"
 	"github.com/iotaledger/hive.go/types/confirmation"
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 
 	"github.com/iotaledger/goshimmer/packages/consensus/acceptance"
@@ -61,10 +59,6 @@ func NewEventMock(t *testing.T, notarizationManager *Manager) *EventMock {
 	notarizationManager.Events.EpochCommittable.Hook(event.NewClosure(e.EpochCommittable))
 	notarizationManager.Events.ManaVectorUpdate.Hook(event.NewClosure(e.ManaVectorUpdate))
 
-	// assure that all available events are mocked
-	numEvents := reflect.ValueOf(notarizationManager.Events).Elem().NumField()
-	assert.Equalf(t, len(e.attached)+2, numEvents, "not all events in notarizationManager.Events have been attached")
-
 	return e
 }
 

packages/tangle/messagefactory.go

@@ -126,15 +126,14 @@ func (f *MessageFactory) issuePayload(p payload.Payload, references ParentMessag
 	issuerPublicKey := f.localIdentity.PublicKey()
 
 	epochCommitment, lastConfirmedEpochIndex, epochCommitmentErr := f.tangle.Options.CommitmentFunc()
-
-	// select tips, perform PoW and prepare references
-	references, nonce, issuingTime, err := f.selectTipsAndPerformPoW(p, references, parentsCount, issuerPublicKey, sequenceNumber, lastConfirmedEpochIndex, epochCommitment)
 	if epochCommitmentErr != nil {
 		err = errors.Errorf("cannot retrieve epoch commitment: %w", epochCommitmentErr)
 		f.Events.Error.Trigger(err)
 		return nil, err
 	}
 
+	// select tips, perform PoW and prepare references
+	references, nonce, issuingTime, err := f.selectTipsAndPerformPoW(p, references, parentsCount, issuerPublicKey, sequenceNumber, lastConfirmedEpochIndex, epochCommitment)
 	if err != nil {
 		return nil, errors.Errorf("could not select tips and perform PoW: %w", err)
 	}

packages/tangle/ratesetter_test.go

@@ -1,10 +1,11 @@
 package tangle
 
 import (
-	"github.com/iotaledger/goshimmer/packages/epoch"
 	"testing"
 	"time"
 
+	"github.com/iotaledger/goshimmer/packages/epoch"
+
 	"github.com/iotaledger/goshimmer/packages/tangle/payload"
 
 	"github.com/iotaledger/hive.go/crypto/ed25519"