Handle empty blocks and attestations in interchange json and sort int…

…erchange json by public key (#8132)

* Handle empty blocks and attestations in interchange json

* add test

* sort json

* easier empty arrays

* pass test

Co-authored-by: Raul Jordan <raul@prysmaticlabs.com>

validator/slashing-protection/cli_export.go

@@ -2,8 +2,6 @@ package slashingprotection
 
 import (
 	"encoding/json"
-	"path/filepath"
-
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/shared/cmd"
 	"github.com/prysmaticlabs/prysm/shared/fileutil"
@@ -12,6 +10,7 @@ import (
 	"github.com/prysmaticlabs/prysm/validator/flags"
 	export "github.com/prysmaticlabs/prysm/validator/slashing-protection/local/standard-protection-format"
 	"github.com/urfave/cli/v2"
+	"path/filepath"
 )
 
 const (

validator/slashing-protection/cli_import_export_test.go

@@ -46,7 +46,7 @@ func TestImportExportSlashingProtectionCli_RoundTrip(t *testing.T) {
 	// Create some mock slashing protection history. and JSON file
 	pubKeys, err := mocks.CreateRandomPubKeys(numValidators)
 	require.NoError(t, err)
-	attestingHistory, proposalHistory, err := mocks.MockAttestingAndProposalHistories(pubKeys, numEpochs)
+	attestingHistory, proposalHistory, err := mocks.MockAttestingAndProposalHistories(pubKeys, numEpochs, numEpochs)
 	require.NoError(t, err)
 	mockJSON, err := mocks.MockSlashingProtectionJSON(pubKeys, attestingHistory, proposalHistory)
 	require.NoError(t, err)
@@ -101,3 +101,69 @@ func TestImportExportSlashingProtectionCli_RoundTrip(t *testing.T) {
 		require.DeepEqual(t, wanted.SignedAttestations, item.SignedAttestations)
 	}
 }
+
+func TestImportExportSlashingProtectionCli_EmptyData(t *testing.T) {
+	numValidators := 10
+	outputPath := filepath.Join(os.TempDir(), "slashing-exports")
+	err := fileutil.MkdirAll(outputPath)
+	require.NoError(t, err)
+	protectionFileName := "slashing_history_import.json"
+
+	// Create some mock slashing protection history. and JSON file
+	pubKeys, err := mocks.CreateRandomPubKeys(numValidators)
+	require.NoError(t, err)
+	attestingHistory, proposalHistory, err := mocks.MockAttestingAndProposalHistories(pubKeys, 0, 0)
+	require.NoError(t, err)
+	mockJSON, err := mocks.MockSlashingProtectionJSON(pubKeys, attestingHistory, proposalHistory)
+	require.NoError(t, err)
+
+	// We JSON encode the protection file and save it to disk as a JSON file.
+	encoded, err := json.Marshal(mockJSON)
+	require.NoError(t, err)
+
+	protectionFilePath := filepath.Join(outputPath, protectionFileName)
+	err = fileutil.WriteFile(protectionFilePath, encoded)
+	require.NoError(t, err)
+
+	// We create a CLI context with the required values, such as the database datadir and output directory.
+	validatorDB := dbTest.SetupDB(t, pubKeys)
+	dbPath := validatorDB.DatabasePath()
+	require.NoError(t, validatorDB.Close())
+	cliCtx := setupCliCtx(t, dbPath, protectionFilePath, outputPath)
+
+	// We import the slashing protection history file via CLI.
+	err = ImportSlashingProtectionCLI(cliCtx)
+	require.NoError(t, err)
+
+	// We export the slashing protection history file via CLI.
+	err = ExportSlashingProtectionJSONCli(cliCtx)
+	require.NoError(t, err)
+
+	// Attempt to read the exported file from the output directory.
+	enc, err := fileutil.ReadFileAsBytes(filepath.Join(outputPath, jsonExportFileName))
+	require.NoError(t, err)
+
+	receivedJSON := &protectionFormat.EIPSlashingProtectionFormat{}
+	err = json.Unmarshal(enc, receivedJSON)
+	require.NoError(t, err)
+
+	// We verify the parsed JSON file matches. Given there is no guarantee of order,
+	// we will have to carefully compare and sort values as needed.
+	//
+	// First, we compare basic data such as the Metadata value in the JSON file.
+	require.DeepEqual(t, mockJSON.Metadata, receivedJSON.Metadata)
+	wantedHistoryByPublicKey := make(map[string]*protectionFormat.ProtectionData)
+	for _, item := range mockJSON.Data {
+		wantedHistoryByPublicKey[item.Pubkey] = item
+	}
+
+	// Next, we compare all the data for each validator public key.
+	for _, item := range receivedJSON.Data {
+		wanted, ok := wantedHistoryByPublicKey[item.Pubkey]
+		require.Equal(t, true, ok)
+		require.Equal(t, len(wanted.SignedBlocks), len(item.SignedBlocks))
+		require.Equal(t, len(wanted.SignedAttestations), len(item.SignedAttestations))
+		require.DeepEqual(t, make([]*protectionFormat.SignedBlock, 0), item.SignedBlocks)
+		require.DeepEqual(t, make([]*protectionFormat.SignedAttestation, 0), item.SignedAttestations)
+	}
+}

validator/slashing-protection/local/standard-protection-format/export.go

@@ -4,10 +4,11 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/validator/db"
+	"sort"
+	"strings"
 )
 
 // ExportStandardProtectionJSON extracts all slashing protection data from a validator database
@@ -77,8 +78,17 @@ func ExportStandardProtectionJSON(ctx context.Context, validatorDB db.Database)
 	// Next we turn our map into a slice as expected by the EIP-3076 JSON standard.
 	dataList := make([]*ProtectionData, 0)
 	for _, item := range dataByPubKey {
+		if item.SignedAttestations == nil {
+			item.SignedAttestations = make([]*SignedAttestation, 0)
+		}
+		if item.SignedBlocks == nil {
+			item.SignedBlocks = make([]*SignedBlock, 0)
+		}
 		dataList = append(dataList, item)
 	}
+	sort.Slice(dataList, func(i, j int) bool {
+		return strings.Compare(dataList[i].Pubkey, dataList[j].Pubkey) < 0
+	})
 	interchangeJSON.Data = dataList
 	return interchangeJSON, nil
 }

validator/slashing-protection/local/standard-protection-format/format.go

@@ -3,7 +3,9 @@
 // is critical to allow safe interoperability between eth2 clients.
 package interchangeformat
 
-import "github.com/sirupsen/logrus"
+import (
+	"github.com/sirupsen/logrus"
+)
 
 var log = logrus.WithField("prefix", "slashing-protection-format")
 

validator/slashing-protection/local/standard-protection-format/round_trip_test.go

@@ -26,7 +26,7 @@ func TestImportExport_RoundTrip(t *testing.T) {
 
 	// First we setup some mock attesting and proposal histories and create a mock
 	// standard slashing protection format JSON struct.
-	attestingHistory, proposalHistory, err := mocks.MockAttestingAndProposalHistories(publicKeys, numEpochs)
+	attestingHistory, proposalHistory, err := mocks.MockAttestingAndProposalHistories(publicKeys, numEpochs, numEpochs)
 	require.NoError(t, err)
 	wanted, err := mocks.MockSlashingProtectionJSON(publicKeys, attestingHistory, proposalHistory)
 	require.NoError(t, err)
@@ -90,6 +90,7 @@ func TestImportExport_RoundTrip_SkippedAttestationEpochs(t *testing.T) {
 						TargetEpoch: "9",
 					},
 				},
+				SignedBlocks: make([]*protectionFormat.SignedBlock, 0),
 			},
 		},
 	}
@@ -126,7 +127,7 @@ func TestImportInterchangeData_OK(t *testing.T) {
 
 	// First we setup some mock attesting and proposal histories and create a mock
 	// standard slashing protection format JSON struct.
-	attestingHistory, proposalHistory, err := mocks.MockAttestingAndProposalHistories(publicKeys, numEpochs)
+	attestingHistory, proposalHistory, err := mocks.MockAttestingAndProposalHistories(publicKeys, numEpochs, numEpochs)
 	require.NoError(t, err)
 	standardProtectionFormat, err := mocks.MockSlashingProtectionJSON(publicKeys, attestingHistory, proposalHistory)
 	require.NoError(t, err)
@@ -170,7 +171,7 @@ func TestImportInterchangeData_OK_SavesBlacklistedPublicKeys(t *testing.T) {
 
 	// First we setup some mock attesting and proposal histories and create a mock
 	// standard slashing protection format JSON struct.
-	attestingHistory, proposalHistory, err := mocks.MockAttestingAndProposalHistories(publicKeys, numEpochs)
+	attestingHistory, proposalHistory, err := mocks.MockAttestingAndProposalHistories(publicKeys, numEpochs, numEpochs)
 	require.NoError(t, err)
 
 	standardProtectionFormat, err := mocks.MockSlashingProtectionJSON(publicKeys, attestingHistory, proposalHistory)
@@ -259,7 +260,7 @@ func TestStore_ImportInterchangeData_BadFormat_PreventsDBWrites(t *testing.T) {
 
 	// First we setup some mock attesting and proposal histories and create a mock
 	// standard slashing protection format JSON struct.
-	attestingHistory, proposalHistory, err := mocks.MockAttestingAndProposalHistories(publicKeys, numEpochs)
+	attestingHistory, proposalHistory, err := mocks.MockAttestingAndProposalHistories(publicKeys, numEpochs, numEpochs)
 	require.NoError(t, err)
 	standardProtectionFormat, err := mocks.MockSlashingProtectionJSON(publicKeys, attestingHistory, proposalHistory)
 	require.NoError(t, err)

validator/testing/protection_history.go

@@ -61,17 +61,16 @@ func MockSlashingProtectionJSON(
 // MockAttestingAndProposalHistories given a number of validators, creates mock attesting
 // and proposing histories within WEAK_SUBJECTIVITY_PERIOD bounds.
 func MockAttestingAndProposalHistories(
-	pubKeys [][48]byte, highestEpoch int,
+	pubKeys [][48]byte, numberOfProposals, numberOfAttestations int,
 ) (map[[48]byte]kv.EncHistoryData, map[[48]byte]kv.ProposalHistoryForPubkey, error) {
 	attData := make(map[[48]byte]kv.EncHistoryData, len(pubKeys))
 	proposalData := make(map[[48]byte]kv.ProposalHistoryForPubkey, len(pubKeys))
 	ctx := context.Background()
 	for v := 0; v < len(pubKeys); v++ {
 		var err error
-		latestTarget := highestEpoch
-		hd := kv.NewAttestationHistoryArray(uint64(latestTarget))
+		hd := kv.NewAttestationHistoryArray(uint64(numberOfAttestations))
 		proposals := make([]kv.Proposal, 0)
-		for i := 1; i <= latestTarget; i++ {
+		for i := 1; i <= numberOfAttestations; i++ {
 			signingRoot := [32]byte{}
 			signingRootStr := fmt.Sprintf("%d", i)
 			copy(signingRoot[:], signingRootStr)
@@ -84,7 +83,7 @@ func MockAttestingAndProposalHistories(
 				return nil, nil, err
 			}
 		}
-		for i := 1; i <= latestTarget; i++ {
+		for i := 1; i <= numberOfProposals; i++ {
 			signingRoot := [32]byte{}
 			signingRootStr := fmt.Sprintf("%d", i)
 			copy(signingRoot[:], signingRootStr)
@@ -94,7 +93,7 @@ func MockAttestingAndProposalHistories(
 			})
 		}
 		proposalData[pubKeys[v]] = kv.ProposalHistoryForPubkey{Proposals: proposals}
-		hd, err = hd.SetLatestEpochWritten(ctx, uint64(latestTarget))
+		hd, err = hd.SetLatestEpochWritten(ctx, uint64(numberOfAttestations))
 		if err != nil {
 			return nil, nil, err
 		}