Merge remote-tracking branch 'ConsenSys/master' into merge/ref-tests-…

…update

CHANGELOG.md

@@ -18,6 +18,7 @@ For information on changes in released versions of Teku, see the [releases page]
 ## Unreleased Changes
 
 ### Additions and Improvements
+- Sets Altair fork epoch for MainNet to 74240
 - Upgraded to BLST 0.3.5.
 - `/teku/v1/admin/readiness` endpoint now accepts a `target_peer_count` param to require a minimum number of peers before the node is considered ready.
 - Support for building on JDK 17.
@@ -26,3 +27,6 @@ For information on changes in released versions of Teku, see the [releases page]
 ### Bug Fixes
 - Fix `NoSuchElementException` reported at startup by validator performance tracking module. 
 - Fix `IllegalStateException` reported on altair networks, when none of your validators are in a sync committee for the current epoch.
+- Fix failed to export/import slashing protection data.
+- Fix incorrect gossip validation when processing duplicate sync committee messages which could lead to an error being reported instead of ignoring the duplicate.
+- Added additional verification of message lengths when decoding snappy messages.

data/dataexchange/src/main/java/tech/pegasys/teku/data/SlashingProtectionExporter.java

@@ -20,6 +20,7 @@
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -45,7 +46,7 @@ public class SlashingProtectionExporter {
 
   public SlashingProtectionExporter(final SubCommandLogger log, final String path) {
     this.log = log;
-    this.syncDataAccessor = SyncDataAccessor.createWithoutAtomicMove();
+    syncDataAccessor = SyncDataAccessor.create(Paths.get(path));
   }
 
   public void initialise(final Path slashProtectionPath) {

data/dataexchange/src/main/java/tech/pegasys/teku/data/SlashingProtectionImporter.java

@@ -22,9 +22,11 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.stream.Collectors;
 import tech.pegasys.teku.data.signingrecord.ValidatorSigningRecord;
@@ -47,7 +49,7 @@ public class SlashingProtectionImporter {
 
   public SlashingProtectionImporter(final SubCommandLogger log, final String path) {
     this.log = log;
-    syncDataAccessor = SyncDataAccessor.createWithoutAtomicMove();
+    syncDataAccessor = SyncDataAccessor.create(Paths.get(path));
   }
 
   public void initialise(final File inputFile) throws IOException {
@@ -98,23 +100,33 @@ private List<SigningHistory> summariseCompleteInterchangeFormat(
   }
 
   private SigningHistory signingHistoryConverter(final SigningHistory signingHistory) {
-    final Optional<UInt64> lastSlot =
-        signingHistory.signedBlocks.stream().map(SignedBlock::getSlot).max(UInt64::compareTo);
-    final Optional<UInt64> sourceEpoch =
-        signingHistory.signedAttestations.stream()
-            .map(SignedAttestation::getSourceEpoch)
-            .max(UInt64::compareTo);
-    final Optional<UInt64> targetEpoch =
-        signingHistory.signedAttestations.stream()
-            .map(SignedAttestation::getTargetEpoch)
-            .max(UInt64::compareTo);
-    final ValidatorSigningRecord record =
-        new ValidatorSigningRecord(
-            metadata.genesisValidatorsRoot,
-            lastSlot.orElse(UInt64.ZERO),
-            sourceEpoch.orElse(ValidatorSigningRecord.NEVER_SIGNED),
-            targetEpoch.orElse(ValidatorSigningRecord.NEVER_SIGNED));
-    return new SigningHistory(signingHistory.pubkey, record);
+    try {
+      final Optional<UInt64> lastSlot =
+          signingHistory.signedBlocks.stream()
+              .map(SignedBlock::getSlot)
+              .filter(Objects::nonNull)
+              .max(UInt64::compareTo);
+      final Optional<UInt64> sourceEpoch =
+          signingHistory.signedAttestations.stream()
+              .map(SignedAttestation::getSourceEpoch)
+              .filter(Objects::nonNull)
+              .max(UInt64::compareTo);
+      final Optional<UInt64> targetEpoch =
+          signingHistory.signedAttestations.stream()
+              .map(SignedAttestation::getTargetEpoch)
+              .filter(Objects::nonNull)
+              .max(UInt64::compareTo);
+      final ValidatorSigningRecord record =
+          new ValidatorSigningRecord(
+              metadata.genesisValidatorsRoot,
+              lastSlot.orElse(UInt64.ZERO),
+              sourceEpoch.orElse(ValidatorSigningRecord.NEVER_SIGNED),
+              targetEpoch.orElse(ValidatorSigningRecord.NEVER_SIGNED));
+      return new SigningHistory(signingHistory.pubkey, record);
+    } catch (NullPointerException e) {
+      System.out.println(signingHistory.pubkey);
+      throw e;
+    }
   }
 
   public void updateLocalRecords(final Path slashingProtectionPath) {
@@ -133,10 +145,12 @@ private void updateLocalRecord(final SigningHistory signingHistory) {
       try {
         existingRecord = syncDataAccessor.read(outputFile).map(ValidatorSigningRecord::fromBytes);
       } catch (IOException e) {
-        log.exit(1, "Failed to read existing file: " + outputFile.toString());
+        log.exit(1, "Failed to read existing file: " + outputFile);
       }
     }
     if (existingRecord.isPresent()
+        && existingRecord.get().getGenesisValidatorsRoot() != null
+        && metadata.genesisValidatorsRoot != null
         && metadata.genesisValidatorsRoot.compareTo(existingRecord.get().getGenesisValidatorsRoot())
             != 0) {
       log.exit(

data/dataexchange/src/main/java/tech/pegasys/teku/data/slashinginterchange/SigningHistory.java

@@ -13,6 +13,8 @@
 
 package tech.pegasys.teku.data.slashinginterchange;
 
+import static tech.pegasys.teku.data.signingrecord.ValidatorSigningRecord.isNeverSigned;
+
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.MoreObjects;
@@ -50,9 +52,12 @@ public SigningHistory(final BLSPubKey pubkey, final ValidatorSigningRecord recor
     this.signedBlocks = new ArrayList<>();
     signedBlocks.add(new SignedBlock(record.getBlockSlot(), null));
     this.signedAttestations = new ArrayList<>();
-    signedAttestations.add(
-        new SignedAttestation(
-            record.getAttestationSourceEpoch(), record.getAttestationTargetEpoch(), null));
+    if (!isNeverSigned(record.getAttestationSourceEpoch())
+        || !isNeverSigned(record.getAttestationTargetEpoch())) {
+      signedAttestations.add(
+          new SignedAttestation(
+              record.getAttestationSourceEpoch(), record.getAttestationTargetEpoch(), null));
+    }
   }
 
   @Override
@@ -83,15 +88,21 @@ public ValidatorSigningRecord toValidatorSigningRecord(
       final Optional<ValidatorSigningRecord> maybeRecord, final Bytes32 genesisValidatorsRoot) {
 
     final UInt64 lastSignedBlockSlot =
-        signedBlocks.stream().map(SignedBlock::getSlot).max(UInt64::compareTo).orElse(UInt64.ZERO);
+        signedBlocks.stream()
+            .map(SignedBlock::getSlot)
+            .filter(Objects::nonNull)
+            .max(UInt64::compareTo)
+            .orElse(UInt64.ZERO);
     final UInt64 lastSignedAttestationSourceEpoch =
         signedAttestations.stream()
             .map(SignedAttestation::getSourceEpoch)
+            .filter(Objects::nonNull)
             .max(UInt64::compareTo)
             .orElse(null);
     final UInt64 lastSignedAttestationTargetEpoch =
         signedAttestations.stream()
             .map(SignedAttestation::getTargetEpoch)
+            .filter(Objects::nonNull)
             .max(UInt64::compareTo)
             .orElse(null);
 

data/dataexchange/src/test/java/tech/pegasys/teku/data/SlashingProtectionExporterTest.java

@@ -39,6 +39,7 @@
 import tech.pegasys.teku.cli.OSUtils;
 import tech.pegasys.teku.data.signingrecord.ValidatorSigningRecord;
 import tech.pegasys.teku.data.slashinginterchange.Metadata;
+import tech.pegasys.teku.data.slashinginterchange.SignedBlock;
 import tech.pegasys.teku.data.slashinginterchange.SigningHistory;
 import tech.pegasys.teku.data.slashinginterchange.SlashingProtectionInterchangeFormat;
 import tech.pegasys.teku.infrastructure.logging.SubCommandLogger;
@@ -155,6 +156,49 @@ public void shouldPrintIfFileCannotBeRead(@TempDir Path tempDir)
     assertThat(stringArgs.getValue()).startsWith("Failed to read from file");
   }
 
+  @Test
+  public void shouldExportSlashProtection(@TempDir Path tempDir)
+      throws IOException, URISyntaxException {
+    final Path exportedFile = tempDir.resolve("exportedFile.json").toAbsolutePath();
+    final SlashingProtectionExporter exporter =
+        new SlashingProtectionExporter(logger, tempDir.toString());
+
+    exporter.readSlashProtectionFile(usingResourceFile("slashProtection.yml", tempDir));
+
+    assertThat(Files.exists(exportedFile)).isFalse();
+    exporter.saveToFile(exportedFile.toString());
+    assertThat(Files.exists(exportedFile)).isTrue();
+  }
+
+  @Test
+  void shouldHaveNoSignedAttestationsWhenNoAttestationsSigned(@TempDir Path tempDir)
+      throws Exception {
+    final Path exportedFile = tempDir.resolve("exportedFile.json").toAbsolutePath();
+    final SlashingProtectionExporter exporter =
+        new SlashingProtectionExporter(logger, tempDir.toString());
+
+    final UInt64 blockSlot = UInt64.ONE;
+    final ValidatorSigningRecord signingRecord =
+        new ValidatorSigningRecord(validatorsRoot)
+            .maySignBlock(validatorsRoot, blockSlot)
+            .orElseThrow();
+    final Path recordFile = tempDir.resolve(pubkey + ".yml");
+    Files.write(recordFile, signingRecord.toBytes().toArrayUnsafe());
+    exporter.readSlashProtectionFile(recordFile.toFile());
+
+    assertThat(exportedFile).doesNotExist();
+    exporter.saveToFile(exportedFile.toString());
+    assertThat(exportedFile).exists();
+
+    final SlashingProtectionInterchangeFormat exportedRecords =
+        jsonProvider.jsonToObject(
+            Files.readString(exportedFile), SlashingProtectionInterchangeFormat.class);
+    assertThat(exportedRecords.data).hasSize(1);
+    final SigningHistory signingHistory = exportedRecords.data.get(0);
+    assertThat(signingHistory.signedBlocks).containsExactly(new SignedBlock(blockSlot, null));
+    assertThat(signingHistory.signedAttestations).isEmpty();
+  }
+
   private File usingResourceFile(final String resourceFileName, final Path tempDir)
       throws URISyntaxException, IOException {
     final Path tempFile = tempDir.resolve(pubkey + ".yml").toAbsolutePath();

data/dataexchange/src/test/java/tech/pegasys/teku/data/SlashingProtectionImporterTest.java

@@ -14,21 +14,32 @@
 package tech.pegasys.teku.data;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 
 import com.google.common.io.Resources;
 import java.io.File;
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import org.apache.tuweni.bytes.Bytes;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
 import org.mockito.ArgumentCaptor;
+import tech.pegasys.teku.data.signingrecord.ValidatorSigningRecord;
 import tech.pegasys.teku.data.slashinginterchange.Metadata;
 import tech.pegasys.teku.infrastructure.logging.SubCommandLogger;
+import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 
 public class SlashingProtectionImporterTest {
   private final ArgumentCaptor<String> stringArgs = ArgumentCaptor.forClass(String.class);
+  private final String pubkey =
+      "b845089a1457f811bfc000588fbb4e713669be8ce060ea6be3c6ece09afc3794106c91ca73acda5e5457122d58723bed";
 
   @Test
   public void shouldFailWithParseError() throws URISyntaxException, IOException {
@@ -55,6 +66,89 @@ public void shouldFailIfMetadataNotPresent() throws IOException, URISyntaxExcept
     assertThat(errorString).contains("does not appear to have metadata");
   }
 
+  @Test
+  public void shouldExportAndImportFile(@TempDir Path tempDir)
+      throws IOException, URISyntaxException {
+    final SubCommandLogger logger = mock(SubCommandLogger.class);
+    final Path exportedFile = tempDir.resolve("exportedFile.json").toAbsolutePath();
+
+    final SlashingProtectionExporter exporter =
+        new SlashingProtectionExporter(logger, tempDir.toString());
+    final File ruleFile = usingResourceFile("slashProtection.yml", tempDir);
+    exporter.readSlashProtectionFile(ruleFile);
+    final String originalFileContent = Files.readString(ruleFile.toPath());
+
+    assertThat(Files.exists(ruleFile.toPath())).isTrue();
+    assertThat(Files.exists(exportedFile)).isFalse();
+    exporter.saveToFile(exportedFile.toString());
+    ruleFile.delete();
+    assertThat(Files.exists(exportedFile)).isTrue();
+    assertThat(Files.exists(ruleFile.toPath())).isFalse();
+
+    SlashingProtectionImporter importer =
+        new SlashingProtectionImporter(logger, exportedFile.toString());
+    importer.initialise(new File(exportedFile.toString()));
+    importer.updateLocalRecords(tempDir);
+    assertThat(Files.exists(ruleFile.toPath())).isTrue();
+
+    assertThat(originalFileContent).isEqualTo(Files.readString(ruleFile.toPath()));
+  }
+
+  @Test
+  void shouldImportFileOverRepairedRecords(@TempDir Path tempDir) throws Exception {
+    final SubCommandLogger logger = mock(SubCommandLogger.class);
+    final Path initialRecords = tempDir.resolve("initial");
+    final Path repairedRecords = tempDir.resolve("repaired");
+    assertThat(initialRecords.toFile().mkdirs()).isTrue();
+    assertThat(repairedRecords.toFile().mkdirs()).isTrue();
+
+    final Path exportedFile = tempDir.resolve("exportedFile.json").toAbsolutePath();
+    final File initialRuleFile =
+        usingResourceFile("slashProtectionWithGenesisRoot.yml", initialRecords);
+    final File repairedRuleFile =
+        usingResourceFile("slashProtectionWithGenesisRoot.yml", repairedRecords);
+
+    final SlashingProtectionExporter exporter =
+        new SlashingProtectionExporter(logger, tempDir.toString());
+    exporter.readSlashProtectionFile(repairedRuleFile);
+    final String originalFileContent = Files.readString(initialRuleFile.toPath());
+
+    assertThat(exportedFile).doesNotExist();
+    exporter.saveToFile(exportedFile.toString());
+    assertThat(exportedFile).exists();
+
+    final SlashingProtectionRepairer repairer =
+        SlashingProtectionRepairer.create(logger, repairedRecords, true);
+    final UInt64 repairedSlot = UInt64.valueOf(1566);
+    final UInt64 repairedEpoch = UInt64.valueOf(7668);
+    repairer.updateRecords(repairedSlot, repairedEpoch);
+    verify(logger, never()).error(any());
+    verify(logger, never()).error(any(), any());
+    assertThat(Files.readString(repairedRuleFile.toPath())).isNotEqualTo(originalFileContent);
+
+    SlashingProtectionImporter importer =
+        new SlashingProtectionImporter(logger, exportedFile.toString());
+    importer.initialise(exportedFile.toFile());
+    importer.updateLocalRecords(repairedRecords);
+
+    // Should wind up with a file that contains the slot and epochs from the repair, combined with
+    // the genesis root from the initial file
+    final ValidatorSigningRecord initialRecord = loadSigningRecord(initialRuleFile);
+    final ValidatorSigningRecord importedRecord = loadSigningRecord(repairedRuleFile);
+    assertThat(importedRecord)
+        .isEqualTo(
+            new ValidatorSigningRecord(
+                initialRecord.getGenesisValidatorsRoot(),
+                repairedSlot,
+                repairedEpoch,
+                repairedEpoch));
+  }
+
+  private ValidatorSigningRecord loadSigningRecord(final File repairedRuleFile) throws IOException {
+    return ValidatorSigningRecord.fromBytes(
+        Bytes.wrap(Files.readAllBytes(repairedRuleFile.toPath())));
+  }
+
   private String loadAndGetErrorText(final String resourceFile)
       throws URISyntaxException, IOException {
     final SubCommandLogger logger = mock(SubCommandLogger.class);
@@ -65,4 +159,14 @@ private String loadAndGetErrorText(final String resourceFile)
     verify(logger).exit(eq(1), stringArgs.capture());
     return stringArgs.getValue();
   }
+
+  private File usingResourceFile(final String resourceFileName, final Path tempDir)
+      throws URISyntaxException, IOException {
+    final Path tempFile = tempDir.resolve(pubkey + ".yml").toAbsolutePath();
+    Files.copy(
+        new File(Resources.getResource(resourceFileName).toURI()).toPath(),
+        tempFile,
+        StandardCopyOption.REPLACE_EXISTING);
+    return tempFile.toFile();
+  }
 }

data/dataexchange/src/test/resources/slashProtectionNoAttestations.yml

@@ -0,0 +1,4 @@
+---
+lastSignedBlockSlot: 327
+lastSignedAttestationSourceEpoch: 51
+lastSignedAttestationTargetEpoch: 1741

data/dataexchange/src/test/resources/slashProtectionWithGenesisRoot.yml

@@ -1,5 +1,5 @@
 ---
+genesisValidatorsRoot: "0x6e2c5d8a89dfe121a92c8812bea69fe9f84ae48f63aafe34ef7e18c7eac9af70"
 lastSignedBlockSlot: 327
 lastSignedAttestationSourceEpoch: 51
 lastSignedAttestationTargetEpoch: 1741
-genesisValidatorsRoot: "0x6e2c5d8a89dfe121a92c8812bea69fe9f84ae48f63aafe34ef7e18c7eac9af70"

eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/Eth2ReferenceTestCase.java

@@ -46,6 +46,7 @@ public abstract class Eth2ReferenceTestCase {
           .putAll(OperationsTestExecutor.OPERATIONS_TEST_TYPES)
           .putAll(SanityTests.SANITY_TEST_TYPES)
           .putAll(SszTestExecutorDeprecated.SSZ_TEST_TYPES)
+          .put("merkle/single_proof", TestExecutor.IGNORE_TESTS)
           .build();
 
   private final ImmutableMap<String, TestExecutor> PHASE_0_TEST_TYPES =

ethereum/spec/src/test/java/tech/pegasys/teku/spec/SpecFactoryTest.java

@@ -31,12 +31,12 @@
 
 public class SpecFactoryTest {
 
-  private static final Set<String> ALTAIR_NETWORKS = Set.of("pyrmont", "prater");
+  private static final Set<String> ALTAIR_NETWORKS = Set.of("pyrmont", "prater", "mainnet");
 
   @Test
-  public void defaultFactoryShouldOnlySupportPhase0_mainnet() {
+  public void defaultFactoryShouldScheduleAltairForMainNet() {
     final Spec spec = SpecFactory.create("mainnet");
-    assertThat(spec.getForkSchedule().getSupportedMilestones()).containsExactly(PHASE0);
+    assertThat(spec.getForkSchedule().getSupportedMilestones()).containsExactly(PHASE0, ALTAIR);
   }
 
   @ParameterizedTest(name = "{0}")