v0.7.1 Helper function updates (Consensys#730)

* Implement Test Suites to Run Shuffling Reference Tests from Eth2 Testing Spec (Consensys#715)

* Update BLS tests to use official eth2 spec vectors.

* Test shuffling with different validator set sizes from eth2-test spec reference tests.

* Test shuffling with default shuffling test vectors from eth2-test spec reference tests.

* Fix get shuffling

* changes made to give p2pNetwork access to the data needed for RPC calls (Consensys#726)

* V0.7.1 integration (#1)

* Constants update (v0.7.1) (Consensys#725)

* Change mainnet constants

* Change minimal constants in config.toml

* Update data structures (Consensys#727)

* add bls_domain

* get_churn_limit added

* is_slashable_attestation_data added

* validate_indexed_attestation added

* get_domain modified

* convert_to_indexed

* get_attesting_indices modified

* get_shuffled_index added

* compute_committee added

* get_crosslink_committee added

* get_epoch_start_shard added

* get_shard_delta added

* get_epoch_committee_count added

* get_block_root_at_slot added

* get_attestation_data_slot added

* decrease_balance added

* increase_balance added

* is_slashable_validator added

* get_previous_epoch modified

* get_active_validator_indices modified

* get_randao_mix modified

* get_active_index_root modified

* generate_seed modified

* get_beacon_proposer_index modified

* removed methods

* replacce assert

* bug fixes

* bug fixes

* couture

* bug fixes

* best dev evar

* bug fixes

* oops

ethereum/datastructures/src/main/java/tech/pegasys/artemis/datastructures/operations/Deposit.java

@@ -64,7 +64,7 @@ public Bytes toBytes() {
  return SSZ.encode(
  writer -> {
  writer.writeFixedBytesVector(filledProofList);
- writer.writeBytes(deposit_data.toBytes());
+ writer.writeBytes(data.toBytes());
  });
  }
 

ethereum/datastructures/src/main/java/tech/pegasys/artemis/datastructures/operations/IndexedAttestation.java

@@ -52,6 +52,37 @@ public IndexedAttestation(IndexedAttestation indexedAttestation) {
  this.signature = new BLSSignature(indexedAttestation.getSignature().getSignature());
  }
 
+ //TODO Indexed Attestation should be converted to unsigned longs before serialization
+ public static IndexedAttestation fromBytes(Bytes bytes) {
+ return SSZ.decode(
+ bytes,
+ reader ->
+ new IndexedAttestation(
+ reader.readUInt64List().stream()
+ .map(UnsignedLong::fromLongBits)
+ .collect(Collectors.toList()),
+ reader.readUInt64List().stream()
+ .map(UnsignedLong::fromLongBits)
+ .collect(Collectors.toList()),
+ AttestationData.fromBytes(reader.readBytes()),
+ BLSSignature.fromBytes(reader.readBytes())));
+ }
+
+ //TODO Indexed Attestation should be converted to unsigned longs before serialization
+ public Bytes toBytes() {
+ return SSZ.encode(
+ writer -> {
+ writer.writeULongIntList(
+ 64,
+ custody_bit_0_indices.stream().map(UnsignedLong::longValue).collect(Collectors.toList()));
+ writer.writeULongIntList(
+ 64,
+ custody_bit_1_indices.stream().map(UnsignedLong::longValue).collect(Collectors.toList()));
+ writer.writeBytes(data.toBytes());
+ writer.writeBytes(signature.toBytes());
+ });
+ }
+
  @Override
  public int hashCode() {
  return Objects.hash(custody_bit_0_indices, custody_bit_1_indices, data, signature);
@@ -111,4 +142,22 @@ public BLSSignature getSignature() {
  public void setSignature(BLSSignature signature) {
  this.signature = signature;
  }
+
+ @Override
+ public Bytes32 hash_tree_root() {
+ return HashTreeUtil.merkleize(
+ Arrays.asList(
+ HashTreeUtil.hash_tree_root(
+ SSZTypes.LIST_OF_BASIC,
+ custody_bit_0_indices.stream()
+ .map(item -> SSZ.encodeUInt64(item.longValue()))
+ .collect(Collectors.toList())),
+ HashTreeUtil.hash_tree_root(
+ SSZTypes.LIST_OF_BASIC,
+ custody_bit_1_indices.stream()
+ .map(item -> SSZ.encodeUInt64(item.longValue()))
+ .collect(Collectors.toList())),
+ data.hash_tree_root(),
+ HashTreeUtil.hash_tree_root(SSZTypes.TUPLE_OF_BASIC, signature.toBytes())));
+ }
 }

ethereum/datastructures/src/main/java/tech/pegasys/artemis/datastructures/state/BeaconState.java

@@ -119,8 +119,7 @@ public BeaconState() {
  this.latest_slashed_balances =
  new ArrayList<>(
  Collections.nCopies(Constants.LATEST_SLASHED_EXIT_LENGTH, UnsignedLong.ZERO));
- this.latest_block_header =
- BeaconBlockUtil.get_temporary_block_header(BeaconBlockUtil.get_empty_block());
+ this.latest_block_header = new BeaconBlockHeader(null, null, null, null, null);
  this.historical_roots = new ArrayList<>();
 
  this.latest_eth1_data = new Eth1Data(ZERO_HASH, UnsignedLong.ZERO, ZERO_HASH);

ethereum/datastructures/src/main/java/tech/pegasys/artemis/datastructures/state/Crosslink.java

@@ -14,15 +14,16 @@
 package tech.pegasys.artemis.datastructures.state;
 
 import com.google.common.primitives.UnsignedLong;
-import java.util.Arrays;
-import java.util.Objects;
 import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.bytes.Bytes32;
 import org.apache.tuweni.ssz.SSZ;
 import tech.pegasys.artemis.datastructures.Copyable;
 import tech.pegasys.artemis.util.hashtree.HashTreeUtil;
 import tech.pegasys.artemis.util.hashtree.HashTreeUtil.SSZTypes;
 
+import java.util.Arrays;
+import java.util.Objects;
+
 public class Crosslink implements Copyable<Crosslink> {
 
  private UnsignedLong shard;

ethereum/datastructures/src/main/java/tech/pegasys/artemis/datastructures/util/AttestationUtil.java

@@ -14,9 +14,6 @@
 package tech.pegasys.artemis.datastructures.util;
 
 import com.google.common.primitives.UnsignedLong;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
 import org.apache.commons.lang3.tuple.ImmutableTriple;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.commons.lang3.tuple.Triple;
@@ -27,11 +24,41 @@
 import tech.pegasys.artemis.datastructures.operations.Attestation;
 import tech.pegasys.artemis.datastructures.operations.AttestationData;
 import tech.pegasys.artemis.datastructures.operations.AttestationDataAndCustodyBit;
+import tech.pegasys.artemis.datastructures.operations.IndexedAttestation;
 import tech.pegasys.artemis.datastructures.state.BeaconState;
 import tech.pegasys.artemis.datastructures.state.Crosslink;
 import tech.pegasys.artemis.datastructures.state.CrosslinkCommittee;
+import tech.pegasys.artemis.datastructures.state.Validator;
 import tech.pegasys.artemis.util.bls.BLSKeyPair;
 import tech.pegasys.artemis.util.bls.BLSPublicKey;
+import tech.pegasys.artemis.util.bls.BLSSignature;
+import tech.pegasys.artemis.util.bls.BLSVerify;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static tech.pegasys.artemis.datastructures.Constants.DOMAIN_ATTESTATION;
+import static tech.pegasys.artemis.datastructures.Constants.MAX_INDICES_PER_ATTESTATION;
+import static tech.pegasys.artemis.datastructures.Constants.SHARD_COUNT;
+import static tech.pegasys.artemis.datastructures.Constants.SLOTS_PER_EPOCH;
+import static tech.pegasys.artemis.datastructures.util.BeaconStateUtil.get_bitfield_bit;
+import static tech.pegasys.artemis.datastructures.util.BeaconStateUtil.get_domain;
+import static tech.pegasys.artemis.datastructures.util.BeaconStateUtil.get_epoch_committee_count;
+import static tech.pegasys.artemis.datastructures.util.BeaconStateUtil.get_epoch_start_slot;
+import static tech.pegasys.artemis.datastructures.util.BeaconStateUtil.verify_bitfield;
+import static tech.pegasys.artemis.datastructures.util.CrosslinkCommitteeUtil.get_crosslink_committee;
+import static tech.pegasys.artemis.datastructures.util.CrosslinkCommitteeUtil.get_epoch_committee_count;
+import static tech.pegasys.artemis.datastructures.util.CrosslinkCommitteeUtil.get_epoch_start_shard;
+import static tech.pegasys.artemis.util.bls.BLSAggregate.bls_aggregate_pubkeys;
+import static tech.pegasys.artemis.util.hashtree.HashTreeUtil.SSZTypes.LIST_OF_BASIC;
+import static tech.pegasys.artemis.util.hashtree.HashTreeUtil.hash_tree_root;
 
 public class AttestationUtil {
 
@@ -50,6 +77,7 @@ public static List<Triple<BLSPublicKey, Integer, CrosslinkCommittee>> getAtteste
  BeaconState headState, HashMap<BLSPublicKey, Pair<BLSKeyPair, Boolean>> validatorSet) {
 
  UnsignedLong slot = headState.getSlot();
+ //TODO depracated method
  List<CrosslinkCommittee> crosslinkCommittees =
  BeaconStateUtil.get_crosslink_committees_at_slot(headState, slot);
 
@@ -75,15 +103,16 @@ public static List<Triple<BLSPublicKey, Integer, CrosslinkCommittee>> getAtteste
  public static AttestationData getGenericAttestationData(BeaconState state, BeaconBlock block) {
  // Get variables necessary that can be shared among Attestations of all validators
  UnsignedLong slot = state.getSlot();
+ //TODO depracated method
  List<CrosslinkCommittee> crosslinkCommittees =
  BeaconStateUtil.get_crosslink_committees_at_slot(state, slot);
- Bytes32 headBlockRoot = block.signed_root("signature");
+ Bytes32 headBlockRoot = block.signing_root("signature");
  Bytes32 crosslinkDataRoot = Bytes32.ZERO;
  UnsignedLong epochStartSlot =
- BeaconStateUtil.get_epoch_start_slot(BeaconStateUtil.get_current_epoch(state));
+ get_epoch_start_slot(BeaconStateUtil.get_current_epoch(state));
  Bytes32 epochBoundaryRoot;
  if (epochStartSlot.compareTo(slot) == 0) {
- epochBoundaryRoot = block.signed_root("signature");
+ epochBoundaryRoot = block.signing_root("signature");
  } else {
  epochBoundaryRoot = BeaconStateUtil.get_block_root(state, epochStartSlot);
  }
@@ -92,6 +121,7 @@ public static AttestationData getGenericAttestationData(BeaconState state, Beaco
 
  // Set attestation data
  // TODO: change the generic shard number and crosslink usage to something safer
+ //TODO: Crosslink constructor has changed
  AttestationData attestationData =
  new AttestationData(
  slot,
@@ -137,10 +167,150 @@ public static Bytes32 getAttestationMessageToSign(AttestationData attestationDat
  }
 
  public static int getDomain(BeaconState state, AttestationData attestationData) {
- return BeaconStateUtil.get_domain(
+ return get_domain(
  state.getFork(),
  BeaconStateUtil.slot_to_epoch(attestationData.getSlot()),
  Constants.DOMAIN_ATTESTATION)
  .intValue();
  }
+
+ /**
+ * Check if ``data_1`` and ``data_2`` are slashable according to Casper FFG rules.
+ *
+ * @param data_1
+ * @param data_2
+ * @return
+ *
+ * @see <a>https://github.com/ethereum/eth2.0-specs/blob/v0.7.1/specs/core/0_beacon-chain.md#is_slashable_attestation_data</a>
+ */
+ public static boolean is_slashable_attestation_data(AttestationData data_1, AttestationData data_2){
+ return (
+ //case 1: double vote || case 2: surround vote
+ (!data_1.equals(data_2) && data_1.getTarget_epoch().equals(data_2.getTarget_epoch()) ||
+ (data_1.getSource_epoch().compareTo(data_2.getSource_epoch()) < 0 &&
+ data_2.getTarget_epoch().compareTo(data_1.getTarget_epoch()) < 0))
+ );
+ }
+
+ /**
+ * Convert ``attestation`` to (almost) indexed-verifiable form.
+ *
+ * @param state
+ * @param attestation
+ * @return
+ *
+ * @see <a>https://github.com/ethereum/eth2.0-specs/blob/v0.7.1/specs/core/0_beacon-chain.md#convert_to_indexed</a>
+ */
+ public static IndexedAttestation convert_to_indexed(BeaconState state, Attestation attestation){
+ List<Integer> attesting_indices = get_attesting_indices(state, attestation.getData(), attestation.getAggregation_bitfield());
+ List<Integer> custody_bit_1_indices = get_attesting_indices(state, attestation.getData(), attestation.getCustody_bitfield());
+
+ List<Integer> custody_bit_0_indices = new ArrayList<Integer>();
+ for(int i = 0; i < attesting_indices.size(); i++){
+ Integer index = attesting_indices.get(i);
+ if(!custody_bit_1_indices.contains(index))custody_bit_0_indices.add(index);
+ }
+ return new IndexedAttestation(
+ custody_bit_0_indices,
+ custody_bit_1_indices,
+ attestation.getData(),
+ attestation.getAggregate_signature()
+ );
+ }
+
+ /**
+ * Return the sorted attesting indices corresponding to ``attestation_data`` and ``bitfield``.
+ *
+ * @param state
+ * @param attestation_data
+ * @param bitfield
+ * @return
+ * @throws IllegalArgumentException
+ *
+ * @see <a>https://github.com/ethereum/eth2.0-specs/blob/v0.7.1/specs/core/0_beacon-chain.md#get_attesting_indices</a>
+ */
+ public static List<Integer> get_attesting_indices(
+ BeaconState state, AttestationData attestation_data, Bytes bitfield) throws IllegalArgumentException {
+ List<Integer> committee = get_crosslink_committee(state, attestation_data.getTarget_epoch(), attestation_data.getCrosslink().getShard());
+ checkArgument(verify_bitfield(bitfield, committee.size()), "AttestationUtil.get_attesting_indices");
+
+ List<Integer> attesting_indices = new ArrayList<Integer>();
+ for(int i=0; i < committee.size(); i++){
+ int index = committee.get(i).intValue();
+ int bitfieldBit = get_bitfield_bit(bitfield, i);
+ if((bitfieldBit & 1) == 1) attesting_indices.add(index);
+ }
+ return attesting_indices;
+ }
+
+ /**
+ * Verify validity of ``indexed_attestation``.
+ *
+ * @param state
+ * @param indexed_attestation
+ *
+ * @see <a>https://github.com/ethereum/eth2.0-specs/blob/v0.7.1/specs/core/0_beacon-chain.md#validate_indexed_attestation</a>
+ */
+ public static void validate_indexed_attestation(BeaconState state, IndexedAttestation indexed_attestation){
+ List<Integer> bit_0_indices = indexed_attestation.getCustody_bit_0_indices();
+ List<Integer> bit_1_indices = indexed_attestation.getCustody_bit_1_indices();
+
+ checkArgument(bit_1_indices.size() == 0, "AttestationUtil.validate_indexed_attestation: Verify no index has custody bit equal to 1 [to be removed in phase 1]");
+ checkArgument((bit_0_indices.size() + bit_1_indices.size()) <= MAX_INDICES_PER_ATTESTATION, "AttestationUtil.validate_indexed_attestation: Verify max number of indices");
+ checkArgument(intersection(bit_0_indices, bit_1_indices).size() == 0, "AttestationUtil.validate_indexed_attestation: Verify index sets are disjoint");
+
+ //Verify indices are sorted
+ List<Integer> bit_0_indices_sorted = new ArrayList<Integer>(bit_0_indices);
+ Collections.sort(bit_0_indices_sorted);
+ List<Integer> bit_1_indices_sorted = new ArrayList<Integer>(bit_1_indices);
+ Collections.sort(bit_1_indices_sorted);
+ checkArgument(bit_0_indices.equals(bit_0_indices_sorted) && bit_1_indices.equals(bit_1_indices_sorted));
+
+ List<Validator> validators = state.getValidator_registry();
+ List<BLSPublicKey> pubkeys = new ArrayList<BLSPublicKey>();
+ pubkeys.add(bls_aggregate_pubkeys(bit_0_indices.stream()
+ .map(i -> validators.get(i).getPubkey()).collect(Collectors.toList())));
+ pubkeys.add(bls_aggregate_pubkeys(bit_1_indices.stream()
+ .map(i -> validators.get(i).getPubkey()).collect(Collectors.toList())));
+
+ List<Bytes32> message_hashes = new ArrayList<Bytes32>();
+ message_hashes.add(new AttestationDataAndCustodyBit(indexed_attestation.getData(), false).hash_tree_root());
+ message_hashes.add(new AttestationDataAndCustodyBit(indexed_attestation.getData(), true).hash_tree_root());
+
+ BLSSignature signature = indexed_attestation.getSignature();
+ UnsignedLong domain = UnsignedLong.valueOf(get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.getData().getTarget_epoch()));
+ //Verify aggregate signature
+ checkArgument(BLSVerify.bls_verify_multiple(pubkeys, message_hashes, signature, domain));
+ }
+
+ /**
+ * Returns the data slot for the provided AttestationData
+ *
+ * @param state
+ * @param data
+ * @return
+ *
+ * @see <a>https://github.com/ethereum/eth2.0-specs/blob/v0.7.1/specs/core/0_beacon-chain.md#get_attestation_data_slot</a>
+ */
+ public static UnsignedLong get_attestation_data_slot(BeaconState state, AttestationData data) {
+ UnsignedLong committee_count = get_epoch_committee_count(state, data.getTarget_epoch());
+ UnsignedLong offset = (data.getCrosslink().getShard()
+ .plus(UnsignedLong.valueOf(SHARD_COUNT))
+ .minus(get_epoch_start_shard(state, data.getTarget_epoch())))
+ .mod(UnsignedLong.valueOf(SHARD_COUNT));
+ return get_epoch_start_slot(data.getTarget_epoch()).plus(offset.dividedBy(committee_count.dividedBy(UnsignedLong.valueOf(SLOTS_PER_EPOCH))));
+ }
+
+ public static <T> List<T> intersection(List<T> list1, List<T> list2) {
+ List<T> list = new ArrayList<T>();
+
+ for (T t : list1) {
+ if(list2.contains(t)) {
+ list.add(t);
+ }
+ }
+
+ return list;
+ }
+
 }

ethereum/datastructures/src/main/java/tech/pegasys/artemis/datastructures/util/BeaconBlockUtil.java

@@ -24,20 +24,6 @@
 
 public class BeaconBlockUtil {
  private static final ALogger LOG = new ALogger(BeaconBlockUtil.class.getName());
- /**
- * Return the block header corresponding to a block with ``state_root`` set to ``ZERO_HASH``.
- *
- * @param block
- * @return
- */
- public static BeaconBlockHeader get_temporary_block_header(BeaconBlock block) {
- return new BeaconBlockHeader(
- UnsignedLong.valueOf(block.getSlot()),
- block.getPrevious_block_root(),
- Constants.ZERO_HASH,
- block.getBody().hash_tree_root(),
- Constants.EMPTY_SIGNATURE);
- }
 
  /**
  * Get an empty ``BeaconBlock``.