diff --git a/infrastructure/ssz/build.gradle b/infrastructure/ssz/build.gradle index 856f5dc971e..396e8bb1fd7 100644 --- a/infrastructure/ssz/build.gradle +++ b/infrastructure/ssz/build.gradle @@ -11,6 +11,7 @@ dependencies { implementation 'org.apache.tuweni:tuweni-units' testImplementation testFixtures(project(':infrastructure:collections')) + testImplementation testFixtures(project(':infrastructure:json')) testImplementation testFixtures(project(':infrastructure:serviceutils')) testFixturesApi 'org.apache.tuweni:tuweni-bytes' diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/SszComposite.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/SszComposite.java index 491aaf0c267..ea55a7565e0 100644 --- a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/SszComposite.java +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/SszComposite.java @@ -13,6 +13,7 @@ package tech.pegasys.teku.infrastructure.ssz; +import java.util.NoSuchElementException; import tech.pegasys.teku.infrastructure.ssz.schema.SszCompositeSchema; /** @@ -30,7 +31,10 @@ default int size() { /** * Returns the child at index * - * @throws IndexOutOfBoundsException if index >= size() + * @throws IndexOutOfBoundsException if index > last valid index (which is size() - 1 for + * non-sparse containers like StableContainer) + * @throws NoSuchElementException if index <= last valid index but field was not found (for sparse + * containers like StableContainer) */ SszChildT get(int index); diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/SszProfile.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/SszProfile.java new file mode 100644 index 00000000000..d114b6b0279 --- /dev/null +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/SszProfile.java @@ -0,0 +1,16 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.infrastructure.ssz; + +public interface SszProfile extends SszStableContainerBase {} diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/SszStableContainer.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/SszStableContainer.java new file mode 100644 index 00000000000..3edb0bd52c8 --- /dev/null +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/SszStableContainer.java @@ -0,0 +1,16 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.infrastructure.ssz; + +public interface SszStableContainer extends SszStableContainerBase {} diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/SszStableContainerBase.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/SszStableContainerBase.java new file mode 100644 index 00000000000..3fa5a37b837 --- /dev/null +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/SszStableContainerBase.java @@ -0,0 +1,39 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.infrastructure.ssz; + +import java.util.NoSuchElementException; +import java.util.Optional; +import tech.pegasys.teku.infrastructure.ssz.collections.SszBitvector; + +public interface SszStableContainerBase extends SszContainer { + boolean isFieldActive(int index); + + SszBitvector getActiveFields(); + + default Optional getOptional(final int index) { + try { + return Optional.of(get(index)); + } catch (final NoSuchElementException __) { + return Optional.empty(); + } + } + + @SuppressWarnings({"unchecked", "TypeParameterUnusedInFormals"}) + // container is heterogeneous by its nature so making unsafe cast here + // is more convenient and is not less safe + default Optional getAnyOptional(final int index) { + return (Optional) getOptional(index); + } +} diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/AbstractSszComposite.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/AbstractSszComposite.java index 3569b49d89b..b45c36ed1ff 100644 --- a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/AbstractSszComposite.java +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/AbstractSszComposite.java @@ -14,6 +14,7 @@ package tech.pegasys.teku.infrastructure.ssz.impl; import com.google.common.base.Suppliers; +import java.util.NoSuchElementException; import java.util.Optional; import java.util.function.Supplier; import tech.pegasys.teku.infrastructure.ssz.SszComposite; @@ -132,6 +133,8 @@ public final int size() { * Checks the child index * * @throws IndexOutOfBoundsException if index is invalid + * @throws NoSuchElementException if field is not present (for sparse data structures like + * StableContainers) */ protected abstract void checkIndex(int index); diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/AbstractSszImmutableContainer.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/AbstractSszImmutableContainer.java index 24814aee630..56c14eae639 100644 --- a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/AbstractSszImmutableContainer.java +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/AbstractSszImmutableContainer.java @@ -22,6 +22,7 @@ import tech.pegasys.teku.infrastructure.ssz.SszMutableContainer; import tech.pegasys.teku.infrastructure.ssz.cache.ArrayIntCache; import tech.pegasys.teku.infrastructure.ssz.cache.IntCache; +import tech.pegasys.teku.infrastructure.ssz.schema.SszCompositeSchema; import tech.pegasys.teku.infrastructure.ssz.schema.SszContainerSchema; import tech.pegasys.teku.infrastructure.ssz.schema.impl.AbstractSszContainerSchema; import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode; @@ -40,6 +41,11 @@ protected AbstractSszImmutableContainer( super(schema, backingNode); } + public AbstractSszImmutableContainer( + final SszCompositeSchema type, final TreeNode backingNode, final IntCache cache) { + super(type, backingNode, cache); + } + protected AbstractSszImmutableContainer( final SszContainerSchema schema, final SszData... memberValues) { @@ -62,7 +68,7 @@ protected AbstractSszImmutableContainer( } private static IntCache createCache(final SszData... memberValues) { - ArrayIntCache cache = new ArrayIntCache<>(memberValues.length); + final ArrayIntCache cache = new ArrayIntCache<>(memberValues.length); for (int i = 0; i < memberValues.length; i++) { cache.invalidateWithNewValue(i, memberValues[i]); } diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/AbstractSszMutableComposite.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/AbstractSszMutableComposite.java index 55d499128bf..d91e58f9f05 100644 --- a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/AbstractSszMutableComposite.java +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/AbstractSszMutableComposite.java @@ -91,7 +91,7 @@ public void set(final int index, final SszChildT value) { childrenChanges.put(index, createChangeRecordByValue(immutableValue)); - sizeCache = index >= sizeCache ? index + 1 : sizeCache; + sizeCache = calcNewSize(index); invalidate(); } @@ -239,6 +239,15 @@ public SszMutableComposite createWritableCopy() { */ protected abstract void checkIndex(int index, boolean set); + /** + * Determines the new size given the field has been set or updated + * + * @param index of the field set or updated + */ + protected int calcNewSize(final int index) { + return index >= sizeCache ? index + 1 : sizeCache; + } + private static final class ChildChangeRecord< SszChildT extends SszData, SszMutableChildT extends SszChildT> { diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszContainerImpl.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszContainerImpl.java index 1429b6bf768..00c938439b8 100644 --- a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszContainerImpl.java +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszContainerImpl.java @@ -22,7 +22,6 @@ import tech.pegasys.teku.infrastructure.ssz.cache.IntCache; import tech.pegasys.teku.infrastructure.ssz.schema.SszCompositeSchema; import tech.pegasys.teku.infrastructure.ssz.schema.SszContainerSchema; -import tech.pegasys.teku.infrastructure.ssz.schema.impl.AbstractSszContainerSchema; import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode; public class SszContainerImpl extends AbstractSszComposite implements SszContainer { @@ -48,8 +47,8 @@ protected SszData getImpl(final int index) { } @Override - public AbstractSszContainerSchema getSchema() { - return (AbstractSszContainerSchema) super.getSchema(); + public SszContainerSchema getSchema() { + return (SszContainerSchema) super.getSchema(); } @Override diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszMutableContainerImpl.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszMutableContainerImpl.java index f1536658c91..702a930f045 100644 --- a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszMutableContainerImpl.java +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszMutableContainerImpl.java @@ -19,7 +19,7 @@ import tech.pegasys.teku.infrastructure.ssz.SszMutableData; import tech.pegasys.teku.infrastructure.ssz.SszMutableRefContainer; import tech.pegasys.teku.infrastructure.ssz.cache.IntCache; -import tech.pegasys.teku.infrastructure.ssz.schema.impl.AbstractSszContainerSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.SszContainerSchema; import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode; public class SszMutableContainerImpl extends AbstractSszMutableComposite @@ -36,8 +36,8 @@ protected SszContainerImpl createImmutableSszComposite( } @Override - public AbstractSszContainerSchema getSchema() { - return (AbstractSszContainerSchema) super.getSchema(); + public SszContainerSchema getSchema() { + return (SszContainerSchema) super.getSchema(); } @Override diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszMutableListImpl.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszMutableListImpl.java index 88ab58c6cbd..c83af2b22df 100644 --- a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszMutableListImpl.java +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszMutableListImpl.java @@ -102,4 +102,9 @@ public SszList commitChanges() { public SszMutableList createWritableCopy() { throw new UnsupportedOperationException("Creating a copy from writable list is not supported"); } + + @Override + public String toString() { + return "Mutable" + backingImmutableData; + } } diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszMutableStableContainerBaseImpl.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszMutableStableContainerBaseImpl.java new file mode 100644 index 00000000000..e97eef7e4a3 --- /dev/null +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszMutableStableContainerBaseImpl.java @@ -0,0 +1,69 @@ +/* + * Copyright Consensys Software Inc., 2022 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.infrastructure.ssz.impl; + +import java.util.NoSuchElementException; +import tech.pegasys.teku.infrastructure.ssz.SszData; +import tech.pegasys.teku.infrastructure.ssz.cache.IntCache; +import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode; + +public class SszMutableStableContainerBaseImpl extends SszMutableContainerImpl { + protected SszStableContainerBaseImpl backingStableContainerBaseView; + + public SszMutableStableContainerBaseImpl(final SszStableContainerBaseImpl backingImmutableView) { + super(backingImmutableView); + this.backingStableContainerBaseView = backingImmutableView; + } + + @Override + protected SszContainerImpl createImmutableSszComposite( + final TreeNode backingNode, final IntCache viewCache) { + return new SszStableContainerBaseImpl( + getSchema().toStableContainerSchemaBaseRequired(), backingNode, viewCache); + } + + @Override + protected void checkIndex(final int index, final boolean set) { + // we currently not support StableContainers with optional fields, so we expect get\set over an + // already active field + if (backingStableContainerBaseView.isFieldActive(index)) { + return; + } + + if (index > backingStableContainerBaseView.getActiveFields().getLastSetBitIndex()) { + throw new IndexOutOfBoundsException( + "Invalid index " + + index + + " for container with last active index " + + backingStableContainerBaseView.getActiveFields().getLastSetBitIndex()); + } + + throw new NoSuchElementException("Index " + index + " is not active in the stable container"); + } + + @Override + protected int calcNewSize(final int index) { + // StableContainer have sparse index so size cannot be compared with index. + // Currently, the only mutable StableContainer we support is a Profile with no optional + // fields, so we can assume the size never changes. + // See: + // tech.pegasys.teku.infrastructure.ssz.impl.SszStableContainerBaseImpl.createWritableCopy + return size(); + } + + @Override + public String toString() { + return "Mutable " + backingStableContainerBaseView.toString(); + } +} diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszProfileImpl.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszProfileImpl.java new file mode 100644 index 00000000000..1c0c67e3a38 --- /dev/null +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszProfileImpl.java @@ -0,0 +1,63 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.infrastructure.ssz.impl; + +import com.google.common.base.Preconditions; +import java.util.Arrays; +import tech.pegasys.teku.infrastructure.ssz.SszData; +import tech.pegasys.teku.infrastructure.ssz.SszProfile; +import tech.pegasys.teku.infrastructure.ssz.cache.ArrayIntCache; +import tech.pegasys.teku.infrastructure.ssz.cache.IntCache; +import tech.pegasys.teku.infrastructure.ssz.schema.SszProfileSchema; +import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode; + +public class SszProfileImpl extends SszStableContainerBaseImpl implements SszProfile { + + public SszProfileImpl(final SszProfileSchema type) { + super(type); + } + + public SszProfileImpl(final SszProfileSchema type, final TreeNode backingNode) { + super(type, backingNode); + } + + public SszProfileImpl( + final SszProfileSchema type, final TreeNode backingNode, final IntCache cache) { + super(type, backingNode, cache); + } + + public SszProfileImpl(final SszProfileSchema type, final SszData... memberValues) { + super( + type, + type.createTreeFromFieldValues(Arrays.asList(memberValues)), + createCache(memberValues)); + + for (int i = 0; i < memberValues.length; i++) { + Preconditions.checkArgument( + memberValues[i].getSchema().equals(type.getChildSchema(i)), + "Wrong child schema at index %s. Expected: %s, was %s", + i, + type.getChildSchema(i), + memberValues[i].getSchema()); + } + } + + private static IntCache createCache(final SszData... memberValues) { + final ArrayIntCache cache = new ArrayIntCache<>(memberValues.length); + for (int i = 0; i < memberValues.length; i++) { + cache.invalidateWithNewValue(i, memberValues[i]); + } + return cache; + } +} diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszStableContainerBaseImpl.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszStableContainerBaseImpl.java new file mode 100644 index 00000000000..f1f0271eeb9 --- /dev/null +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszStableContainerBaseImpl.java @@ -0,0 +1,135 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.infrastructure.ssz.impl; + +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.stream.Collectors; +import tech.pegasys.teku.infrastructure.ssz.SszData; +import tech.pegasys.teku.infrastructure.ssz.SszMutableContainer; +import tech.pegasys.teku.infrastructure.ssz.SszStableContainerBase; +import tech.pegasys.teku.infrastructure.ssz.cache.ArrayIntCache; +import tech.pegasys.teku.infrastructure.ssz.cache.IntCache; +import tech.pegasys.teku.infrastructure.ssz.collections.SszBitvector; +import tech.pegasys.teku.infrastructure.ssz.schema.SszCompositeSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.SszStableContainerBaseSchema; +import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode; + +public class SszStableContainerBaseImpl extends SszContainerImpl implements SszStableContainerBase { + + private final SszBitvector activeFields; + + public SszStableContainerBaseImpl( + final SszStableContainerBaseSchema type) { + this(type, type.getDefaultTree()); + } + + public SszStableContainerBaseImpl( + final SszStableContainerBaseSchema type, + final TreeNode backingNode) { + this(type, backingNode, Optional.empty()); + } + + public SszStableContainerBaseImpl( + final SszStableContainerBaseSchema type, + final TreeNode backingNode, + final IntCache cache) { + this(type, backingNode, Optional.of(cache)); + } + + private SszStableContainerBaseImpl( + final SszStableContainerBaseSchema type, + final TreeNode backingNode, + final Optional> cache) { + this( + type, + backingNode, + cache, + type.toStableContainerSchemaBaseRequired() + .getActiveFieldsBitvectorFromBackingNode(backingNode)); + } + + /** + * The composite class creates the cache with a capacity set as the size of the composite. In + * Stable Container and Profile case, the actual the size will be the max theoretical size. To + * avoid this waste we always pre-create the cache with the effective required size. + */ + private SszStableContainerBaseImpl( + final SszCompositeSchema type, + final TreeNode backingNode, + final Optional> cache, + final SszBitvector activeFields) { + super(type, backingNode, cache.orElse(createCache(activeFields))); + this.activeFields = activeFields; + } + + private static IntCache createCache(final SszBitvector activeFields) { + return new ArrayIntCache<>(activeFields.getLastSetBitIndex() + 1); + } + + @Override + public SszMutableContainer createWritableCopy() { + if (isWritableSupported()) { + return new SszMutableStableContainerBaseImpl(this); + } + throw new UnsupportedOperationException( + "Mutation on Stable Containers or Profiles with optional fields is not currently supported"); + } + + @Override + public boolean isWritableSupported() { + return !getSchema().toStableContainerSchemaBaseRequired().hasOptionalFields(); + } + + @Override + public boolean isFieldActive(final int index) { + return activeFields.getBit(index); + } + + @Override + public SszBitvector getActiveFields() { + return activeFields; + } + + @Override + protected int sizeImpl() { + return activeFields.getBitCount(); + } + + @Override + protected void checkIndex(final int index) { + if (isFieldActive(index)) { + return; + } + if (getSchema().toStableContainerSchemaBaseRequired().isFieldAllowed(index)) { + throw new NoSuchElementException("Index " + index + " is not active in the stable container"); + } + throw new IndexOutOfBoundsException("Index " + index + " is not allowed"); + } + + @Override + public String toString() { + return getSchema().getContainerName() + + "{activeFields=" + + activeFields + + ", optionalFields: " + + getSchema().toStableContainerSchemaBaseRequired().getOptionalFields() + + ", " + + activeFields + .streamAllSetBits() + .mapToObj(idx -> getSchema().getFieldNames().get(idx) + "=" + get(idx)) + .collect(Collectors.joining(", ")) + + "}"; + } +} diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszStableContainerImpl.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszStableContainerImpl.java new file mode 100644 index 00000000000..d3d004ea5f3 --- /dev/null +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/impl/SszStableContainerImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.infrastructure.ssz.impl; + +import tech.pegasys.teku.infrastructure.ssz.SszData; +import tech.pegasys.teku.infrastructure.ssz.SszStableContainer; +import tech.pegasys.teku.infrastructure.ssz.cache.IntCache; +import tech.pegasys.teku.infrastructure.ssz.schema.SszStableContainerSchema; +import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode; + +public class SszStableContainerImpl extends SszStableContainerBaseImpl + implements SszStableContainer { + + public SszStableContainerImpl(final SszStableContainerSchema type) { + super(type); + } + + public SszStableContainerImpl( + final SszStableContainerSchema type, + final TreeNode backingNode) { + super(type, backingNode); + } + + public SszStableContainerImpl( + final SszStableContainerSchema type, + final TreeNode backingNode, + final IntCache cache) { + super(type, backingNode, cache); + } +} diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszCompositeSchema.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszCompositeSchema.java index 4b13f2bc5ca..a4d767c9992 100644 --- a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszCompositeSchema.java +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszCompositeSchema.java @@ -36,7 +36,10 @@ public interface SszCompositeSchema> * schema is the same for any index For heterogeneous structures (like Container) each child has * individual schema * - * @throws IndexOutOfBoundsException if index >= getMaxLength + * @throws IndexOutOfBoundsException if index >= getMaxLength for all schemas except Profile + * (stable container). A Profile can have more schemas than the actual maxLength, so for those + * schemas the exception will be thrown when index is >= than number of defined children + * schema. */ SszSchema getChildSchema(int index); diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszContainerSchema.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszContainerSchema.java index 1667ccadc4a..c92709ea054 100644 --- a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszContainerSchema.java +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszContainerSchema.java @@ -14,6 +14,7 @@ package tech.pegasys.teku.infrastructure.ssz.schema; import java.util.List; +import java.util.Optional; import java.util.function.BiFunction; import org.apache.tuweni.bytes.Bytes32; import tech.pegasys.teku.infrastructure.ssz.SszContainer; @@ -139,4 +140,16 @@ private TreeNode loadChildNode( /** Return this container field schemas */ List> getFieldSchemas(); + + default SszStableContainerBaseSchema toStableContainerSchemaBaseRequired() { + throw new UnsupportedOperationException("Not a StableContainer schema"); + } + + default Optional> toStableContainerSchema() { + return Optional.empty(); + } + + default Optional> toProfileSchema() { + return Optional.empty(); + } } diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszOptionalSchema.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszOptionalSchema.java index ce4654b129a..3c452dcfa19 100644 --- a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszOptionalSchema.java +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszOptionalSchema.java @@ -42,6 +42,11 @@ default boolean isFixedSize() { return false; } + @Override + default boolean hasExtraDataInBackingTree() { + return true; + } + @Override default int getSszFixedPartSize() { return 0; diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszProfileSchema.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszProfileSchema.java new file mode 100644 index 00000000000..4abad71f28c --- /dev/null +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszProfileSchema.java @@ -0,0 +1,39 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.infrastructure.ssz.schema; + +import java.util.Optional; +import tech.pegasys.teku.infrastructure.ssz.SszProfile; +import tech.pegasys.teku.infrastructure.ssz.SszStableContainer; +import tech.pegasys.teku.infrastructure.ssz.collections.SszBitvector; +import tech.pegasys.teku.infrastructure.ssz.schema.collections.SszBitvectorSchema; + +public interface SszProfileSchema extends SszStableContainerBaseSchema { + SszStableContainerSchema getStableContainerSchema(); + + @Override + default SszBitvectorSchema getActiveFieldsSchema() { + return getStableContainerSchema().getActiveFieldsSchema(); + } + + @Override + default SszStableContainerBaseSchema toStableContainerSchemaBaseRequired() { + return this; + } + + @Override + default Optional> toProfileSchema() { + return Optional.of(this); + } +} diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszStableContainerBaseSchema.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszStableContainerBaseSchema.java new file mode 100644 index 00000000000..9f741350e9c --- /dev/null +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszStableContainerBaseSchema.java @@ -0,0 +1,76 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.infrastructure.ssz.schema; + +import java.util.List; +import java.util.Optional; +import tech.pegasys.teku.infrastructure.ssz.SszData; +import tech.pegasys.teku.infrastructure.ssz.SszStableContainerBase; +import tech.pegasys.teku.infrastructure.ssz.collections.SszBitvector; +import tech.pegasys.teku.infrastructure.ssz.schema.collections.SszBitvectorSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.impl.AbstractSszContainerSchema.NamedSchema; +import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode; + +/** + * {@link SszSchema} representing base functionalities that the Profile and StableContainer schemas + * have in common as per eip-7495 + * specifications. In particular, they both have: + * + *
    + *
  1. A theoretical future maximum field count. + *
  2. A bitvector schema representing which fields are active (as a required field or as an + * optional field). + *
  3. Due to possible field optionality, they can both be created from a list of optional {@link + * SszData}. + *
+ * + * @param the type of actual container class + */ +public interface SszStableContainerBaseSchema + extends SszContainerSchema { + + /** + * The potential maximum number of fields to which the container can ever grow in the future. + * + * @return stable container maximum field count. + */ + int getMaxFieldCount(); + + List> getChildrenNamedSchemas(); + + SszBitvectorSchema getActiveFieldsSchema(); + + TreeNode createTreeFromOptionalFieldValues(List> fieldValues); + + default C createFromOptionalFieldValues(final List> fieldValues) { + return createFromBackingNode(createTreeFromOptionalFieldValues(fieldValues)); + } + + SszBitvector getRequiredFields(); + + SszBitvector getOptionalFields(); + + default boolean isFieldAllowed(final int index) { + if (index >= getMaxFieldCount()) { + return false; + } + return getRequiredFields().getBit(index) || getOptionalFields().getBit(index); + } + + default boolean hasOptionalFields() { + return getOptionalFields().getBitCount() > 0; + } + + SszBitvector getActiveFieldsBitvectorFromBackingNode(TreeNode node); +} diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszStableContainerSchema.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszStableContainerSchema.java new file mode 100644 index 00000000000..848454b107d --- /dev/null +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszStableContainerSchema.java @@ -0,0 +1,81 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.infrastructure.ssz.schema; + +import static tech.pegasys.teku.infrastructure.ssz.schema.impl.AbstractSszContainerSchema.namedSchema; + +import java.util.List; +import java.util.Optional; +import java.util.function.BiFunction; +import java.util.stream.IntStream; +import tech.pegasys.teku.infrastructure.ssz.SszStableContainer; +import tech.pegasys.teku.infrastructure.ssz.schema.impl.AbstractSszContainerSchema.NamedSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.impl.AbstractSszStableContainerSchema; +import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode; + +public interface SszStableContainerSchema + extends SszStableContainerBaseSchema { + + /** + * Creates a new {@link SszStableContainer} schema with specified field schemas and container + * instance constructor + */ + static SszStableContainerSchema create( + final String name, + final List> activeChildrenSchemas, + final int maxFieldCount, + final BiFunction, TreeNode, C> instanceCtor) { + return new AbstractSszStableContainerSchema<>(name, activeChildrenSchemas, maxFieldCount) { + @Override + public C createFromBackingNode(TreeNode node) { + return instanceCtor.apply(this, node); + } + }; + } + + /** + * Creates a new {@link SszStableContainer} schema with specified field schemas. It is designed to + * be used in profile schema creation only. There will be no ssz views for it. + */ + static + SszStableContainerSchema createFromNamedSchemasForProfileOnly( + final int maxFieldCount, final List> activeChildrenSchemas) { + return new AbstractSszStableContainerSchema<>("", activeChildrenSchemas, maxFieldCount) { + @Override + public C createFromBackingNode(final TreeNode node) { + throw new UnsupportedOperationException( + "This stable container schema is meant to be used for creating a profile schema"); + } + }; + } + + static SszStableContainerSchema createFromSchemasForProfileOnly( + final int maxFieldCount, final List> activeChildrenSchemas) { + return createFromNamedSchemasForProfileOnly( + maxFieldCount, + IntStream.range(0, activeChildrenSchemas.size()) + .>mapToObj(i -> namedSchema("field-" + i, activeChildrenSchemas.get(i))) + .toList()); + } + + @Override + default Optional> toStableContainerSchema() { + return Optional.of(this); + } + + @Override + default SszStableContainerBaseSchema toStableContainerSchemaBaseRequired() { + return this; + } +} diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszType.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszType.java index 9d373aad2b2..97b351c8f48 100644 --- a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszType.java +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszType.java @@ -45,6 +45,13 @@ static int sszBytesToLength(final Bytes bytes) { return ret; } + /** + * Indicates whether the type stores extra data in backing tree, like "size" or "bitvector" + * subtrees which act like "service branches" which requires special treatment during + * serialization and deserialization + */ + boolean hasExtraDataInBackingTree(); + /** Indicates whether the type is fixed or variable size */ boolean isFixedSize(); diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszUnionSchema.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszUnionSchema.java index 47337c9e51e..635cfa4ffef 100644 --- a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszUnionSchema.java +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszUnionSchema.java @@ -53,6 +53,11 @@ default boolean isFixedSize() { return false; } + @Override + default boolean hasExtraDataInBackingTree() { + return true; + } + @Override default int getSszFixedPartSize() { return SELECTOR_SIZE_BYTES; diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszCollectionSchema.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszCollectionSchema.java index 1e6980a8e04..14a5b41170e 100644 --- a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszCollectionSchema.java +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszCollectionSchema.java @@ -140,12 +140,32 @@ protected int getVariablePartSize(final TreeNode vectorNode, final int length) { public int sszSerializeVector( final TreeNode vectorNode, final SszWriter writer, final int elementsCount) { if (getElementSchema().isFixedSize()) { + if (getElementSchema().hasExtraDataInBackingTree()) { + return sszSerializeFixedVector(vectorNode, writer, elementsCount); + } return sszSerializeFixedVectorFast(vectorNode, writer, elementsCount); } else { return sszSerializeVariableVector(vectorNode, writer, elementsCount); } } + private int sszSerializeFixedVector( + final TreeNode vectorNode, final SszWriter writer, final int elementsCount) { + if (elementsCount == 0) { + return 0; + } + + final SszSchema elementType = getElementSchema(); + final int nodesCount = getChunks(elementsCount); + int writtenData = 0; + for (int i = 0; i < nodesCount; i++) { + TreeNode childSubtree = vectorNode.get(getChildGeneralizedIndex(i)); + writtenData += elementType.sszSerializeTree(childSubtree, writer); + } + + return writtenData; + } + private int sszSerializeFixedVectorFast( final TreeNode vectorNode, final SszWriter writer, final int elementsCount) { if (elementsCount == 0) { diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszContainerSchema.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszContainerSchema.java index a735b09e141..41e2dc7545e 100644 --- a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszContainerSchema.java +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszContainerSchema.java @@ -14,6 +14,11 @@ package tech.pegasys.teku.infrastructure.ssz.schema.impl; import static com.google.common.base.Preconditions.checkArgument; +import static tech.pegasys.teku.infrastructure.ssz.schema.impl.ContainerSchemaUtil.deserializeFixedChild; +import static tech.pegasys.teku.infrastructure.ssz.schema.impl.ContainerSchemaUtil.deserializeVariableChild; +import static tech.pegasys.teku.infrastructure.ssz.schema.impl.ContainerSchemaUtil.serializeFixedChild; +import static tech.pegasys.teku.infrastructure.ssz.schema.impl.ContainerSchemaUtil.serializeVariableChild; +import static tech.pegasys.teku.infrastructure.ssz.schema.impl.ContainerSchemaUtil.validateAndPrepareForVariableChildrenDeserialization; import com.google.common.base.Suppliers; import it.unimi.dsi.fastutil.ints.IntArrayList; @@ -34,9 +39,7 @@ import tech.pegasys.teku.infrastructure.ssz.schema.SszContainerSchema; import tech.pegasys.teku.infrastructure.ssz.schema.SszFieldName; import tech.pegasys.teku.infrastructure.ssz.schema.SszSchema; -import tech.pegasys.teku.infrastructure.ssz.schema.SszType; import tech.pegasys.teku.infrastructure.ssz.schema.json.SszContainerTypeDefinition; -import tech.pegasys.teku.infrastructure.ssz.sos.SszDeserializeException; import tech.pegasys.teku.infrastructure.ssz.sos.SszLengthBounds; import tech.pegasys.teku.infrastructure.ssz.sos.SszReader; import tech.pegasys.teku.infrastructure.ssz.sos.SszWriter; @@ -55,7 +58,7 @@ public static NamedSchema of( return new NamedSchema<>(name, schema); } - private NamedSchema(final String name, final SszSchema schema) { + protected NamedSchema(final String name, final SszSchema schema) { this.name = name; this.schema = schema; } @@ -69,12 +72,12 @@ public SszSchema getSchema() { } } - protected static NamedSchema namedSchema( + public static NamedSchema namedSchema( final SszFieldName fieldName, final SszSchema schema) { return namedSchema(fieldName.getSszFieldName(), schema); } - protected static NamedSchema namedSchema( + public static NamedSchema namedSchema( final String fieldName, final SszSchema schema) { return new NamedSchema<>(fieldName, schema); } @@ -88,6 +91,8 @@ protected static NamedSchema namedSchema( private final TreeNode defaultTree; private final long treeWidth; private final int fixedPartSize; + private final boolean hasExtraDataInBackingTree; + private final boolean isFixedSize; private final DeserializableTypeDefinition jsonTypeDefinition; protected AbstractSszContainerSchema( @@ -106,6 +111,8 @@ protected AbstractSszContainerSchema( this.defaultTree = createDefaultTree(); this.treeWidth = SszContainerSchema.super.treeWidth(); this.fixedPartSize = calcSszFixedPartSize(); + this.isFixedSize = calcIsFixedSize(); + this.hasExtraDataInBackingTree = calcHasExtraDataInBackingTree(); this.jsonTypeDefinition = SszContainerTypeDefinition.createFor(this); } @@ -120,6 +127,8 @@ protected AbstractSszContainerSchema(final List> childrenSchemas) { this.defaultTree = createDefaultTree(); this.treeWidth = SszContainerSchema.super.treeWidth(); this.fixedPartSize = calcSszFixedPartSize(); + this.isFixedSize = calcIsFixedSize(); + this.hasExtraDataInBackingTree = calcHasExtraDataInBackingTree(); this.jsonTypeDefinition = SszContainerTypeDefinition.createFor(this); } @@ -205,6 +214,10 @@ public int hashCode() { @Override public boolean isFixedSize() { + return isFixedSize; + } + + private boolean calcIsFixedSize() { for (int i = 0; i < getFieldsCount(); i++) { if (!getChildSchema(i).isFixedSize()) { return false; @@ -213,6 +226,20 @@ public boolean isFixedSize() { return true; } + @Override + public boolean hasExtraDataInBackingTree() { + return hasExtraDataInBackingTree; + } + + private boolean calcHasExtraDataInBackingTree() { + for (int i = 0; i < getFieldsCount(); i++) { + if (getChildSchema(i).hasExtraDataInBackingTree()) { + return true; + } + } + return false; + } + @Override public int getSszFixedPartSize() { return fixedPartSize; @@ -253,25 +280,11 @@ public int sszSerializeTree(final TreeNode node, final SszWriter writer) { int variableChildOffset = getSszFixedPartSize(); int[] variableSizes = new int[getFieldsCount()]; for (int i = 0; i < getFieldsCount(); i++) { - TreeNode childSubtree = node.get(getChildGeneralizedIndex(i)); - SszSchema childType = getChildSchema(i); - if (childType.isFixedSize()) { - int size = childType.sszSerializeTree(childSubtree, writer); - assert size == childType.getSszFixedPartSize(); - } else { - writer.write(SszType.sszLengthToBytes(variableChildOffset)); - int childSize = childType.getSszSize(childSubtree); - variableSizes[i] = childSize; - variableChildOffset += childSize; - } + variableChildOffset += + serializeFixedChild(writer, this, i, node, variableSizes, variableChildOffset); } for (int i = 0; i < childrenSchemas.size(); i++) { - SszSchema childType = getChildSchema(i); - if (!childType.isFixedSize()) { - TreeNode childSubtree = node.get(getChildGeneralizedIndex(i)); - int size = childType.sszSerializeTree(childSubtree, writer); - assert size == variableSizes[i]; - } + serializeVariableChild(writer, this, i, variableSizes, node); } return variableChildOffset; } @@ -280,56 +293,21 @@ public int sszSerializeTree(final TreeNode node, final SszWriter writer) { public TreeNode sszDeserializeTree(final SszReader reader) { int endOffset = reader.getAvailableBytes(); int childCount = getFieldsCount(); - Queue fixedChildrenSubtrees = new ArrayDeque<>(childCount); - IntList variableChildrenOffsets = new IntArrayList(childCount); - for (int i = 0; i < childCount; i++) { - SszSchema childType = getChildSchema(i); - if (childType.isFixedSize()) { - try (SszReader sszReader = reader.slice(childType.getSszFixedPartSize())) { - TreeNode childNode = childType.sszDeserializeTree(sszReader); - fixedChildrenSubtrees.add(childNode); - } - } else { - int childOffset = SszType.sszBytesToLength(reader.read(SSZ_LENGTH_SIZE)); - variableChildrenOffsets.add(childOffset); - } - } + final Queue fixedChildrenSubtrees = new ArrayDeque<>(childCount); + final IntList variableChildrenOffsets = new IntArrayList(childCount); - if (variableChildrenOffsets.isEmpty()) { - if (reader.getAvailableBytes() > 0) { - throw new SszDeserializeException("Invalid SSZ: unread bytes for fixed size container"); - } - } else { - if (variableChildrenOffsets.getInt(0) != endOffset - reader.getAvailableBytes()) { - throw new SszDeserializeException( - "First variable element offset doesn't match the end of fixed part"); - } - } - - variableChildrenOffsets.add(endOffset); - - ArrayDeque variableChildrenSizes = - new ArrayDeque<>(variableChildrenOffsets.size() - 1); - for (int i = 0; i < variableChildrenOffsets.size() - 1; i++) { - variableChildrenSizes.add( - variableChildrenOffsets.getInt(i + 1) - variableChildrenOffsets.getInt(i)); + for (int i = 0; i < childCount; i++) { + deserializeFixedChild(reader, fixedChildrenSubtrees, variableChildrenOffsets, this, i); } - if (variableChildrenSizes.stream().anyMatch(s -> s < 0)) { - throw new SszDeserializeException("Invalid SSZ: wrong child offsets"); - } + final ArrayDeque variableChildrenSizes = + validateAndPrepareForVariableChildrenDeserialization( + reader, variableChildrenOffsets, endOffset); List childrenSubtrees = new ArrayList<>(childCount); for (int i = 0; i < childCount; i++) { - SszSchema childType = getChildSchema(i); - if (childType.isFixedSize()) { - childrenSubtrees.add(fixedChildrenSubtrees.remove()); - } else { - try (SszReader sszReader = reader.slice(variableChildrenSizes.remove())) { - TreeNode childNode = childType.sszDeserializeTree(sszReader); - childrenSubtrees.add(childNode); - } - } + deserializeVariableChild( + reader, childrenSubtrees, fixedChildrenSubtrees, variableChildrenSizes, this, i); } return TreeUtil.createTree(childrenSubtrees); diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszListSchema.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszListSchema.java index 373f4aff076..d9ae1540985 100644 --- a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszListSchema.java +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszListSchema.java @@ -106,11 +106,12 @@ public int getSszVariablePartSize(final TreeNode node) { int length = getLength(node); SszSchema elementSchema = getElementSchema(); if (elementSchema.isFixedSize()) { - if (getSszElementBitSize() == 1) { + final int elementSize = getSszElementBitSize(); + if (elementSize == 1) { // BitlistImpl is handled specially return length / 8 + 1; } else { - return bitsCeilToBytes(length * getSszElementBitSize()); + return bitsCeilToBytes(length * elementSize); } } else { return getCompatibleVectorSchema().getVariablePartSize(getVectorNode(node), length) @@ -123,6 +124,11 @@ public boolean isFixedSize() { return false; } + @Override + public boolean hasExtraDataInBackingTree() { + return true; + } + @Override public int sszSerializeTree(final TreeNode node, final SszWriter writer) { int elementsCount = getLength(node); diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszPrimitiveSchema.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszPrimitiveSchema.java index ee3480cd5ac..96835e0289b 100644 --- a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszPrimitiveSchema.java +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszPrimitiveSchema.java @@ -136,6 +136,11 @@ public boolean isFixedSize() { return true; } + @Override + public boolean hasExtraDataInBackingTree() { + return false; + } + @Override public int getSszFixedPartSize() { return sszSize; diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszProfileSchema.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszProfileSchema.java new file mode 100644 index 00000000000..26e18c139b7 --- /dev/null +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszProfileSchema.java @@ -0,0 +1,167 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.infrastructure.ssz.schema.impl; + +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.ints.IntList; +import java.util.Comparator; +import java.util.Optional; +import java.util.Set; +import java.util.stream.IntStream; +import tech.pegasys.teku.infrastructure.ssz.SszProfile; +import tech.pegasys.teku.infrastructure.ssz.SszStableContainer; +import tech.pegasys.teku.infrastructure.ssz.collections.SszBitvector; +import tech.pegasys.teku.infrastructure.ssz.schema.SszProfileSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.SszStableContainerSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.collections.SszBitvectorSchema; +import tech.pegasys.teku.infrastructure.ssz.sos.SszLengthBounds; +import tech.pegasys.teku.infrastructure.ssz.sos.SszReader; +import tech.pegasys.teku.infrastructure.ssz.sos.SszWriter; +import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode; + +/** + * The Profile overrides the stable container logic by: + * + *
    + *
  1. Skipping active bitvector serialization\deserialization when there are no optional fields + *
  2. In case of allowed optional fields, the active bitvector will only represents the optional + * fields. Required fields are not represented + *
+ */ +public abstract class AbstractSszProfileSchema + extends AbstractSszStableContainerBaseSchema implements SszProfileSchema { + + private final IntList optionalFieldIndexToSchemaIndexCache; + private final int[] schemaIndexToOptionalFieldIndexCache; + private final Set optionalFieldIndices; + private final Optional> optionalFieldsSchema; + private final SszStableContainerSchema stableContainer; + + public AbstractSszProfileSchema( + final String name, + final SszStableContainerSchema stableContainerSchema, + final Set requiredFieldIndices, + final Set optionalFieldIndices) { + super( + name, + stableContainerSchema.getChildrenNamedSchemas(), + requiredFieldIndices, + optionalFieldIndices, + stableContainerSchema.getMaxFieldCount()); + this.optionalFieldIndices = optionalFieldIndices; + if (optionalFieldIndices.isEmpty()) { + this.optionalFieldsSchema = Optional.empty(); + this.schemaIndexToOptionalFieldIndexCache = new int[0]; + this.optionalFieldIndexToSchemaIndexCache = IntList.of(); + + } else { + // we need create a dedicated bitvector schema. The optional fields bitvector will represent + // which of the optional fields are being used (i.e. first bit represent the first optional + // field in order of declaration, the second bit the second one and so on) + this.optionalFieldsSchema = + Optional.of(SszBitvectorSchema.create(optionalFieldIndices.size())); + + // to support serialization and deserialization we need two caches be able to quickly get + // schema field index of a given optional field index and vice versa. + // + // Example: + // + // optional index to schema index (optionalFieldIndexToSchemaIndexCache): + // 0->2 + // 1->4 + // 2->5 + // + // schema index to optional index (schemaIndexToOptionalFieldIndexCache): + // 0->null + // 1->null + // 2->0 + // 3->null + // 4->1 + // 5->1 + + this.optionalFieldIndexToSchemaIndexCache = + IntList.of( + optionalFieldIndices.stream() + .sorted(Comparator.naturalOrder()) + .mapToInt(i -> i) + .toArray()); + + this.schemaIndexToOptionalFieldIndexCache = + new int[optionalFieldIndexToSchemaIndexCache.getInt(optionalFieldIndices.size() - 1) + 1]; + optionalFieldIndices.stream() + .sorted(Comparator.naturalOrder()) + .forEach( + i -> + schemaIndexToOptionalFieldIndexCache[i] = + optionalFieldIndexToSchemaIndexCache.indexOf((int) i)); + } + + this.stableContainer = stableContainerSchema; + } + + @Override + public SszStableContainerSchema getStableContainerSchema() { + return stableContainer; + } + + @Override + SszLengthBounds computeActiveFieldsSszLengthBounds() { + return optionalFieldsSchema + .map(SszBitvectorSchema::getSszLengthBounds) + .orElse(SszLengthBounds.ZERO); + } + + @Override + int getSszActiveFieldsSize(final TreeNode node) { + return optionalFieldsSchema.map(schema -> schema.getSszSize(node)).orElse(0); + } + + @Override + int sszSerializeActiveFields(final SszBitvector activeFieldsBitvector, final SszWriter writer) { + if (optionalFieldsSchema.isEmpty()) { + // without optional fields, a profile won't serialize the bitvector + return 0; + } + final IntList optionalFieldsBits = new IntArrayList(optionalFieldIndices.size()); + optionalFieldIndices.stream() + .filter(activeFieldsBitvector::getBit) + .mapToInt(i -> schemaIndexToOptionalFieldIndexCache[i]) + .forEach(optionalFieldsBits::add); + + return optionalFieldsSchema + .get() + .sszSerializeTree( + optionalFieldsSchema.get().ofBits(optionalFieldsBits).getBackingNode(), writer); + } + + @Override + SszBitvector sszDeserializeActiveFieldsTree(final SszReader reader) { + if (optionalFieldsSchema.isEmpty()) { + // without optional fields the active fields corresponds to the required fields of the schema + return getRequiredFields(); + } + final SszReader optionalFieldsReader = + reader.slice(optionalFieldsSchema.get().getSszFixedPartSize()); + final SszBitvector optionalFields = + optionalFieldsSchema.get().sszDeserialize(optionalFieldsReader); + return getActiveFieldsSchema() + .ofBits( + IntStream.concat( + getRequiredFields().streamAllSetBits(), + optionalFields + .streamAllSetBits() + .map(optionalFieldIndexToSchemaIndexCache::getInt)) + .toArray()); + } +} diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszStableContainerBaseSchema.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszStableContainerBaseSchema.java new file mode 100644 index 00000000000..4c30d0ca41f --- /dev/null +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszStableContainerBaseSchema.java @@ -0,0 +1,729 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.infrastructure.ssz.schema.impl; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static tech.pegasys.teku.infrastructure.ssz.schema.impl.ContainerSchemaUtil.deserializeFixedChild; +import static tech.pegasys.teku.infrastructure.ssz.schema.impl.ContainerSchemaUtil.deserializeVariableChild; +import static tech.pegasys.teku.infrastructure.ssz.schema.impl.ContainerSchemaUtil.serializeFixedChild; +import static tech.pegasys.teku.infrastructure.ssz.schema.impl.ContainerSchemaUtil.serializeVariableChild; +import static tech.pegasys.teku.infrastructure.ssz.schema.impl.ContainerSchemaUtil.validateAndPrepareForVariableChildrenDeserialization; + +import com.google.common.base.Suppliers; +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.ints.IntList; +import it.unimi.dsi.fastutil.objects.Object2IntMap; +import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Queue; +import java.util.Set; +import java.util.function.Supplier; +import org.apache.tuweni.bytes.Bytes32; +import tech.pegasys.teku.infrastructure.json.types.DeserializableTypeDefinition; +import tech.pegasys.teku.infrastructure.ssz.SszData; +import tech.pegasys.teku.infrastructure.ssz.SszStableContainerBase; +import tech.pegasys.teku.infrastructure.ssz.collections.SszBitvector; +import tech.pegasys.teku.infrastructure.ssz.primitive.SszNone; +import tech.pegasys.teku.infrastructure.ssz.schema.SszPrimitiveSchemas; +import tech.pegasys.teku.infrastructure.ssz.schema.SszSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.SszStableContainerBaseSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.SszType; +import tech.pegasys.teku.infrastructure.ssz.schema.collections.SszBitvectorSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.impl.AbstractSszContainerSchema.NamedSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.json.SszStableContainerBaseTypeDefinition; +import tech.pegasys.teku.infrastructure.ssz.sos.SszLengthBounds; +import tech.pegasys.teku.infrastructure.ssz.sos.SszReader; +import tech.pegasys.teku.infrastructure.ssz.sos.SszWriter; +import tech.pegasys.teku.infrastructure.ssz.tree.BranchNode; +import tech.pegasys.teku.infrastructure.ssz.tree.GIndexUtil; +import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode; +import tech.pegasys.teku.infrastructure.ssz.tree.TreeNodeSource; +import tech.pegasys.teku.infrastructure.ssz.tree.TreeNodeSource.CompressedBranchInfo; +import tech.pegasys.teku.infrastructure.ssz.tree.TreeNodeStore; +import tech.pegasys.teku.infrastructure.ssz.tree.TreeUtil; + +/** + * Implements the common container logic shared among Profile and StableContainer as per eip-7495 specifications. + * + *

With a combination of: + * + *

    + *
  1. A NamedSchema list + *
  2. A set of required field indices. + *
  3. A set of optional field indices. + *
  4. A theoretical future maximum field count. + *
+ * + * this class can represent: + * + *
    + *
  1. A StableContainer (empty required field indices, non-empty optional field indices) + *
  2. A Profile (non-empty required field indices, optional field indices can be both empty and + * not-empty) + *
+ * + * @param the type of actual container class + */ +public abstract class AbstractSszStableContainerBaseSchema + implements SszStableContainerBaseSchema { + public static final long CONTAINER_G_INDEX = GIndexUtil.LEFT_CHILD_G_INDEX; + public static final long BITVECTOR_G_INDEX = GIndexUtil.RIGHT_CHILD_G_INDEX; + + private final Supplier sszLengthBounds = + Suppliers.memoize(this::computeSszLengthBounds); + private final int cachedMaxLength; + private final String containerName; + private final List> definedChildrenNamedSchemas; + private final Object2IntMap definedChildrenNamesToFieldIndex; + private final List definedChildrenNames; + private final List> definedChildrenSchemas; + private final int sszFixedPartSize; + private final boolean isFixedSize; + private final int maxFieldCount; + private final long treeWidth; + private final SszBitvectorSchema activeFieldsSchema; + private final SszBitvector requiredFields; + private final SszBitvector optionalFields; + private final boolean hasOptionalFields; + private final TreeNode defaultTreeNode; + + private final DeserializableTypeDefinition jsonTypeDefinition; + + private static long getContainerGIndex(final long rootGIndex) { + return GIndexUtil.gIdxLeftGIndex(rootGIndex); + } + + private static long getBitvectorGIndex(final long rootGIndex) { + return GIndexUtil.gIdxRightGIndex(rootGIndex); + } + + public AbstractSszStableContainerBaseSchema( + final String name, + final List> definedChildrenNamedSchemas, + final Set requiredFieldIndices, + final Set optionalFieldIndices, + final int maxFieldCount) { + checkArgument( + optionalFieldIndices.stream().noneMatch(requiredFieldIndices::contains), + "optional and active fields must not overlap"); + + this.containerName = name; + this.maxFieldCount = maxFieldCount; + this.definedChildrenNamedSchemas = definedChildrenNamedSchemas; + this.definedChildrenSchemas = + definedChildrenNamedSchemas.stream().map(NamedSchema::getSchema).toList(); + this.activeFieldsSchema = SszBitvectorSchema.create(maxFieldCount); + this.requiredFields = activeFieldsSchema.ofBits(requiredFieldIndices); + this.optionalFields = activeFieldsSchema.ofBits(optionalFieldIndices); + this.cachedMaxLength = requiredFields.getBitCount() + optionalFields.getBitCount(); + this.treeWidth = SszStableContainerBaseSchema.super.treeWidth(); + this.definedChildrenNames = + definedChildrenNamedSchemas.stream().map(NamedSchema::getName).toList(); + this.definedChildrenNamesToFieldIndex = new Object2IntOpenHashMap<>(); + for (int i = 0; i < definedChildrenNamedSchemas.size(); i++) { + definedChildrenNamesToFieldIndex.put(definedChildrenNamedSchemas.get(i).getName(), i); + } + this.defaultTreeNode = + BranchNode.create( + createDefaultContainerTreeNode(requiredFieldIndices), requiredFields.getBackingNode()); + this.hasOptionalFields = optionalFields.getBitCount() > 0; + this.jsonTypeDefinition = SszStableContainerBaseTypeDefinition.createFor(this); + this.sszFixedPartSize = calcSszFixedPartSize(); + this.isFixedSize = calcIsFixedSize(); + } + + protected TreeNode createDefaultContainerTreeNode(final Set activeFieldIndices) { + final List defaultChildren = new ArrayList<>(getMaxFieldCount()); + for (int i = 0; i < getMaxFieldCount(); i++) { + if (activeFieldIndices.contains(i)) { + defaultChildren.add(getChildSchema(i).getDefaultTree()); + } else { + defaultChildren.add(SszNone.INSTANCE.getBackingNode()); + } + } + return TreeUtil.createTree(defaultChildren); + } + + @Override + public DeserializableTypeDefinition getJsonTypeDefinition() { + return jsonTypeDefinition; + } + + /** MaxLength exposes the effective potential numbers of fields */ + @Override + public long getMaxLength() { + return cachedMaxLength; + } + + /** The backing tree node is always filled up maxFieldCount, so maxChunks must reflect it */ + @Override + public long maxChunks() { + return ((long) getMaxFieldCount() - 1) / getElementsPerChunk() + 1; + } + + @Override + public long treeWidth() { + return treeWidth; + } + + @Override + public List> getChildrenNamedSchemas() { + return definedChildrenNamedSchemas; + } + + @Override + public SszBitvector getActiveFieldsBitvectorFromBackingNode(final TreeNode node) { + if (hasOptionalFields) { + return activeFieldsSchema.createFromBackingNode(node.get(BITVECTOR_G_INDEX)); + } + return requiredFields; + } + + @Override + public SszBitvector getRequiredFields() { + return requiredFields; + } + + @Override + public SszBitvector getOptionalFields() { + return optionalFields; + } + + @Override + public int getMaxFieldCount() { + return maxFieldCount; + } + + @Override + public TreeNode getDefaultTree() { + return defaultTreeNode; + } + + @Override + public boolean hasOptionalFields() { + return hasOptionalFields; + } + + @Override + public TreeNode createTreeFromFieldValues(final List fieldValues) { + if (hasOptionalFields) { + throw new UnsupportedOperationException( + "createTreeFromFieldValues can be used only when the schema has no optional fields"); + } + + final int fieldsCount = getMaxFieldCount(); + checkArgument(fieldValues.size() <= fieldsCount, "Wrong number of filed values"); + + final List allFields = new ArrayList<>(fieldsCount); + + for (int index = 0, fieldIndex = 0; index < fieldsCount; index++) { + if (fieldIndex >= fieldValues.size() || !requiredFields.getBit(index)) { + allFields.add(SszNone.INSTANCE); + } else { + allFields.add(fieldValues.get(fieldIndex++)); + } + } + + assert allFields.size() == fieldsCount; + + final TreeNode containerTree = + TreeUtil.createTree(allFields.stream().map(SszData::getBackingNode).toList()); + + return BranchNode.create(containerTree, requiredFields.getBackingNode()); + } + + @Override + public TreeNode createTreeFromOptionalFieldValues( + final List> fieldValues) { + final int fieldsCount = getMaxFieldCount(); + checkArgument(fieldValues.size() < fieldsCount, "Wrong number of filed values"); + + final List allFields = new ArrayList<>(fieldsCount); + + final IntList activeFieldIndices = new IntArrayList(); + + for (int index = 0, fieldIndex = 0; index < fieldsCount; index++) { + final Optional currentOptionalField; + if (fieldIndex >= fieldValues.size()) { + currentOptionalField = Optional.empty(); + } else { + currentOptionalField = fieldValues.get(fieldIndex++); + } + + if (currentOptionalField.isPresent()) { + activeFieldIndices.add(index); + allFields.add(currentOptionalField.get()); + } else { + if (requiredFields.getBit(index)) { + throw new IllegalArgumentException("supposed to be active"); + } + allFields.add(SszNone.INSTANCE); + } + } + + assert allFields.size() == fieldsCount; + + final TreeNode activeFieldsTree = + activeFieldsSchema.ofBits(activeFieldIndices).getBackingNode(); + final TreeNode containerTree = + TreeUtil.createTree(allFields.stream().map(SszData::getBackingNode).toList()); + + return BranchNode.create(containerTree, activeFieldsTree); + } + + @Override + public int getFieldsCount() { + return definedChildrenNamedSchemas.size(); + } + + @Override + public boolean isFixedSize() { + return isFixedSize; + } + + private boolean calcIsFixedSize() { + if (hasOptionalFields) { + // for containers with optional fields we behave as variable ssz + return false; + } + + return requiredFields + .streamAllSetBits() + .mapToObj(this::getChildSchema) + .allMatch(SszType::isFixedSize); + } + + @Override + public boolean hasExtraDataInBackingTree() { + return true; + } + + @Override + public int getSszFixedPartSize() { + return sszFixedPartSize; + } + + int calcSszFixedPartSize() { + if (hasOptionalFields) { + // for containers with optional fields we behave as variable ssz + return 0; + } + + return calcSszFixedPartSize(requiredFields); + } + + /** + * Delegates active fields size: Profile and StableContainer have different + * serialization\deserialization rules + */ + abstract int getSszActiveFieldsSize(final TreeNode node); + + @Override + public int getSszVariablePartSize(final TreeNode node) { + if (hasOptionalFields) { + // containers with optional fields always behaves as variable ssz + final SszBitvector activeFields = getActiveFieldsBitvectorFromBackingNode(node); + + final int containerSize = + activeFields + .streamAllSetBits() + .map( + activeFieldIndex -> { + final SszSchema schema = getChildSchema(activeFieldIndex); + final int childSize = + schema.getSszSize(node.get(getChildGeneralizedIndex(activeFieldIndex))); + return schema.isFixedSize() ? childSize : childSize + SSZ_LENGTH_SIZE; + }) + .sum(); + + final int activeFieldsSize = getSszActiveFieldsSize(activeFields.getBackingNode()); + return containerSize + activeFieldsSize; + } + + if (isFixedSize()) { + return 0; + } + + final int[] size = {0}; + requiredFields + .streamAllSetBits() + .forEach( + index -> { + final SszSchema childType = getChildSchema(index); + if (!childType.isFixedSize()) { + size[0] += childType.getSszSize(node.get(getChildGeneralizedIndex(index))); + } + }); + + return size[0]; + } + + /** + * Delegates serialization to deriving classes: Profile and StableContainer have different + * serialization\deserialization rules + */ + abstract int sszSerializeActiveFields( + final SszBitvector activeFieldsBitvector, final SszWriter writer); + + @Override + public int sszSerializeTree(final TreeNode node, final SszWriter writer) { + final SszBitvector activeFieldsBitvector = getActiveFieldsBitvectorFromBackingNode(node); + // we won't write active field when no optional fields are permitted + final int activeFieldsWroteBytes = sszSerializeActiveFields(activeFieldsBitvector, writer); + + final int[] variableSizes = new int[activeFieldsBitvector.size()]; + + final int variableChildOffset = + activeFieldsBitvector + .streamAllSetBits() + .reduce( + calcSszFixedPartSize(activeFieldsBitvector), + (accumulatedOffset, activeFieldIndex) -> + accumulatedOffset + + serializeFixedChild( + writer, + this, + activeFieldIndex, + node, + variableSizes, + accumulatedOffset)); + + activeFieldsBitvector + .streamAllSetBits() + .forEach( + activeFieldIndex -> + serializeVariableChild(writer, this, activeFieldIndex, variableSizes, node)); + + return activeFieldsWroteBytes + variableChildOffset; + } + + protected int calcSszFixedPartSize(final SszBitvector activeFieldBitvector) { + return activeFieldBitvector + .streamAllSetBits() + .mapToObj(this::getChildSchema) + .mapToInt( + childType -> + childType.isFixedSize() ? childType.getSszFixedPartSize() : SSZ_LENGTH_SIZE) + .sum(); + } + + /** + * Delegates deserialization to deriving classes: Profile and StableContainer have different + * serialization\deserialization rules + */ + abstract SszBitvector sszDeserializeActiveFieldsTree(final SszReader reader); + + @Override + public TreeNode sszDeserializeTree(final SszReader reader) { + final SszBitvector activeFields = sszDeserializeActiveFieldsTree(reader); + checkArgument( + activeFields.getLastSetBitIndex() < definedChildrenSchemas.size(), + "All extra bits in the Bitvector[N] that exceed the number of fields MUST be 0"); + return BranchNode.create( + deserializeContainer(reader, activeFields), activeFields.getBackingNode()); + } + + private TreeNode deserializeContainer(final SszReader reader, final SszBitvector activeFields) { + int endOffset = reader.getAvailableBytes(); + int childCount = getMaxFieldCount(); + final Queue fixedChildrenSubtrees = new ArrayDeque<>(childCount); + final IntList variableChildrenOffsets = new IntArrayList(childCount); + activeFields + .streamAllSetBits() + .forEach( + i -> + deserializeFixedChild( + reader, fixedChildrenSubtrees, variableChildrenOffsets, this, i)); + + final ArrayDeque variableChildrenSizes = + validateAndPrepareForVariableChildrenDeserialization( + reader, variableChildrenOffsets, endOffset); + + final List childrenSubtrees = new ArrayList<>(childCount); + for (int i = 0; i < childCount; i++) { + if (!activeFields.getBit(i)) { + childrenSubtrees.add(SszPrimitiveSchemas.NONE_SCHEMA.getDefaultTree()); + continue; + } + deserializeVariableChild( + reader, childrenSubtrees, fixedChildrenSubtrees, variableChildrenSizes, this, i); + } + + return TreeUtil.createTree(childrenSubtrees); + } + + @Override + public SszBitvectorSchema getActiveFieldsSchema() { + return activeFieldsSchema; + } + + @Override + public long getChildGeneralizedIndex(final long elementIndex) { + return GIndexUtil.gIdxCompose( + CONTAINER_G_INDEX, + SszStableContainerBaseSchema.super.getChildGeneralizedIndex(elementIndex)); + } + + @Override + public SszLengthBounds getSszLengthBounds() { + return sszLengthBounds.get(); + } + + /** + * Delegates activeFields bounds: Profile and StableContainer have different + * serialization\deserialization rules + */ + abstract SszLengthBounds computeActiveFieldsSszLengthBounds(); + + /** + * Bounds are calculate as follows: + * + *

ActiveFields bitvector bounds added to + * + *

    + *
  1. Min bound is the min bound for the required only fields + *
  2. Max bound is the max bound for the required fields and all optional fields + *
+ */ + private SszLengthBounds computeSszLengthBounds() { + final SszLengthBounds requiredOnlyFieldsBounds = + requiredFields + .streamAllSetBits() + .mapToObj(this::computeSingleSchemaLengthBounds) + .reduce(SszLengthBounds.ZERO, SszLengthBounds::add); + + final SszLengthBounds includingOptionalFieldsBounds = + optionalFields + .streamAllSetBits() + .mapToObj(this::computeSingleSchemaLengthBounds) + .reduce(requiredOnlyFieldsBounds, SszLengthBounds::add); + + return requiredOnlyFieldsBounds + .or(includingOptionalFieldsBounds) + .add(computeActiveFieldsSszLengthBounds()); + } + + private SszLengthBounds computeSingleSchemaLengthBounds(final int fieldIndex) { + final SszSchema schema = getChildSchema(fieldIndex); + return schema + .getSszLengthBounds() + // dynamic sized children need 4-byte offset + .addBytes((schema.isFixedSize() ? 0 : SSZ_LENGTH_SIZE)) + // elements are not packed in containers + .ceilToBytes(); + } + + @Override + public void storeBackingNodes( + final TreeNodeStore nodeStore, + final int maxBranchLevelsSkipped, + final long rootGIndex, + final TreeNode node) { + final TreeNode containerSubtree = node.get(CONTAINER_G_INDEX); + final TreeNode activeFieldsBitvectorSubtree = node.get(BITVECTOR_G_INDEX); + + storeContainerBackingNodes( + getActiveFieldsBitvectorFromBackingNode(node), + nodeStore, + maxBranchLevelsSkipped, + getContainerGIndex(rootGIndex), + containerSubtree); + + final Bytes32[] children; + if (hasOptionalFields) { + activeFieldsSchema.storeBackingNodes( + nodeStore, + maxBranchLevelsSkipped, + getBitvectorGIndex(rootGIndex), + activeFieldsBitvectorSubtree); + + children = + new Bytes32[] { + containerSubtree.hashTreeRoot(), activeFieldsBitvectorSubtree.hashTreeRoot() + }; + } else { + children = new Bytes32[] {containerSubtree.hashTreeRoot()}; + } + + nodeStore.storeBranchNode(node.hashTreeRoot(), rootGIndex, 1, children); + } + + private void storeContainerBackingNodes( + final SszBitvector activeFields, + final TreeNodeStore nodeStore, + final int maxBranchLevelsSkipped, + final long rootGIndex, + final TreeNode node) { + + final int childDepth = treeDepth(); + if (childDepth == 0) { + // Only one child so wrapper is omitted + storeChildNode(nodeStore, maxBranchLevelsSkipped, rootGIndex, node); + return; + } + final long lastUsefulGIndex = + GIndexUtil.gIdxChildGIndex(rootGIndex, maxChunks() - 1, childDepth); + StoringUtil.storeNodesToDepth( + nodeStore, + maxBranchLevelsSkipped, + node, + rootGIndex, + childDepth, + lastUsefulGIndex, + (targetDepthNode, targetDepthGIndex) -> + storeChildNode( + nodeStore, + maxBranchLevelsSkipped, + targetDepthGIndex, + targetDepthNode, + activeFields)); + } + + public void storeChildNode( + final TreeNodeStore nodeStore, + final int maxBranchLevelsSkipped, + final long gIndex, + final TreeNode node, + final SszBitvector activeFields) { + final int childIndex = GIndexUtil.gIdxChildIndexFromGIndex(gIndex, treeDepth()); + if (activeFields.getBit(childIndex)) { + final SszSchema childSchema = getChildSchema(childIndex); + childSchema.storeBackingNodes(nodeStore, maxBranchLevelsSkipped, gIndex, node); + } else { + SszPrimitiveSchemas.NONE_SCHEMA.storeBackingNodes( + nodeStore, maxBranchLevelsSkipped, gIndex, node); + } + } + + @Override + public TreeNode loadBackingNodes( + final TreeNodeSource nodeSource, final Bytes32 rootHash, final long rootGIndex) { + if (TreeUtil.ZERO_TREES_BY_ROOT.containsKey(rootHash) || rootHash.equals(Bytes32.ZERO)) { + return getDefaultTree(); + } + final CompressedBranchInfo branchData = nodeSource.loadBranchNode(rootHash, rootGIndex); + if (hasOptionalFields) { + checkState( + branchData.getChildren().length == 2, + "Stable container root node must have exactly 2 children when allows optional fields"); + } else { + checkState( + branchData.getChildren().length == 1, + "Stable container root node must have exactly 1 children when optional fields are not allowed"); + } + + checkState(branchData.getDepth() == 1, "Stable container root node must have depth of 1"); + final Bytes32 containerHash = branchData.getChildren()[0]; + + final SszBitvector activeFields; + if (hasOptionalFields) { + final Bytes32 activeFieldsBitvectorHash = branchData.getChildren()[1]; + final TreeNode activeFieldsTreeNode = + activeFieldsSchema.loadBackingNodes( + nodeSource, activeFieldsBitvectorHash, getBitvectorGIndex(rootGIndex)); + activeFields = activeFieldsSchema.createFromBackingNode(activeFieldsTreeNode); + } else { + activeFields = requiredFields; + } + + int lastActiveFieldOrFirst = Math.max(0, activeFields.getLastSetBitIndex()); + final long lastUsefulGIndex = + GIndexUtil.gIdxChildGIndex(rootGIndex, lastActiveFieldOrFirst, treeDepth()); + final TreeNode containerTreeNode = + LoadingUtil.loadNodesToDepth( + nodeSource, + containerHash, + getContainerGIndex(rootGIndex), + treeDepth(), + defaultTreeNode.get(CONTAINER_G_INDEX), + lastUsefulGIndex, + (tns, childHash, childGIndex) -> + loadChildNode(activeFields, tns, childHash, childGIndex)); + + return BranchNode.create(containerTreeNode, activeFields.getBackingNode()); + } + + private TreeNode loadChildNode( + final SszBitvector activeFields, + final TreeNodeSource nodeSource, + final Bytes32 childHash, + final long childGIndex) { + final int childIndex = GIndexUtil.gIdxChildIndexFromGIndex(childGIndex, treeDepth()); + if (activeFields.getLastSetBitIndex() >= childIndex && activeFields.getBit(childIndex)) { + return getChildSchema(childIndex).loadBackingNodes(nodeSource, childHash, childGIndex); + } + return SszPrimitiveSchemas.NONE_SCHEMA.loadBackingNodes(nodeSource, childHash, childGIndex); + } + + @Override + public SszSchema getChildSchema(final int index) { + return definedChildrenSchemas.get(index); + } + + @Override + public List> getFieldSchemas() { + return definedChildrenSchemas; + } + + /** + * Get the index of a field by name + * + * @param fieldName the name of the field + * @return The index if it exists, otherwise -1 + */ + @Override + public int getFieldIndex(final String fieldName) { + return definedChildrenNamesToFieldIndex.getOrDefault(fieldName, -1); + } + + @Override + public String getContainerName() { + return !containerName.isEmpty() ? containerName : getClass().getName(); + } + + @Override + public List getFieldNames() { + return definedChildrenNames; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + AbstractSszStableContainerBaseSchema that = (AbstractSszStableContainerBaseSchema) o; + return definedChildrenSchemas.equals(that.definedChildrenSchemas) + && requiredFields.equals(that.requiredFields) + && optionalFields.equals(that.optionalFields); + } + + @Override + public int hashCode() { + return Objects.hash(definedChildrenSchemas, requiredFields, optionalFields); + } + + @Override + public String toString() { + return getContainerName(); + } +} diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszStableContainerSchema.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszStableContainerSchema.java new file mode 100644 index 00000000000..113498370d2 --- /dev/null +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszStableContainerSchema.java @@ -0,0 +1,65 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.infrastructure.ssz.schema.impl; + +import it.unimi.dsi.fastutil.ints.IntSet; +import java.util.List; +import java.util.Set; +import java.util.stream.IntStream; +import tech.pegasys.teku.infrastructure.ssz.SszStableContainer; +import tech.pegasys.teku.infrastructure.ssz.collections.SszBitvector; +import tech.pegasys.teku.infrastructure.ssz.schema.SszStableContainerSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.impl.AbstractSszContainerSchema.NamedSchema; +import tech.pegasys.teku.infrastructure.ssz.sos.SszLengthBounds; +import tech.pegasys.teku.infrastructure.ssz.sos.SszReader; +import tech.pegasys.teku.infrastructure.ssz.sos.SszWriter; +import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode; + +public abstract class AbstractSszStableContainerSchema + extends AbstractSszStableContainerBaseSchema implements SszStableContainerSchema { + + public AbstractSszStableContainerSchema( + final String name, + final List> definedChildrenSchemas, + final int maxFieldCount) { + super( + name, + definedChildrenSchemas, + Set.of(), + IntSet.of(IntStream.range(0, definedChildrenSchemas.size()).toArray()), + maxFieldCount); + } + + @Override + SszLengthBounds computeActiveFieldsSszLengthBounds() { + return getActiveFieldsSchema().getSszLengthBounds(); + } + + @Override + int getSszActiveFieldsSize(final TreeNode node) { + return getActiveFieldsSchema().getSszSize(node); + } + + @Override + int sszSerializeActiveFields(final SszBitvector activeFieldsBitvector, final SszWriter writer) { + return getActiveFieldsSchema().sszSerializeTree(activeFieldsBitvector.getBackingNode(), writer); + } + + @Override + SszBitvector sszDeserializeActiveFieldsTree(final SszReader reader) { + final SszReader activeFieldsReader = + reader.slice(getActiveFieldsSchema().getSszFixedPartSize()); + return getActiveFieldsSchema().sszDeserialize(activeFieldsReader); + } +} diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszVectorSchema.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszVectorSchema.java index 4116de5b953..1c4917b9e85 100644 --- a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszVectorSchema.java +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszVectorSchema.java @@ -142,6 +142,11 @@ public boolean isFixedSize() { return getElementSchema().isFixedSize(); } + @Override + public boolean hasExtraDataInBackingTree() { + return getElementSchema().hasExtraDataInBackingTree(); + } + @Override public int getSszVariablePartSize(final TreeNode node) { return getVariablePartSize(node, getLength()); diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/ContainerSchemaUtil.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/ContainerSchemaUtil.java new file mode 100644 index 00000000000..425ab037767 --- /dev/null +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/ContainerSchemaUtil.java @@ -0,0 +1,133 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.infrastructure.ssz.schema.impl; + +import static tech.pegasys.teku.infrastructure.ssz.schema.SszType.SSZ_LENGTH_SIZE; + +import it.unimi.dsi.fastutil.ints.IntList; +import java.util.ArrayDeque; +import java.util.List; +import java.util.Queue; +import tech.pegasys.teku.infrastructure.ssz.schema.SszCompositeSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.SszSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.SszType; +import tech.pegasys.teku.infrastructure.ssz.sos.SszDeserializeException; +import tech.pegasys.teku.infrastructure.ssz.sos.SszReader; +import tech.pegasys.teku.infrastructure.ssz.sos.SszWriter; +import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode; + +public class ContainerSchemaUtil { + static void deserializeFixedChild( + final SszReader reader, + final Queue fixedChildrenSubtrees, + final IntList variableChildrenOffsets, + final SszCompositeSchema containerSchema, + final int index) { + final SszSchema childType = containerSchema.getChildSchema(index); + if (childType.isFixedSize()) { + try (SszReader sszReader = reader.slice(childType.getSszFixedPartSize())) { + final TreeNode childNode = childType.sszDeserializeTree(sszReader); + fixedChildrenSubtrees.add(childNode); + } + } else { + final int childOffset = SszType.sszBytesToLength(reader.read(SSZ_LENGTH_SIZE)); + variableChildrenOffsets.add(childOffset); + } + } + + static ArrayDeque validateAndPrepareForVariableChildrenDeserialization( + final SszReader reader, final IntList variableChildrenOffsets, final int endOffset) { + + if (variableChildrenOffsets.isEmpty()) { + if (reader.getAvailableBytes() > 0) { + throw new SszDeserializeException("Invalid SSZ: unread bytes for fixed size container"); + } + } else { + if (variableChildrenOffsets.getInt(0) != endOffset - reader.getAvailableBytes()) { + throw new SszDeserializeException( + "First variable element offset doesn't match the end of fixed part"); + } + } + + variableChildrenOffsets.add(endOffset); + + final ArrayDeque variableChildrenSizes = + new ArrayDeque<>(variableChildrenOffsets.size() - 1); + for (int i = 0; i < variableChildrenOffsets.size() - 1; i++) { + variableChildrenSizes.add( + variableChildrenOffsets.getInt(i + 1) - variableChildrenOffsets.getInt(i)); + } + + if (variableChildrenSizes.stream().anyMatch(s -> s < 0)) { + throw new SszDeserializeException("Invalid SSZ: wrong child offsets"); + } + + return variableChildrenSizes; + } + + static void deserializeVariableChild( + final SszReader reader, + final List childrenSubtrees, + final Queue fixedChildrenSubtrees, + final ArrayDeque variableChildrenSizes, + final SszCompositeSchema containerSchema, + final int index) { + final SszSchema childType = containerSchema.getChildSchema(index); + if (childType.isFixedSize()) { + childrenSubtrees.add(fixedChildrenSubtrees.remove()); + } else { + try (SszReader sszReader = reader.slice(variableChildrenSizes.remove())) { + TreeNode childNode = childType.sszDeserializeTree(sszReader); + childrenSubtrees.add(childNode); + } + } + } + + /** + * @return variable child size, 0 if it is fixed. + */ + static int serializeFixedChild( + final SszWriter writer, + final SszCompositeSchema containerSchema, + final int index, + final TreeNode node, + final int[] variableSizes, + final int variableChildOffset) { + final TreeNode childSubtree = node.get(containerSchema.getChildGeneralizedIndex(index)); + final SszSchema childType = containerSchema.getChildSchema(index); + if (childType.isFixedSize()) { + final int size = childType.sszSerializeTree(childSubtree, writer); + assert size == childType.getSszFixedPartSize(); + return 0; + } + writer.write(SszType.sszLengthToBytes(variableChildOffset)); + final int variableChildSize = childType.getSszSize(childSubtree); + variableSizes[index] = variableChildSize; + return variableChildSize; + } + + static void serializeVariableChild( + final SszWriter writer, + final SszCompositeSchema containerSchema, + final int index, + final int[] variableSizes, + final TreeNode node) { + final SszSchema childType = containerSchema.getChildSchema(index); + if (!childType.isFixedSize()) { + TreeNode childSubtree = node.get(containerSchema.getChildGeneralizedIndex(index)); + int size = childType.sszSerializeTree(childSubtree, writer); + assert size == variableSizes[index]; + } + } +} diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/json/SszContainerTypeDefinition.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/json/SszContainerTypeDefinition.java index c70d57defe7..0252b3bf7cd 100644 --- a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/json/SszContainerTypeDefinition.java +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/json/SszContainerTypeDefinition.java @@ -19,6 +19,7 @@ import tech.pegasys.teku.infrastructure.ssz.SszContainer; import tech.pegasys.teku.infrastructure.ssz.SszData; import tech.pegasys.teku.infrastructure.ssz.schema.SszContainerSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.SszPrimitiveSchemas; import tech.pegasys.teku.infrastructure.ssz.schema.SszSchema; public class SszContainerTypeDefinition { @@ -45,6 +46,9 @@ void addField( final String childName, final int fieldIndex) { final SszSchema childSchema = schema.getChildSchema(fieldIndex); + if (childSchema.equals(SszPrimitiveSchemas.NONE_SCHEMA)) { + return; + } builder.withField( childName, childSchema.getJsonTypeDefinition(), diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/json/SszStableContainerBaseTypeDefinition.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/json/SszStableContainerBaseTypeDefinition.java new file mode 100644 index 00000000000..436f32c0402 --- /dev/null +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/json/SszStableContainerBaseTypeDefinition.java @@ -0,0 +1,83 @@ +/* + * Copyright Consensys Software Inc., 2022 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.infrastructure.ssz.schema.json; + +import java.util.List; +import java.util.Optional; +import tech.pegasys.teku.infrastructure.json.types.DeserializableObjectTypeDefinitionBuilder; +import tech.pegasys.teku.infrastructure.json.types.DeserializableTypeDefinition; +import tech.pegasys.teku.infrastructure.ssz.SszData; +import tech.pegasys.teku.infrastructure.ssz.SszStableContainerBase; +import tech.pegasys.teku.infrastructure.ssz.schema.SszSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.SszStableContainerBaseSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.impl.AbstractSszContainerSchema.NamedSchema; + +public class SszStableContainerBaseTypeDefinition { + + public static < + DataT extends SszStableContainerBase, SchemaT extends SszStableContainerBaseSchema> + DeserializableTypeDefinition createFor(final SchemaT schema) { + final DeserializableObjectTypeDefinitionBuilder> builder = + DeserializableTypeDefinition.object(); + builder + .name(schema.getContainerName()) + .initializer(() -> new StableContainerBuilder<>(schema)) + .finisher(StableContainerBuilder::build); + final List> definedChildrenSchemas = schema.getChildrenNamedSchemas(); + + for (int index = 0; index < definedChildrenSchemas.size(); index++) { + final NamedSchema schemaChild = definedChildrenSchemas.get(index); + addField(schema, builder, schemaChild.getSchema(), schemaChild.getName(), index); + } + return builder.build(); + } + + private static < + DataT extends SszStableContainerBase, SchemaT extends SszStableContainerBaseSchema> + void addField( + final SchemaT schema, + final DeserializableObjectTypeDefinitionBuilder> + builder, + final SszSchema childSchema, + final String childName, + final int fieldIndex) { + builder.withOptionalField( + childName, + childSchema.getJsonTypeDefinition(), + schema.isFieldAllowed(fieldIndex) + ? value -> value.getAnyOptional(fieldIndex) + : value -> Optional.empty(), + (b, value) -> b.setValue(fieldIndex, value)); + } + + private static class StableContainerBuilder { + private final SszStableContainerBaseSchema schema; + private final Optional[] values; + + @SuppressWarnings("unchecked") + public StableContainerBuilder(final SszStableContainerBaseSchema schema) { + this.schema = schema; + this.values = + (Optional[]) new Optional[schema.getChildrenNamedSchemas().size()]; + } + + public void setValue(final int childIndex, final Optional value) { + values[childIndex] = value; + } + + public DataT build() { + return schema.createFromOptionalFieldValues(List.of(values)); + } + } +} diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/tree/SszNodeTemplate.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/tree/SszNodeTemplate.java index d6ad2e65010..bd9768c07e8 100644 --- a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/tree/SszNodeTemplate.java +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/tree/SszNodeTemplate.java @@ -69,6 +69,9 @@ public boolean isLeaf() { public static SszNodeTemplate createFromType(final SszSchema sszSchema) { checkArgument(sszSchema.isFixedSize(), "Only fixed size types supported"); + checkArgument( + !sszSchema.hasExtraDataInBackingTree(), + "Types containing extra data in backing tree are not supported"); return createFromTree(sszSchema.getDefaultTree()); } diff --git a/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/AbstractSszStableContainerBaseTest.java b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/AbstractSszStableContainerBaseTest.java new file mode 100644 index 00000000000..895352968e7 --- /dev/null +++ b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/AbstractSszStableContainerBaseTest.java @@ -0,0 +1,94 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.infrastructure.ssz; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.NoSuchElementException; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import tech.pegasys.teku.infrastructure.ssz.collections.SszBitvector; +import tech.pegasys.teku.infrastructure.ssz.schema.SszStableContainerBaseSchema; + +public abstract class AbstractSszStableContainerBaseTest + implements SszCompositeTestBase, SszMutableRefCompositeTestBase { + + @Override + @MethodSource("sszMutableCompositeArguments") + @ParameterizedTest + @Disabled + public void set_shouldThrowWhenAppendingAboveMaxLen(final SszMutableComposite data) { + // doesn't apply to stable container + } + + @Override + public Stream sszMutableCompositeWithUpdateIndicesArguments() { + return SszDataTestBase.passWhenEmpty( + sszMutableComposites() + .filter(data -> data.size() > 0) + .map(data -> Arguments.of(data, streamValidIndices(data).boxed().toList()))); + } + + @Override + public IntStream streamOutOfBoundsIndices(final SszComposite data) { + final SszStableContainerBaseSchema schema = + (SszStableContainerBaseSchema) data.getSchema(); + + final IntStream notAllowedStream = + IntStream.range(0, schema.getMaxFieldCount()).filter(i -> !schema.isFieldAllowed(i)); + + return IntStream.concat( + notAllowedStream, + IntStream.of(-1, (int) Long.min(Integer.MAX_VALUE, schema.getMaxFieldCount()))); + } + + @Override + public IntStream streamValidIndices(final SszComposite data) { + final SszBitvector activeFields = + ((SszStableContainerBaseSchema) data.getSchema()) + .getActiveFieldsBitvectorFromBackingNode(data.getBackingNode()); + return activeFields.streamAllSetBits(); + } + + protected IntStream streamNotPresentIndices(final SszComposite data) { + final SszStableContainerBaseSchema schema = + ((SszStableContainerBaseSchema) data.getSchema()); + final SszBitvector activeFields = + schema.getActiveFieldsBitvectorFromBackingNode(data.getBackingNode()); + + return activeFields + .getSchema() + .ofBits( + IntStream.range(0, activeFields.getSchema().getLength()) + .filter(schema::isFieldAllowed) + .filter(i -> !activeFields.getBit(i)) + .toArray()) + .streamAllSetBits(); + } + + @MethodSource("sszDataArguments") + @ParameterizedTest + void get_throwsNoSuchElement(final SszComposite data) { + streamNotPresentIndices(data) + .forEach( + wrongIndex -> + assertThatThrownBy(() -> data.get(wrongIndex)) + .as("child %s", wrongIndex) + .isInstanceOf(NoSuchElementException.class)); + } +} diff --git a/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszCompositeListTest.java b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszCompositeListTest.java index 5108cddcfc7..df2e622bed0 100644 --- a/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszCompositeListTest.java +++ b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszCompositeListTest.java @@ -84,6 +84,11 @@ public boolean isFixedSize() { return true; } + @Override + public boolean hasExtraDataInBackingTree() { + return false; + } + @Override public int getSszFixedPartSize() { return 0; diff --git a/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszCompositeTestBase.java b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszCompositeTestBase.java index 7b216dec86d..47d4770a47f 100644 --- a/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszCompositeTestBase.java +++ b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszCompositeTestBase.java @@ -16,6 +16,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import java.util.stream.IntStream; import org.assertj.core.api.Assertions; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; @@ -28,23 +29,33 @@ public interface SszCompositeTestBase extends SszDataTestBase { @ParameterizedTest default void get_childSchemaMatches(final SszComposite data) { SszCompositeSchema schema = data.getSchema(); - for (int i = 0; i < data.size(); i++) { - SszData child = data.get(i); - assertThat(child).isNotNull(); - SszSchema childSchema = child.getSchema(); - assertThat(childSchema).isNotNull(); - assertThat(childSchema).isEqualTo(schema.getChildSchema(i)); - } + streamValidIndices(data) + .forEach( + i -> { + SszData child = data.get(i); + assertThat(child).as("child %s", i).isNotNull(); + SszSchema childSchema = child.getSchema(); + assertThat(childSchema) + .as("child schema %s", i) + .isNotNull() + .isEqualTo(schema.getChildSchema(i)); + }); + } + + default IntStream streamOutOfBoundsIndices(final SszComposite data) { + return IntStream.of( + -1, data.size(), (int) Long.min(Integer.MAX_VALUE, data.getSchema().getMaxLength())); } @MethodSource("sszDataArguments") @ParameterizedTest default void get_throwsOutOfBounds(final SszComposite data) { - assertThatThrownBy(() -> data.get(-1)).isInstanceOf(IndexOutOfBoundsException.class); - assertThatThrownBy(() -> data.get(data.size())).isInstanceOf(IndexOutOfBoundsException.class); - assertThatThrownBy( - () -> data.get((int) Long.min(Integer.MAX_VALUE, data.getSchema().getMaxLength()))) - .isInstanceOf(IndexOutOfBoundsException.class); + streamOutOfBoundsIndices(data) + .forEach( + wrongIndex -> + assertThatThrownBy(() -> data.get(wrongIndex)) + .as("child %s", wrongIndex) + .isInstanceOf(IndexOutOfBoundsException.class)); } @MethodSource("sszDataArguments") diff --git a/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszDataTestBase.java b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszDataTestBase.java index edc43272330..e2adc5cae99 100644 --- a/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszDataTestBase.java +++ b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszDataTestBase.java @@ -17,7 +17,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.util.List; -import java.util.stream.Collectors; +import java.util.stream.IntStream; import java.util.stream.Stream; import org.apache.tuweni.bytes.Bytes; import org.assertj.core.api.Assertions; @@ -32,9 +32,13 @@ @TestInstance(Lifecycle.PER_CLASS) public interface SszDataTestBase { + default IntStream streamValidIndices(final SszComposite data) { + return IntStream.range(0, data.size()); + } + // workaround for https://github.com/junit-team/junit5/issues/1477 static Stream passWhenEmpty(final Stream args) { - List list = args.collect(Collectors.toList()); + List list = args.toList(); Assumptions.assumeFalse(list.isEmpty()); return list.stream(); } diff --git a/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszMutableCompositeTestBase.java b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszMutableCompositeTestBase.java index 922d779a6c7..1997a3362b6 100644 --- a/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszMutableCompositeTestBase.java +++ b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszMutableCompositeTestBase.java @@ -17,8 +17,10 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static tech.pegasys.teku.infrastructure.ssz.SszDataTestBase.passWhenEmpty; +import it.unimi.dsi.fastutil.ints.IntList; import java.util.ArrayList; import java.util.List; +import java.util.OptionalInt; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -90,7 +92,10 @@ default Stream sszMutableCompositeWithUpdateIndicesArguments() { @ParameterizedTest default void set_throwsIndexOutOfBounds(final SszMutableComposite data) { SszData someData = getSomeNewChild(data.getSchema()); - assertThatThrownBy(() -> data.set(data.size() + 1, someData)) + + // +2 is required to fail the append usecase too + final int outOfBoundsIndex = streamValidIndices(data).max().orElse(0) + 2; + assertThatThrownBy(() -> data.set(outOfBoundsIndex, someData)) .isInstanceOf(IndexOutOfBoundsException.class); assertThatThrownBy(() -> data.set(-1, someData)).isInstanceOf(IndexOutOfBoundsException.class); } @@ -98,19 +103,22 @@ default void set_throwsIndexOutOfBounds(final SszMutableComposite data) @MethodSource("sszMutableCompositeArguments") @ParameterizedTest default void set_throwsNPE(final SszMutableComposite data) { - if (data.getSchema().getMaxLength() == 0) { - return; - } - assertThatThrownBy(() -> data.set(0, null)).isInstanceOf(NullPointerException.class); + final OptionalInt firstValidIndex = streamValidIndices(data).findFirst(); + Assumptions.assumeThat(firstValidIndex).isPresent(); + + assertThatThrownBy(() -> data.set(firstValidIndex.getAsInt(), null)) + .isInstanceOf(NullPointerException.class); } @MethodSource("sszMutableCompositeArguments") @ParameterizedTest default void set_shouldThrowWhenSchemaMismatch(final SszMutableComposite data) { Assumptions.assumeThat(data).isNotInstanceOf(SszMutablePrimitiveCollection.class); + final OptionalInt firstValidIndex = streamValidIndices(data).findFirst(); + Assumptions.assumeThat(firstValidIndex).isPresent(); for (int i = 0; i < data.size(); i++) { - assertThatThrownBy(() -> data.set(0, NON_EXISTING_SCHEMA_DATA)) + assertThatThrownBy(() -> data.set(firstValidIndex.getAsInt(), NON_EXISTING_SCHEMA_DATA)) .isInstanceOf(InvalidValueSchemaException.class); } } @@ -134,52 +142,53 @@ default void set_shouldMatchGet( data.set(updateIndex, updateValue); } - for (int i = 0; i < data.size(); i++) { - int idx = updateIndices.indexOf(i); - if (idx < 0) { - SszDataAssert.assertThatSszData(data.get(i)).isEqualByAllMeansTo(origData.get(i)); - } else { - SszData updateValue = newChildrenValues.get(idx); - SszDataAssert.assertThatSszData(data.get(i)).isEqualByAllMeansTo(updateValue); - } - } + streamValidIndices(data) + .forEach( + i -> { + int idx = updateIndices.indexOf(i); + if (idx < 0) { + SszDataAssert.assertThatSszData(data.get(i)).isEqualByAllMeansTo(origData.get(i)); + } else { + SszData updateValue = newChildrenValues.get(idx); + SszDataAssert.assertThatSszData(data.get(i)).isEqualByAllMeansTo(updateValue); + } + }); SszComposite data1 = data.commitChanges(); - for (int i = 0; i < data.size(); i++) { - int idx = updateIndices.indexOf(i); - if (idx < 0) { - SszDataAssert.assertThatSszData(data1.get(i)).isEqualByAllMeansTo(origData.get(i)); - } else { - SszData updateValue = newChildrenValues.get(idx); - SszDataAssert.assertThatSszData(data1.get(i)).isEqualByAllMeansTo(updateValue); - } - } + streamValidIndices(data) + .forEach( + i -> { + int idx = updateIndices.indexOf(i); + if (idx < 0) { + SszDataAssert.assertThatSszData(data1.get(i)).isEqualByAllMeansTo(origData.get(i)); + } else { + SszData updateValue = newChildrenValues.get(idx); + SszDataAssert.assertThatSszData(data1.get(i)).isEqualByAllMeansTo(updateValue); + } + }); SszComposite data2 = schema.createFromBackingNode(data1.getBackingNode()); - for (int i = 0; i < data.size(); i++) { - int idx = updateIndices.indexOf(i); - if (idx < 0) { - SszDataAssert.assertThatSszData((SszData) data2.get(i)) - .isEqualByAllMeansTo(origData.get(i)); - } else { - SszData updateValue = newChildrenValues.get(idx); - SszDataAssert.assertThatSszData((SszData) data2.get(i)).isEqualByAllMeansTo(updateValue); - } - } + streamValidIndices(data) + .forEach( + i -> { + int idx = updateIndices.indexOf(i); + if (idx < 0) { + SszDataAssert.assertThatSszData((SszData) data2.get(i)) + .isEqualByAllMeansTo(origData.get(i)); + } else { + SszData updateValue = newChildrenValues.get(idx); + SszDataAssert.assertThatSszData((SszData) data2.get(i)) + .isEqualByAllMeansTo(updateValue); + } + }); } - @MethodSource("sszMutableCompositeArguments") + @MethodSource("sszMutableCompositeWithUpdateIndicesArguments") @ParameterizedTest - default void set_shouldNotHaveSideEffects(final SszMutableComposite data) { - List updatedIndices = - IntStream.concat(IntStream.range(0, 2), IntStream.of(data.size() - 1)) - .distinct() - .filter(i -> i >= 0 && i < data.size()) - .boxed() - .collect(Collectors.toList()); - + default void set_shouldNotHaveSideEffects( + final SszMutableComposite data, final List updatedIndices) { SszComposite origData = data.commitChanges(); SszCompositeSchema schema = data.getSchema(); @@ -202,11 +211,15 @@ default void set_shouldNotHaveSideEffects(final SszMutableComposite dat SszDataAssert.assertThatSszData((SszComposite) data).isEqualByGettersTo(origData); SszDataAssert.assertThatSszData(data1).isEqualByAllMeansTo(origData); + final IntList validIndices = IntList.of(streamValidIndices(data).toArray()); + for (int i = 0; i < updatedIndices.size(); i++) { SszComposite updated = updatedData.get(i); for (int i1 = 0; i1 < updated.size(); i1++) { if (i1 != updatedIndices.get(i)) { - SszDataAssert.assertThatSszData(updated.get(i1)).isEqualByAllMeansTo(origData.get(i1)); + if (validIndices.contains(i1)) { + SszDataAssert.assertThatSszData(updated.get(i1)).isEqualByAllMeansTo(origData.get(i1)); + } } else { SszDataAssert.assertThatSszData(updated.get(i1)).isEqualByAllMeansTo(newChildren.get(i)); } @@ -267,16 +280,20 @@ default void set_shouldThrowWhenAppendingAboveMaxLen(final SszMutableComposite data) { - if (data.size() == 0) { - return; - } + final OptionalInt firstValidIndex = streamValidIndices(data).findFirst(); + Assumptions.assumeThat(firstValidIndex).isPresent(); + AtomicInteger invalidateCount = new AtomicInteger(); data.setInvalidator(__ -> invalidateCount.incrementAndGet()); - data.set(0, GENERATOR.randomData(data.getSchema().getChildSchema(0))); + data.set( + firstValidIndex.getAsInt(), + GENERATOR.randomData(data.getSchema().getChildSchema(firstValidIndex.getAsInt()))); assertThat(invalidateCount).hasValue(1); - data.update(0, __ -> GENERATOR.randomData(data.getSchema().getChildSchema(0))); + data.update( + firstValidIndex.getAsInt(), + __ -> GENERATOR.randomData(data.getSchema().getChildSchema(firstValidIndex.getAsInt()))); assertThat(invalidateCount).hasValue(2); } diff --git a/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszMutableRefCompositeTestBase.java b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszMutableRefCompositeTestBase.java index cc5011067d5..0ab61d943d2 100644 --- a/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszMutableRefCompositeTestBase.java +++ b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszMutableRefCompositeTestBase.java @@ -24,6 +24,7 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import tech.pegasys.teku.infrastructure.ssz.schema.SszSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.SszStableContainerBaseSchema; public interface SszMutableRefCompositeTestBase extends SszMutableCompositeTestBase { RandomSszDataGenerator GENERATOR = new RandomSszDataGenerator(); @@ -34,7 +35,7 @@ default Stream sszMutableByRefCompositeArguments() { .map(d -> (SszComposite) d) .flatMap( data -> - IntStream.range(0, data.size()) + streamValidIndices(data) .limit(32) .filter( i -> @@ -62,7 +63,7 @@ default void getByRef_childUpdatedByRefShouldCommit( SszDataAssert.assertThatSszData(data.get(updateChildIndex)) .isEqualByGettersTo(newChildValue) .isEqualBySszTo(newChildValue); - IntStream.range(0, data.size()) + streamValidIndices(data) .limit(32) .forEach( i -> { @@ -76,7 +77,7 @@ default void getByRef_childUpdatedByRefShouldCommit( SszDataAssert.assertThatSszData(updatedData).isNotEqualByAllMeansTo(data); SszDataAssert.assertThatSszData(updatedData.get(updateChildIndex)) .isEqualByAllMeansTo(newChildValue); - IntStream.range(0, data.size()) + streamValidIndices(data) .limit(32) .forEach( i -> { @@ -184,10 +185,24 @@ static SszData updateSomething(final SszMutableData mutableData) { SszMutableComposite mutableComposite = (SszMutableComposite) mutableData; Assumptions.assumeTrue(mutableComposite.size() > 0); SszComposite orig = mutableComposite.commitChanges(); - SszData newChildData = GENERATOR.randomData(mutableComposite.getSchema().getChildSchema(0)); - mutableComposite.set(0, newChildData); + + int fieldToSet = findFirstValidIndex(mutableData); + + SszData newChildData = + GENERATOR.randomData(mutableComposite.getSchema().getChildSchema(fieldToSet)); + mutableComposite.set(fieldToSet, newChildData); SszMutableComposite writableCopy = orig.createWritableCopy(); - writableCopy.set(0, newChildData); + writableCopy.set(fieldToSet, newChildData); return writableCopy.commitChanges(); } + + static int findFirstValidIndex(final SszMutableData mutableData) { + if (mutableData.getSchema() instanceof SszStableContainerBaseSchema stableSchema) { + return IntStream.range(0, stableSchema.getFieldsCount()) + .filter(stableSchema::isFieldAllowed) + .findFirst() + .orElseThrow(); + } + return 0; + } } diff --git a/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszProfileTest.java b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszProfileTest.java new file mode 100644 index 00000000000..c8b7676da8c --- /dev/null +++ b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszProfileTest.java @@ -0,0 +1,40 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.infrastructure.ssz; + +import java.util.stream.Stream; +import tech.pegasys.teku.infrastructure.ssz.RandomSszDataGenerator.StableContainerMode; +import tech.pegasys.teku.infrastructure.ssz.schema.SszProfileSchemaTest; + +public class SszProfileTest extends AbstractSszStableContainerBaseTest { + + @Override + public Stream sszData() { + RandomSszDataGenerator randomSCGen = + new RandomSszDataGenerator().withStableContainerMode(StableContainerMode.RANDOM); + ; + RandomSszDataGenerator emptySCGen = + new RandomSszDataGenerator().withStableContainerMode(StableContainerMode.EMPTY); + RandomSszDataGenerator fullSCGen = + new RandomSszDataGenerator().withStableContainerMode(StableContainerMode.FULL); + return SszProfileSchemaTest.testContainerSchemas() + .flatMap( + schema -> + Stream.of( + schema.getDefault(), + randomSCGen.randomData(schema), + emptySCGen.randomData(schema), + fullSCGen.randomData(schema))); + } +} diff --git a/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszStableContainerTest.java b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszStableContainerTest.java new file mode 100644 index 00000000000..0ed718fc02f --- /dev/null +++ b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/SszStableContainerTest.java @@ -0,0 +1,49 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.infrastructure.ssz; + +import java.util.stream.Stream; +import tech.pegasys.teku.infrastructure.ssz.RandomSszDataGenerator.StableContainerMode; +import tech.pegasys.teku.infrastructure.ssz.schema.SszStableContainerSchemaTest; + +public class SszStableContainerTest extends AbstractSszStableContainerBaseTest { + + @Override + public Stream sszData() { + RandomSszDataGenerator smallListsRandomSCGen = new RandomSszDataGenerator().withMaxListSize(1); + RandomSszDataGenerator largeListsRandomSCGen = + new RandomSszDataGenerator() + .withStableContainerMode(StableContainerMode.RANDOM) + .withMaxListSize(1024); + RandomSszDataGenerator emptySCGen = + new RandomSszDataGenerator().withStableContainerMode(StableContainerMode.EMPTY); + RandomSszDataGenerator fullSCGen = + new RandomSszDataGenerator().withStableContainerMode(StableContainerMode.FULL); + + RandomSszDataGenerator anotherRound = + new RandomSszDataGenerator().withStableContainerMode(StableContainerMode.RANDOM); + return SszStableContainerSchemaTest.testContainerSchemas() + .flatMap( + schema -> + Stream.of( + schema.getDefault(), + smallListsRandomSCGen.randomData(schema), + largeListsRandomSCGen.randomData(schema), + emptySCGen.randomData(schema), + fullSCGen.randomData(schema), + // more combinations of active fields + anotherRound.randomData(schema), + anotherRound.randomData(schema))); + } +} diff --git a/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/TestStableContainers.java b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/TestStableContainers.java new file mode 100644 index 00000000000..71030c4b9c1 --- /dev/null +++ b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/TestStableContainers.java @@ -0,0 +1,140 @@ +/* + * Copyright Consensys Software Inc., 2022 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.infrastructure.ssz; + +import static tech.pegasys.teku.infrastructure.ssz.schema.impl.AbstractSszContainerSchema.namedSchema; + +import java.util.List; +import java.util.Optional; +import java.util.Set; +import tech.pegasys.teku.infrastructure.ssz.TestContainers.TestContainer; +import tech.pegasys.teku.infrastructure.ssz.impl.SszProfileImpl; +import tech.pegasys.teku.infrastructure.ssz.impl.SszStableContainerImpl; +import tech.pegasys.teku.infrastructure.ssz.primitive.SszByte; +import tech.pegasys.teku.infrastructure.ssz.primitive.SszUInt64; +import tech.pegasys.teku.infrastructure.ssz.schema.SszPrimitiveSchemas; +import tech.pegasys.teku.infrastructure.ssz.schema.SszProfileSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.SszStableContainerSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.impl.AbstractSszContainerSchema.NamedSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.impl.AbstractSszProfileSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.impl.AbstractSszStableContainerSchema; +import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode; +import tech.pegasys.teku.infrastructure.unsigned.UInt64; + +public class TestStableContainers { + + static final int MAX_SHAPE_FIELD_COUNT = 17; + + static final List> SHAPE_SCHEMAS = + List.of( + namedSchema("side", SszPrimitiveSchemas.UINT64_SCHEMA), + namedSchema("color", SszPrimitiveSchemas.BYTE_SCHEMA), + namedSchema("radius", SszPrimitiveSchemas.UINT64_SCHEMA)); + + static final int SIDE_INDEX = 0; + static final int COLOR_INDEX = 1; + static final int RADIUS_INDEX = 2; + + public static class ShapeStableContainer extends SszStableContainerImpl { + + ShapeStableContainer( + final SszStableContainerSchema type, + final TreeNode backingNode) { + super(type, backingNode); + } + + Optional getSide() { + final Optional side = getAnyOptional(SIDE_INDEX); + return side.map(SszUInt64::get); + } + + Optional getColor() { + final Optional color = getAnyOptional(COLOR_INDEX); + return color.map(SszByte::get); + } + + Optional getRadius() { + final Optional radius = getAnyOptional(RADIUS_INDEX); + return radius.map(SszUInt64::get); + } + } + + public static final SszStableContainerSchema SHAPE_STABLE_CONTAINER_SCHEMA = + new AbstractSszStableContainerSchema<>("Shape", SHAPE_SCHEMAS, MAX_SHAPE_FIELD_COUNT) { + @Override + public ShapeStableContainer createFromBackingNode(final TreeNode node) { + return new ShapeStableContainer(this, node); + } + }; + + static final List> NESTED_SCHEMAS = + List.of( + namedSchema("byte", SszPrimitiveSchemas.BYTE_SCHEMA), + namedSchema("shapeStableContainer", SHAPE_STABLE_CONTAINER_SCHEMA), + namedSchema("testContainer", TestContainer.SSZ_SCHEMA)); + + public static final SszStableContainerSchema + NESTED_STABLE_CONTAINER_SCHEMA = + new AbstractSszStableContainerSchema<>("NestedStableContainer", NESTED_SCHEMAS, 8) { + @Override + public ShapeStableContainer createFromBackingNode(final TreeNode node) { + return new ShapeStableContainer(this, node); + } + }; + + public static class CircleProfile extends SszProfileImpl { + CircleProfile( + final SszProfileSchema type, final TreeNode backingNode) { + super(type, backingNode); + } + + Byte getColor() { + final SszByte color = getAny(COLOR_INDEX); + return color.get(); + } + + UInt64 getRadius() { + final SszUInt64 radius = getAny(RADIUS_INDEX); + return radius.get(); + } + } + + public static final SszProfileSchema CIRCLE_PROFILE_SCHEMA = + new AbstractSszProfileSchema<>( + "CircleProfile", + SHAPE_STABLE_CONTAINER_SCHEMA, + Set.of(RADIUS_INDEX, COLOR_INDEX), + Set.of()) { + @Override + public CircleProfile createFromBackingNode(final TreeNode node) { + return new CircleProfile(this, node); + } + }; + + static final List> NESTED_PROFILE_SCHEMAS = + List.of( + namedSchema("uint64", SszPrimitiveSchemas.UINT64_SCHEMA), + namedSchema("circleProfile", CIRCLE_PROFILE_SCHEMA), + namedSchema("testContainer", TestContainer.SSZ_SCHEMA)); + + public static final SszStableContainerSchema + NESTED_PROFILE_STABLE_CONTAINER_SCHEMA = + new AbstractSszStableContainerSchema<>( + "NestedProfileListStableContainer", NESTED_PROFILE_SCHEMAS, 8) { + @Override + public ShapeStableContainer createFromBackingNode(final TreeNode node) { + return new ShapeStableContainer(this, node); + } + }; +} diff --git a/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/schema/SszProfileSchemaTest.java b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/schema/SszProfileSchemaTest.java new file mode 100644 index 00000000000..e8defd50024 --- /dev/null +++ b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/schema/SszProfileSchemaTest.java @@ -0,0 +1,81 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.infrastructure.ssz.schema; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static tech.pegasys.teku.infrastructure.ssz.TestStableContainers.NESTED_PROFILE_STABLE_CONTAINER_SCHEMA; +import static tech.pegasys.teku.infrastructure.ssz.TestStableContainers.NESTED_STABLE_CONTAINER_SCHEMA; +import static tech.pegasys.teku.infrastructure.ssz.TestStableContainers.SHAPE_STABLE_CONTAINER_SCHEMA; + +import java.util.function.Function; +import java.util.stream.Stream; +import org.assertj.core.api.Assumptions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import tech.pegasys.teku.infrastructure.ssz.RandomSszProfileSchemaGenerator; + +public class SszProfileSchemaTest extends SszCompositeSchemaTestBase { + + public static Stream> testContainerSchemas() { + + return Stream.of( + // generates 10 variation of profiles over SHAPE stable container with a random mix of + // required and optionals + new RandomSszProfileSchemaGenerator(SHAPE_STABLE_CONTAINER_SCHEMA) + .randomProfileSchemasStream() + .limit(10), + + // generates 10 variation of profiles over NESTED stable container with a random mix of + // required and optionals + new RandomSszProfileSchemaGenerator(NESTED_STABLE_CONTAINER_SCHEMA) + .randomProfileSchemasStream() + .limit(10), + + // generates 10 variation of profiles over PROFILE NESTED stable container with a random + // mix of required and optionals + new RandomSszProfileSchemaGenerator(NESTED_PROFILE_STABLE_CONTAINER_SCHEMA) + .randomProfileSchemasStream() + .limit(10), + + // a nested with all optionals + new RandomSszProfileSchemaGenerator(NESTED_STABLE_CONTAINER_SCHEMA) + .withMaxRequiredFields(0) + .withMinOptionalFields(NESTED_STABLE_CONTAINER_SCHEMA.getFieldsCount()) + .randomProfileSchemasStream() + .limit(1), + + // all required + new RandomSszProfileSchemaGenerator(NESTED_STABLE_CONTAINER_SCHEMA) + .withMinRequiredFields(NESTED_STABLE_CONTAINER_SCHEMA.getFieldsCount()) + .withMaxOptionalFields(0) + .randomProfileSchemasStream() + .limit(1)) + .flatMap(Function.identity()); + } + + @Override + public Stream> testSchemas() { + return testContainerSchemas(); + } + + @MethodSource("testSchemaArguments") + @ParameterizedTest + @Override + void getChildSchema_shouldThrowIndexOutOfBounds(final SszCompositeSchema schema) { + Assumptions.assumeThat(schema.getMaxLength()).isLessThan(Integer.MAX_VALUE); + int tooBigIndex = ((SszProfileSchema) schema).getChildrenNamedSchemas().size(); + assertThatThrownBy(() -> schema.getChildSchema(tooBigIndex)) + .isInstanceOf(IndexOutOfBoundsException.class); + } +} diff --git a/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/schema/SszSchemaTestBase.java b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/schema/SszSchemaTestBase.java index b67f5d19117..8953d5eed0a 100644 --- a/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/schema/SszSchemaTestBase.java +++ b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/schema/SszSchemaTestBase.java @@ -127,6 +127,7 @@ protected void assertTreeRoundtrip( assertThatTreeNode(result).isTreeEqual(node); final SszData rebuiltData = schema.createFromBackingNode(result); assertThat(rebuiltData).isEqualTo(data); + assertThat(rebuiltData.sszSerialize()).isEqualTo(data.sszSerialize()); } @MethodSource("testSchemaArguments") diff --git a/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/schema/SszStableContainerSchemaTest.java b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/schema/SszStableContainerSchemaTest.java new file mode 100644 index 00000000000..733704607ea --- /dev/null +++ b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/schema/SszStableContainerSchemaTest.java @@ -0,0 +1,35 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.infrastructure.ssz.schema; + +import static tech.pegasys.teku.infrastructure.ssz.TestStableContainers.NESTED_PROFILE_STABLE_CONTAINER_SCHEMA; +import static tech.pegasys.teku.infrastructure.ssz.TestStableContainers.NESTED_STABLE_CONTAINER_SCHEMA; +import static tech.pegasys.teku.infrastructure.ssz.TestStableContainers.SHAPE_STABLE_CONTAINER_SCHEMA; + +import java.util.stream.Stream; + +public class SszStableContainerSchemaTest extends SszCompositeSchemaTestBase { + + public static Stream> testContainerSchemas() { + return Stream.of( + SHAPE_STABLE_CONTAINER_SCHEMA, + NESTED_STABLE_CONTAINER_SCHEMA, + NESTED_PROFILE_STABLE_CONTAINER_SCHEMA); + } + + @Override + public Stream> testSchemas() { + return testContainerSchemas(); + } +} diff --git a/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/schema/collections/SszCollectionSchemaTestBase.java b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/schema/collections/SszCollectionSchemaTestBase.java index 173075b1e1e..3e152dce7ad 100644 --- a/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/schema/collections/SszCollectionSchemaTestBase.java +++ b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/schema/collections/SszCollectionSchemaTestBase.java @@ -26,7 +26,9 @@ import tech.pegasys.teku.infrastructure.ssz.schema.SszCollectionSchema; import tech.pegasys.teku.infrastructure.ssz.schema.SszCompositeSchemaTestBase; import tech.pegasys.teku.infrastructure.ssz.schema.SszContainerSchemaTest; +import tech.pegasys.teku.infrastructure.ssz.schema.SszProfileSchemaTest; import tech.pegasys.teku.infrastructure.ssz.schema.SszSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.SszStableContainerSchemaTest; import tech.pegasys.teku.infrastructure.ssz.schema.SszUnionSchemaTest; public abstract class SszCollectionSchemaTestBase extends SszCompositeSchemaTestBase { @@ -35,6 +37,8 @@ public abstract class SszCollectionSchemaTestBase extends SszCompositeSchemaTest static Stream> complexElementSchemas() { return Stream.of( + SszProfileSchemaTest.testContainerSchemas(), + SszStableContainerSchemaTest.testContainerSchemas(), SszContainerSchemaTest.testContainerSchemas(), SszUnionSchemaTest.testUnionSchemas(), Stream.of( diff --git a/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/schema/collections/SszListSchemaTestBase.java b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/schema/collections/SszListSchemaTestBase.java index 223ea4d24f4..e3d1ee4f2ae 100644 --- a/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/schema/collections/SszListSchemaTestBase.java +++ b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/schema/collections/SszListSchemaTestBase.java @@ -40,7 +40,7 @@ public abstract class SszListSchemaTestBase extends SszCollectionSchemaTestBase private static Stream> createSuperNodeVariant( final SszSchema elementSchema) { // SuperNodes only support fixed sized content - return elementSchema.isFixedSize() + return elementSchema.isFixedSize() && !elementSchema.hasExtraDataInBackingTree() ? Stream.of(SszListSchema.create(elementSchema, 1L << 16, SszSchemaHints.sszSuperNode(8))) : Stream.empty(); } diff --git a/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszStableContainerBaseSchemaTest.java b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszStableContainerBaseSchemaTest.java new file mode 100644 index 00000000000..625995cfab9 --- /dev/null +++ b/infrastructure/ssz/src/test/java/tech/pegasys/teku/infrastructure/ssz/schema/impl/AbstractSszStableContainerBaseSchemaTest.java @@ -0,0 +1,537 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.infrastructure.ssz.schema.impl; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static tech.pegasys.teku.infrastructure.ssz.schema.SszStableContainerSchema.createFromNamedSchemasForProfileOnly; +import static tech.pegasys.teku.infrastructure.ssz.schema.impl.AbstractSszContainerSchema.namedSchema; + +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.Set; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.junit.jupiter.api.Test; +import tech.pegasys.teku.infrastructure.json.JsonTestUtil; +import tech.pegasys.teku.infrastructure.json.JsonUtil; +import tech.pegasys.teku.infrastructure.ssz.SszData; +import tech.pegasys.teku.infrastructure.ssz.SszProfile; +import tech.pegasys.teku.infrastructure.ssz.SszStableContainer; +import tech.pegasys.teku.infrastructure.ssz.SszStableContainerBase; +import tech.pegasys.teku.infrastructure.ssz.impl.SszProfileImpl; +import tech.pegasys.teku.infrastructure.ssz.impl.SszStableContainerImpl; +import tech.pegasys.teku.infrastructure.ssz.primitive.SszByte; +import tech.pegasys.teku.infrastructure.ssz.primitive.SszUInt64; +import tech.pegasys.teku.infrastructure.ssz.schema.SszListSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.SszPrimitiveSchemas; +import tech.pegasys.teku.infrastructure.ssz.schema.SszProfileSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.SszStableContainerSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.impl.AbstractSszContainerSchema.NamedSchema; +import tech.pegasys.teku.infrastructure.ssz.sos.SszLengthBounds; +import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode; +import tech.pegasys.teku.infrastructure.unsigned.UInt64; + +public class AbstractSszStableContainerBaseSchemaTest { + static final int MAX_SHAPE_FIELD_COUNT = 16; + + static final List> SHAPE_SCHEMAS = + List.of( + namedSchema("side", SszPrimitiveSchemas.UINT64_SCHEMA), + namedSchema("color", SszPrimitiveSchemas.UINT8_SCHEMA), + namedSchema("radius", SszPrimitiveSchemas.UINT64_SCHEMA), + namedSchema("style", SszPrimitiveSchemas.UINT8_SCHEMA)); + + static final int SIDE_INDEX = 0; + static final int COLOR_INDEX = 1; + static final int RADIUS_INDEX = 2; + static final int STYLE_INDEX = 3; + + static final Set SQUARE_SCHEMA_INDICES = Set.of(SIDE_INDEX, COLOR_INDEX); + + static final Set CIRCLE_SCHEMA_INDICES = Set.of(RADIUS_INDEX, COLOR_INDEX); + + static class ShapeStableContainer extends SszStableContainerImpl { + + ShapeStableContainer( + final SszStableContainerSchema type, + final TreeNode backingNode) { + super(type, backingNode); + } + + Optional getSide() { + final Optional side = getAnyOptional(SIDE_INDEX); + return side.map(SszUInt64::get); + } + + Optional getColor() { + final Optional color = getAnyOptional(COLOR_INDEX); + return color.map(SszByte::get); + } + + Optional getRadius() { + final Optional radius = getAnyOptional(RADIUS_INDEX); + return radius.map(SszUInt64::get); + } + } + + static class CircleProfile extends SszProfileImpl { + CircleProfile( + final SszProfileSchema type, final TreeNode backingNode) { + super(type, backingNode); + } + + Byte getColor() { + final SszByte color = getAny(COLOR_INDEX); + return color.get(); + } + + UInt64 getRadius() { + final SszUInt64 radius = getAny(RADIUS_INDEX); + return radius.get(); + } + } + + static class SquareProfile extends SszProfileImpl { + SquareProfile( + final SszProfileSchema type, final TreeNode backingNode) { + super(type, backingNode); + } + + Byte getColor() { + final SszByte color = getAny(COLOR_INDEX); + return color.get(); + } + + UInt64 getSide() { + final SszUInt64 side = getAny(SIDE_INDEX); + return side.get(); + } + } + + private static final SszStableContainerSchema + SHAPE_STABLE_CONTAINER_SCHEMA = + new AbstractSszStableContainerSchema<>("Shape", SHAPE_SCHEMAS, MAX_SHAPE_FIELD_COUNT) { + @Override + public ShapeStableContainer createFromBackingNode(final TreeNode node) { + return new ShapeStableContainer(this, node); + } + }; + + private static final SszProfileSchema CIRCLE_PROFILE_SCHEMA = + new AbstractSszProfileSchema<>( + "Circle", SHAPE_STABLE_CONTAINER_SCHEMA, CIRCLE_SCHEMA_INDICES, Set.of()) { + @Override + public CircleProfile createFromBackingNode(final TreeNode node) { + return new CircleProfile(this, node); + } + }; + + private static final SszProfileSchema SQUARE_PROFILE_SCHEMA = + new AbstractSszProfileSchema<>( + "Square", SHAPE_STABLE_CONTAINER_SCHEMA, SQUARE_SCHEMA_INDICES, Set.of()) { + @Override + public SquareProfile createFromBackingNode(final TreeNode node) { + return new SquareProfile(this, node); + } + }; + + private static final SszProfileSchema CIRCLE_PROFILE_WITH_OPTIONAL_SCHEMA = + new AbstractSszProfileSchema<>( + "Circle", SHAPE_STABLE_CONTAINER_SCHEMA, CIRCLE_SCHEMA_INDICES, Set.of(STYLE_INDEX)) { + @Override + public CircleProfile createFromBackingNode(final TreeNode node) { + return new CircleProfile(this, node); + } + }; + + private static final SszProfileSchema SQUARE_PROFILE_WITH_OPTIONAL_SCHEMA = + new AbstractSszProfileSchema<>( + "Square", SHAPE_STABLE_CONTAINER_SCHEMA, SQUARE_SCHEMA_INDICES, Set.of(STYLE_INDEX)) { + @Override + public SquareProfile createFromBackingNode(final TreeNode node) { + return new SquareProfile(this, node); + } + }; + + @Test + void stableContainerSanityTest() throws Exception { + + final ShapeStableContainer square = + SHAPE_STABLE_CONTAINER_SCHEMA.createFromOptionalFieldValues( + List.of( + Optional.of(SszUInt64.of(UInt64.valueOf(0x42))), + Optional.of(SszPrimitiveSchemas.UINT8_SCHEMA.boxed((byte) 1)))); + + assertSquare(square, (byte) 1, UInt64.valueOf(0x42)); + assertThat(square.hashTreeRoot()) + .isEqualTo( + Bytes32.fromHexString( + "0x676ebb1aa0a62edeae70c7c1e36be52910662cf662eead3ef524ffdfe7a61c59")); + assertThat(square.sszSerialize()).isEqualTo(Bytes.fromHexString("0x0300420000000000000001")); + + final ShapeStableContainer circle = + SHAPE_STABLE_CONTAINER_SCHEMA.createFromOptionalFieldValues( + List.of( + Optional.empty(), + Optional.of(SszPrimitiveSchemas.UINT8_SCHEMA.boxed((byte) 1)), + Optional.of(SszUInt64.of(UInt64.valueOf(0x42))))); + + assertCircle(circle, (byte) 1, UInt64.valueOf(0x42)); + assertThat(circle.hashTreeRoot()) + .isEqualTo( + Bytes32.fromHexString( + "0x905b9b66f05a75db8441d55fe82b081f00a306467b526f83a4222fa0642211bc")); + assertThat(circle.sszSerialize()).isEqualTo(Bytes.fromHexString("0x0600014200000000000000")); + + // json square round trip + final String squareJson = + JsonUtil.serialize(square, SHAPE_STABLE_CONTAINER_SCHEMA.getJsonTypeDefinition()); + + assertThat(JsonTestUtil.parse(squareJson)).containsOnlyKeys("side", "color"); + + final ShapeStableContainer squareFromJson = + JsonUtil.parse(squareJson, SHAPE_STABLE_CONTAINER_SCHEMA.getJsonTypeDefinition()); + assertThat(squareFromJson).isEqualTo(square); + + // json circle round trip + final String circleJson = + JsonUtil.serialize(circle, SHAPE_STABLE_CONTAINER_SCHEMA.getJsonTypeDefinition()); + + assertThat(JsonTestUtil.parse(circleJson)).containsOnlyKeys("radius", "color"); + + final ShapeStableContainer circleFromJson = + JsonUtil.parse(circleJson, SHAPE_STABLE_CONTAINER_SCHEMA.getJsonTypeDefinition()); + assertThat(circleFromJson).isEqualTo(circle); + + // ssz circle deserialization + + ShapeStableContainer deserializedCircle = + SHAPE_STABLE_CONTAINER_SCHEMA.sszDeserialize( + Bytes.fromHexString("0x0600014200000000000000")); + + assertCircle(circle, (byte) 1, UInt64.valueOf(0x42)); + assertThat(deserializedCircle).isEqualTo(circle); + + // ssz square deserialization + ShapeStableContainer deserializedSquare = + SHAPE_STABLE_CONTAINER_SCHEMA.sszDeserialize( + Bytes.fromHexString("0x0300420000000000000001")); + + assertSquare(square, (byte) 1, UInt64.valueOf(0x42)); + assertThat(deserializedSquare).isEqualTo(square); + } + + @Test + void profileSanityTest() throws Exception { + + final CircleProfile circle = + CIRCLE_PROFILE_SCHEMA.createFromFieldValues( + List.of( + SszPrimitiveSchemas.UINT8_SCHEMA.boxed((byte) 1), + SszUInt64.of(UInt64.valueOf(0x42)))); + + assertCircleProfile(circle, (byte) 1, UInt64.valueOf(0x42), Optional.empty(), false); + assertThat(circle.hashTreeRoot()) + .isEqualTo( + Bytes32.fromHexString( + "0x905b9b66f05a75db8441d55fe82b081f00a306467b526f83a4222fa0642211bc")); + assertThat(circle.sszSerialize()).isEqualTo(Bytes.fromHexString("0x014200000000000000")); + + final SquareProfile square = + SQUARE_PROFILE_SCHEMA.createFromFieldValues( + List.of( + SszUInt64.of(UInt64.valueOf(0x42)), + SszPrimitiveSchemas.UINT8_SCHEMA.boxed((byte) 1))); + + assertSquareProfile(square, (byte) 1, UInt64.valueOf(0x42), Optional.empty(), false); + assertThat(square.hashTreeRoot()) + .isEqualTo( + Bytes32.fromHexString( + "0x676ebb1aa0a62edeae70c7c1e36be52910662cf662eead3ef524ffdfe7a61c59")); + assertThat(square.sszSerialize()).isEqualTo(Bytes.fromHexString("0x420000000000000001")); + + // json square round trip + final String squareJson = + JsonUtil.serialize(square, SQUARE_PROFILE_SCHEMA.getJsonTypeDefinition()); + + assertThat(JsonTestUtil.parse(squareJson)).containsOnlyKeys("side", "color"); + + final SquareProfile squareFromJson = + JsonUtil.parse(squareJson, SQUARE_PROFILE_SCHEMA.getJsonTypeDefinition()); + assertThat(squareFromJson).isEqualTo(square); + + // json circle round trip + final String circleJson = + JsonUtil.serialize(circle, CIRCLE_PROFILE_SCHEMA.getJsonTypeDefinition()); + + assertThat(JsonTestUtil.parse(circleJson)).containsOnlyKeys("radius", "color"); + + final CircleProfile circleFromJson = + JsonUtil.parse(circleJson, CIRCLE_PROFILE_SCHEMA.getJsonTypeDefinition()); + assertThat(circleFromJson).isEqualTo(circle); + + final CircleProfile deserializedCircle = + CIRCLE_PROFILE_SCHEMA.sszDeserialize(Bytes.fromHexString("0x014200000000000000")); + + assertCircleProfile(circle, (byte) 1, UInt64.valueOf(0x42), Optional.empty(), false); + assertThat(deserializedCircle).isEqualTo(circle); + + final SquareProfile deserializedSquare = + SQUARE_PROFILE_SCHEMA.sszDeserialize(Bytes.fromHexString("0x420000000000000001")); + + assertSquareProfile(square, (byte) 1, UInt64.valueOf(0x42), Optional.empty(), false); + assertThat(deserializedSquare).isEqualTo(square); + } + + @Test + void profileWithOptionalSanityTest() throws Exception { + + final CircleProfile circle = + CIRCLE_PROFILE_WITH_OPTIONAL_SCHEMA.createFromOptionalFieldValues( + List.of( + Optional.empty(), + Optional.of(SszPrimitiveSchemas.UINT8_SCHEMA.boxed((byte) 1)), + Optional.of(SszUInt64.of(UInt64.valueOf(0x42))))); + + assertCircleProfile(circle, (byte) 1, UInt64.valueOf(0x42), Optional.empty(), true); + assertThat(circle.hashTreeRoot()) + .isEqualTo( + Bytes32.fromHexString( + "0x905b9b66f05a75db8441d55fe82b081f00a306467b526f83a4222fa0642211bc")); + assertThat(circle.sszSerialize()).isEqualTo(Bytes.fromHexString("0x00014200000000000000")); + + final CircleProfile circleWithOptional = + CIRCLE_PROFILE_WITH_OPTIONAL_SCHEMA.createFromOptionalFieldValues( + List.of( + Optional.empty(), + Optional.of(SszPrimitiveSchemas.UINT8_SCHEMA.boxed((byte) 1)), + Optional.of(SszUInt64.of(UInt64.valueOf(0x42))), + Optional.of(SszPrimitiveSchemas.UINT8_SCHEMA.boxed((byte) 3)))); + + assertCircleProfile( + circleWithOptional, (byte) 1, UInt64.valueOf(0x42), Optional.of((byte) 3), true); + + assertThat(circleWithOptional.hashTreeRoot()) + .isEqualTo( + Bytes32.fromHexString( + "0x969d1bbccfad8aa59dc9f7c0a4a79d80d6dc76ad0b88224ca413983b81eed232")); + assertThat(circleWithOptional.sszSerialize()) + .isEqualTo(Bytes.fromHexString("0x0101420000000000000003")); + + final SquareProfile squareWithOptional = + SQUARE_PROFILE_WITH_OPTIONAL_SCHEMA.createFromOptionalFieldValues( + List.of( + Optional.of(SszUInt64.of(UInt64.valueOf(0x42))), + Optional.of(SszPrimitiveSchemas.UINT8_SCHEMA.boxed((byte) 1)), + Optional.empty(), + Optional.of(SszPrimitiveSchemas.UINT8_SCHEMA.boxed((byte) 3)))); + + assertSquareProfile( + squareWithOptional, (byte) 1, UInt64.valueOf(0x42), Optional.of((byte) 3), true); + assertThat(squareWithOptional.hashTreeRoot()) + .isEqualTo( + Bytes32.fromHexString( + "0x162a5167b484695d5e2d18a5674c6d654650572542e058072431575ff5592db0")); + assertThat(squareWithOptional.sszSerialize()) + .isEqualTo(Bytes.fromHexString("0x0142000000000000000103")); + + // json squareWithOptional round trip + final String squareJson = + JsonUtil.serialize( + squareWithOptional, SQUARE_PROFILE_WITH_OPTIONAL_SCHEMA.getJsonTypeDefinition()); + + assertThat(JsonTestUtil.parse(squareJson)).containsOnlyKeys("side", "color", "style"); + + final SquareProfile squareFromJson = + JsonUtil.parse(squareJson, SQUARE_PROFILE_WITH_OPTIONAL_SCHEMA.getJsonTypeDefinition()); + assertThat(squareFromJson).isEqualTo(squareWithOptional); + + // json circle round trip (optional not set) + final String circleJson = + JsonUtil.serialize(circle, CIRCLE_PROFILE_WITH_OPTIONAL_SCHEMA.getJsonTypeDefinition()); + + assertThat(JsonTestUtil.parse(circleJson)).containsOnlyKeys("radius", "color"); + + final CircleProfile circleFromJson = + JsonUtil.parse(circleJson, CIRCLE_PROFILE_WITH_OPTIONAL_SCHEMA.getJsonTypeDefinition()); + assertThat(circleFromJson).isEqualTo(circle); + + final CircleProfile deserializedCircle = + CIRCLE_PROFILE_WITH_OPTIONAL_SCHEMA.sszDeserialize( + Bytes.fromHexString("0x00014200000000000000")); + + assertCircleProfile(deserializedCircle, (byte) 1, UInt64.valueOf(0x42), Optional.empty(), true); + assertThat(deserializedCircle).isEqualTo(circle); + + final SquareProfile deserializedSquare = + SQUARE_PROFILE_WITH_OPTIONAL_SCHEMA.sszDeserialize( + Bytes.fromHexString("0x0142000000000000000103")); + + assertSquareProfile( + squareWithOptional, (byte) 1, UInt64.valueOf(0x42), Optional.of((byte) 3), true); + assertThat(deserializedSquare).isEqualTo(squareWithOptional); + } + + @Test + void computeSszLengthBounds() { + final SszStableContainerSchema stableContainer = + createFromNamedSchemasForProfileOnly( + 48, + List.of( + namedSchema("side", SszPrimitiveSchemas.UINT64_SCHEMA), + namedSchema("color", SszPrimitiveSchemas.UINT8_SCHEMA), + namedSchema( + "description", SszListSchema.create(SszPrimitiveSchemas.BYTE_SCHEMA, 32)), + namedSchema("radius", SszPrimitiveSchemas.UINT64_SCHEMA), + namedSchema("style", SszPrimitiveSchemas.UINT8_SCHEMA), + namedSchema( + "description2", SszListSchema.create(SszPrimitiveSchemas.BYTE_SCHEMA, 32)))); + + final SszLengthBounds boundsWithOptionals = + getSszLengthBoundsForProfile(stableContainer, Set.of(0, 1, 2), Set.of(4, 5)); + assertThat(boundsWithOptionals.getMinBytes()).isEqualTo(14); + assertThat(boundsWithOptionals.getMaxBytes()).isEqualTo(83); + + final SszLengthBounds boundsWithoutOptionals = + getSszLengthBoundsForProfile(stableContainer, Set.of(0, 1, 2), Set.of()); + assertThat(boundsWithoutOptionals.getMinBytes()).isEqualTo(13); + assertThat(boundsWithoutOptionals.getMaxBytes()).isEqualTo(45); + + final SszLengthBounds stableContainerBounds = stableContainer.getSszLengthBounds(); + assertThat(stableContainerBounds.getMinBytes()).isEqualTo(6); + assertThat(stableContainerBounds.getMaxBytes()).isEqualTo(96); + } + + private SszLengthBounds getSszLengthBoundsForProfile( + final SszStableContainerSchema stableContainer, + final Set requiredFieldIndices, + final Set optionalFieldIndices) { + + return new AbstractSszProfileSchema<>( + "Test", stableContainer, requiredFieldIndices, optionalFieldIndices) { + + @Override + public SszProfileImpl createFromBackingNode(final TreeNode node) { + throw new UnsupportedOperationException(); + } + }.getSszLengthBounds(); + } + + @Test + void shouldThrowOutOfBoundsException() { + final CircleProfile circle = + CIRCLE_PROFILE_WITH_OPTIONAL_SCHEMA.createFromOptionalFieldValues( + List.of( + Optional.empty(), + Optional.of(SszPrimitiveSchemas.UINT8_SCHEMA.boxed((byte) 1)), + Optional.of(SszUInt64.of(UInt64.valueOf(0x42))))); + + assertThatThrownBy(() -> circle.get(MAX_SHAPE_FIELD_COUNT)) + .isInstanceOf(IndexOutOfBoundsException.class); + + final ShapeStableContainer square = + SHAPE_STABLE_CONTAINER_SCHEMA.createFromOptionalFieldValues( + List.of( + Optional.of(SszUInt64.of(UInt64.valueOf(0x42))), + Optional.of(SszPrimitiveSchemas.UINT8_SCHEMA.boxed((byte) 1)))); + + assertThatThrownBy(() -> square.get(MAX_SHAPE_FIELD_COUNT)) + .isInstanceOf(IndexOutOfBoundsException.class); + } + + private void assertSquare( + final SszStableContainer container, final byte color, final UInt64 side) { + assertOptionalFieldValue( + container, COLOR_INDEX, Optional.of(SszPrimitiveSchemas.UINT8_SCHEMA.boxed(color)), true); + assertOptionalFieldValue(container, SIDE_INDEX, Optional.of(SszUInt64.of(side)), true); + assertOptionalFieldValue(container, RADIUS_INDEX, Optional.empty(), true); + } + + private void assertCircle( + final SszStableContainer container, final byte color, final UInt64 radius) { + assertOptionalFieldValue( + container, COLOR_INDEX, Optional.of(SszPrimitiveSchemas.UINT8_SCHEMA.boxed(color)), true); + assertOptionalFieldValue(container, RADIUS_INDEX, Optional.of(SszUInt64.of(radius)), true); + assertOptionalFieldValue(container, SIDE_INDEX, Optional.empty(), true); + } + + private void assertSquareProfile( + final SszProfile container, + final byte color, + final UInt64 side, + final Optional style, + final boolean styleAllowed) { + assertFixedFieldValue( + container, COLOR_INDEX, Optional.of(SszPrimitiveSchemas.UINT8_SCHEMA.boxed(color))); + assertFixedFieldValue(container, SIDE_INDEX, Optional.of(SszUInt64.of(side))); + assertFixedFieldValue(container, RADIUS_INDEX, Optional.empty()); + + assertOptionalFieldValue( + container, STYLE_INDEX, style.map(SszPrimitiveSchemas.UINT8_SCHEMA::boxed), styleAllowed); + } + + private void assertCircleProfile( + final SszProfile container, + final byte color, + final UInt64 radius, + final Optional style, + final boolean styleAllowed) { + assertFixedFieldValue( + container, COLOR_INDEX, Optional.of(SszPrimitiveSchemas.UINT8_SCHEMA.boxed(color))); + assertFixedFieldValue(container, RADIUS_INDEX, Optional.of(SszUInt64.of(radius))); + assertFixedFieldValue(container, SIDE_INDEX, Optional.empty()); + + assertOptionalFieldValue( + container, STYLE_INDEX, style.map(SszPrimitiveSchemas.UINT8_SCHEMA::boxed), styleAllowed); + } + + private void assertOptionalFieldValue( + final SszStableContainerBase container, + final int fieldIndex, + final Optional value, + final boolean isAllowed) { + if (isAllowed) { + value.ifPresentOrElse( + sszData -> assertThat(container.get(fieldIndex)).isEqualTo(sszData), + () -> + assertThatThrownBy(() -> container.get(fieldIndex)) + .isInstanceOf(NoSuchElementException.class)); + + assertThat(container.getOptional(fieldIndex)).isEqualTo(value); + } else { + + assertThatThrownBy(() -> container.getOptional(fieldIndex)) + .isInstanceOf(IndexOutOfBoundsException.class); + assertThatThrownBy(() -> container.get(fieldIndex)) + .isInstanceOf(IndexOutOfBoundsException.class); + } + } + + private void assertFixedFieldValue( + final SszProfile container, final int fieldIndex, final Optional value) { + value.ifPresentOrElse( + sszData -> { + assertThat(container.get(fieldIndex)).isEqualTo(sszData); + assertThat(container.getOptional(fieldIndex)).isEqualTo(value); + }, + () -> { + assertThatThrownBy(() -> container.get(fieldIndex)) + .isInstanceOf(IndexOutOfBoundsException.class); + assertThatThrownBy(() -> container.getOptional(fieldIndex)) + .isInstanceOf(IndexOutOfBoundsException.class); + }); + } +} diff --git a/infrastructure/ssz/src/testFixtures/java/tech/pegasys/teku/infrastructure/ssz/RandomSszDataGenerator.java b/infrastructure/ssz/src/testFixtures/java/tech/pegasys/teku/infrastructure/ssz/RandomSszDataGenerator.java index 02d87211ba3..41afc8645ac 100644 --- a/infrastructure/ssz/src/testFixtures/java/tech/pegasys/teku/infrastructure/ssz/RandomSszDataGenerator.java +++ b/infrastructure/ssz/src/testFixtures/java/tech/pegasys/teku/infrastructure/ssz/RandomSszDataGenerator.java @@ -18,11 +18,13 @@ import java.util.Random; import java.util.function.Supplier; import java.util.stream.Collectors; +import java.util.stream.IntStream; import java.util.stream.Stream; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; import tech.pegasys.teku.infrastructure.bytes.Bytes4; +import tech.pegasys.teku.infrastructure.ssz.collections.SszBitvector; import tech.pegasys.teku.infrastructure.ssz.primitive.SszBit; import tech.pegasys.teku.infrastructure.ssz.primitive.SszByte; import tech.pegasys.teku.infrastructure.ssz.primitive.SszBytes32; @@ -38,6 +40,7 @@ import tech.pegasys.teku.infrastructure.ssz.schema.SszUnionSchema; import tech.pegasys.teku.infrastructure.ssz.schema.impl.AbstractSszContainerSchema; import tech.pegasys.teku.infrastructure.ssz.schema.impl.AbstractSszPrimitiveSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.impl.AbstractSszStableContainerBaseSchema; import tech.pegasys.teku.infrastructure.unsigned.UInt64; public class RandomSszDataGenerator { @@ -50,14 +53,17 @@ public class RandomSszDataGenerator { private final Random random; private final int maxListSize; + private final StableContainerMode stableContainerMode; public RandomSszDataGenerator() { - this(new Random(1), 16 * 1024); + this(new Random(1), 16 * 1024, StableContainerMode.FULL); } - public RandomSszDataGenerator(final Random random, final int maxListSize) { + public RandomSszDataGenerator( + final Random random, final int maxListSize, final StableContainerMode stableContainerMode) { this.random = random; this.maxListSize = maxListSize; + this.stableContainerMode = stableContainerMode; bitSupplier = () -> SszBit.of(random.nextBoolean()); byteSupplier = () -> SszByte.of(random.nextInt()); bytes4Supplier = () -> SszBytes4.of(Bytes4.rightPad(Bytes.random(4, random))); @@ -67,13 +73,25 @@ public RandomSszDataGenerator(final Random random, final int maxListSize) { } public RandomSszDataGenerator withMaxListSize(final int maxListSize) { - return new RandomSszDataGenerator(random, maxListSize); + return new RandomSszDataGenerator(random, maxListSize, stableContainerMode); + } + + public RandomSszDataGenerator withStableContainerMode(final StableContainerMode mode) { + return new RandomSszDataGenerator(random, maxListSize, mode); } public T randomData(final SszSchema schema) { return randomDataStream(schema).findFirst().orElseThrow(); } + private Optional randomStableContainerData(final SszSchema schema) { + return switch (stableContainerMode) { + case EMPTY -> Optional.empty(); + case FULL -> Optional.of(randomData(schema)); + case RANDOM -> random.nextBoolean() ? Optional.of(randomData(schema)) : Optional.empty(); + }; + } + @SuppressWarnings("unchecked") public Stream randomDataStream(final SszSchema schema) { if (schema instanceof AbstractSszPrimitiveSchema) { @@ -94,6 +112,33 @@ public Stream randomDataStream(final SszSchema schema) } else { throw new IllegalArgumentException("Unknown primitive schema: " + schema); } + } else if (schema + instanceof AbstractSszStableContainerBaseSchema stableContainerBaseSchema) { + return Stream.generate( + () -> { + final SszBitvector requiredFields = stableContainerBaseSchema.getRequiredFields(); + final SszBitvector optionalFields = stableContainerBaseSchema.getOptionalFields(); + final int lastIndex = + Math.max(requiredFields.getLastSetBitIndex(), optionalFields.getLastSetBitIndex()); + + final List> values = + IntStream.rangeClosed(0, lastIndex) + .mapToObj( + index -> { + if (requiredFields.getBit(index)) { + return Optional.of( + randomData(stableContainerBaseSchema.getChildSchema(index))); + } + if (optionalFields.getBit(index)) { + return randomStableContainerData( + stableContainerBaseSchema.getChildSchema(index)); + } + return Optional.empty(); + }) + .toList(); + + return (T) stableContainerBaseSchema.createFromOptionalFieldValues(values); + }); } else if (schema instanceof AbstractSszContainerSchema containerSchema) { return Stream.generate( () -> { @@ -147,4 +192,10 @@ public Stream randomDataStream(final SszSchema schema) throw new IllegalArgumentException("Unknown schema: " + schema); } } + + public enum StableContainerMode { + EMPTY, + FULL, + RANDOM + } } diff --git a/infrastructure/ssz/src/testFixtures/java/tech/pegasys/teku/infrastructure/ssz/RandomSszProfileSchemaGenerator.java b/infrastructure/ssz/src/testFixtures/java/tech/pegasys/teku/infrastructure/ssz/RandomSszProfileSchemaGenerator.java new file mode 100644 index 00000000000..320464070f8 --- /dev/null +++ b/infrastructure/ssz/src/testFixtures/java/tech/pegasys/teku/infrastructure/ssz/RandomSszProfileSchemaGenerator.java @@ -0,0 +1,189 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.infrastructure.ssz; + +import it.unimi.dsi.fastutil.Pair; +import java.util.HashSet; +import java.util.Random; +import java.util.Set; +import java.util.stream.Stream; +import tech.pegasys.teku.infrastructure.ssz.impl.SszProfileImpl; +import tech.pegasys.teku.infrastructure.ssz.schema.SszProfileSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.SszStableContainerSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.impl.AbstractSszProfileSchema; +import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode; + +public class RandomSszProfileSchemaGenerator { + private final Random random; + private final SszStableContainerSchema stableContainerSchema; + + private final int minRequiredFields; + private final int maxRequiredFields; + private final int minOptionalFields; + private final int maxOptionalFields; + + public RandomSszProfileSchemaGenerator(final SszStableContainerSchema stableContainerSchema) { + this( + new Random(1), + stableContainerSchema, + 0, + stableContainerSchema.getFieldsCount(), + 0, + stableContainerSchema.getFieldsCount()); + } + + public RandomSszProfileSchemaGenerator( + final Random random, + final SszStableContainerSchema stableContainerSchema, + final int minRequiredFields, + final int maxRequiredFields, + final int minOptionalFields, + final int maxOptionalFields) { + this.stableContainerSchema = stableContainerSchema; + this.random = random; + + this.minRequiredFields = minRequiredFields; + this.maxRequiredFields = maxRequiredFields; + this.minOptionalFields = minOptionalFields; + this.maxOptionalFields = maxOptionalFields; + } + + public Stream> randomProfileSchemasStream() { + return generateDistinctRandomRequiredOptionalSets() + .map(sets -> generateProfile(sets.left(), sets.right())); + } + + public RandomSszProfileSchemaGenerator withMinRequiredFields(final int minRequiredFields) { + return new RandomSszProfileSchemaGenerator( + random, + stableContainerSchema, + minRequiredFields, + maxRequiredFields, + minOptionalFields, + maxOptionalFields); + } + + public RandomSszProfileSchemaGenerator withMaxRequiredFields(final int maxRequiredFields) { + return new RandomSszProfileSchemaGenerator( + random, + stableContainerSchema, + minRequiredFields, + maxRequiredFields, + minOptionalFields, + maxOptionalFields); + } + + public RandomSszProfileSchemaGenerator withMinOptionalFields(final int minOptionalFields) { + return new RandomSszProfileSchemaGenerator( + random, + stableContainerSchema, + minRequiredFields, + maxRequiredFields, + minOptionalFields, + maxOptionalFields); + } + + public RandomSszProfileSchemaGenerator withMaxOptionalFields(final int maxOptionalFields) { + return new RandomSszProfileSchemaGenerator( + random, + stableContainerSchema, + minRequiredFields, + maxRequiredFields, + minOptionalFields, + maxOptionalFields); + } + + private Stream, Set>> generateDistinctRandomRequiredOptionalSets() { + return Stream.generate( + () -> { + final Set requiredFields = new HashSet<>(); + final Set optionalFields = new HashSet<>(); + + generateRandomNonIntersectingSets(requiredFields, optionalFields); + return Pair.of(requiredFields, optionalFields); + }) + // let's have unique pairs with at least an element as required or optional + .distinct() + .filter(pair -> !(pair.left().isEmpty() && pair.right().isEmpty())); + } + + private void generateRandomNonIntersectingSets( + final Set requiredFields, final Set optionalFields) { + + final int rangeSize = stableContainerSchema.getFieldsCount(); + + // Ensure the total minimum size does not exceed the range size + if (minRequiredFields + minOptionalFields > rangeSize) { + throw new IllegalArgumentException( + "The total minimum size of the subsets exceeds the range size."); + } + + // Randomly determine the sizes of the two subsets within the given constraints + final int requiredFieldsSize = + (maxRequiredFields > 0) + ? minRequiredFields + + random.nextInt(Math.min(maxRequiredFields, rangeSize) - minRequiredFields + 1) + : 0; + final int optionalFieldsSize = + (maxOptionalFields > 0) + ? minOptionalFields + + random.nextInt( + Math.min(maxOptionalFields, rangeSize - requiredFieldsSize) + - minOptionalFields + + 1) + : 0; + + final Set allNumbers = new HashSet<>(); + for (int i = 0; i < rangeSize; i++) { + allNumbers.add(i); + } + + while (requiredFields.size() < requiredFieldsSize) { + final int number = getRandomElement(allNumbers, random); + requiredFields.add(number); + allNumbers.remove(number); + } + + while (optionalFields.size() < optionalFieldsSize) { + final int number = getRandomElement(allNumbers, random); + optionalFields.add(number); + allNumbers.remove(number); + } + } + + private static int getRandomElement(final Set set, final Random random) { + final int index = random.nextInt(set.size()); + return set.stream().skip(index).findFirst().orElseThrow(IllegalStateException::new); + } + + private SszProfileSchema generateProfile( + final Set requiredFieldIndices, final Set optionalFieldIndices) { + + return new AbstractSszProfileSchema<>( + stableContainerSchema.getContainerName() + + "-Profile-Req" + + requiredFieldIndices + + "-Opt" + + optionalFieldIndices, + stableContainerSchema, + requiredFieldIndices, + optionalFieldIndices) { + + @Override + public SszProfileImpl createFromBackingNode(final TreeNode node) { + return new SszProfileImpl(this, node); + } + }; + } +} diff --git a/infrastructure/ssz/src/testFixtures/java/tech/pegasys/teku/infrastructure/ssz/SszDataAssert.java b/infrastructure/ssz/src/testFixtures/java/tech/pegasys/teku/infrastructure/ssz/SszDataAssert.java index 6f950ef5c02..295f7005dec 100644 --- a/infrastructure/ssz/src/testFixtures/java/tech/pegasys/teku/infrastructure/ssz/SszDataAssert.java +++ b/infrastructure/ssz/src/testFixtures/java/tech/pegasys/teku/infrastructure/ssz/SszDataAssert.java @@ -17,6 +17,8 @@ import java.util.Collections; import java.util.List; +import java.util.NoSuchElementException; +import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -130,7 +132,25 @@ private static List compareByGetters(final SszData actual, final SszData "Expected SszList size doesn't match actual: " + c2.size() + " != " + c1.size()); } for (int i = 0; i < c1.size(); i++) { - List res = compareByGetters(c1.get(i), c2.get(i)); + List res = List.of(); + + final Optional c1i = getOptionally(c1, i); + final Optional c2i = getOptionally(c2, i); + + if (c2i.isPresent() != c1i.isPresent()) { + res = + List.of( + "Expected field active: " + + c2i.isPresent() + + " Actual field active: " + + c1i.isPresent()); + } else { + if (c1i.isPresent()) { + // fields are both present, we can compare + res = compareByGetters(c1i.get(), c2i.get()); + } + } + if (!res.isEmpty()) { String traceDetails; if (actual instanceof SszContainer) { @@ -153,6 +173,14 @@ private static List compareByGetters(final SszData actual, final SszData } } + private static Optional getOptionally(final SszComposite composite, final int index) { + try { + return Optional.of(composite.get(index)); + } catch (NoSuchElementException | IndexOutOfBoundsException __) { + return Optional.empty(); + } + } + @SuppressWarnings("unchecked") private static List prepend(final List list, final T... args) { return Stream.concat(Stream.of(args), list.stream()).collect(Collectors.toList()); diff --git a/storage/src/property-test/java/tech/pegasys/teku/storage/server/kvstore/serialization/BeaconStateSerializerPropertyTest.java b/storage/src/property-test/java/tech/pegasys/teku/storage/server/kvstore/serialization/BeaconStateSerializerPropertyTest.java index 79ebce5463c..1d0c9344696 100644 --- a/storage/src/property-test/java/tech/pegasys/teku/storage/server/kvstore/serialization/BeaconStateSerializerPropertyTest.java +++ b/storage/src/property-test/java/tech/pegasys/teku/storage/server/kvstore/serialization/BeaconStateSerializerPropertyTest.java @@ -24,7 +24,7 @@ import tech.pegasys.teku.spec.util.DataStructureUtil; public class BeaconStateSerializerPropertyTest { - @Property(tries = 10) + @Property(tries = 20) public boolean roundTrip( @ForAll final int seed, @ForAll(supplier = SpecSupplier.class) final Spec spec,