Merge pull request #121 from vulcanize/allow-for-packed-storage-slots

VDB-318 Allow for packed storage slots

libraries/shared/factories/storage/transformer_test.go

@@ -100,4 +100,54 @@ var _ = Describe("Storage transformer", func() {
 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError(fakes.FakeError))
 	})
+
+	Describe("when a storage row contains more than one item packed in storage", func() {
+		var (
+			rawValue        = common.HexToAddress("000000000000000000000000000000000000000000000002a300000000002a30")
+			fakeBlockNumber = 123
+			fakeBlockHash   = "0x67890"
+			packedTypes     = make(map[int]utils.ValueType)
+		)
+		packedTypes[0] = utils.Uint48
+		packedTypes[1] = utils.Uint48
+
+		var fakeMetadata = utils.StorageValueMetadata{
+			Name:        "",
+			Keys:        nil,
+			Type:        utils.PackedSlot,
+			PackedTypes: packedTypes,
+		}
+
+		It("passes the decoded data items to the repository", func() {
+			mappings.Metadata = fakeMetadata
+			fakeRow := utils.StorageDiffRow{
+				Contract:     common.Address{},
+				BlockHash:    common.HexToHash(fakeBlockHash),
+				BlockHeight:  fakeBlockNumber,
+				StorageKey:   common.Hash{},
+				StorageValue: rawValue.Hash(),
+			}
+
+			err := t.Execute(fakeRow)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(repository.PassedBlockNumber).To(Equal(fakeBlockNumber))
+			Expect(repository.PassedBlockHash).To(Equal(common.HexToHash(fakeBlockHash).Hex()))
+			Expect(repository.PassedMetadata).To(Equal(fakeMetadata))
+			expectedPassedValue := make(map[int]string)
+			expectedPassedValue[0] = "10800"
+			expectedPassedValue[1] = "172800"
+			Expect(repository.PassedValue.(map[int]string)).To(Equal(expectedPassedValue))
+		})
+
+		It("returns error if creating a row fails", func() {
+			mappings.Metadata = fakeMetadata
+			repository.CreateErr = fakes.FakeError
+
+			err := t.Execute(utils.StorageDiffRow{StorageValue: rawValue.Hash()})
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
+		})
+	})
 })

libraries/shared/storage/utils/decoder.go

@@ -23,31 +23,82 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 )
 
+const (
+	bitsPerByte = 8
+)
+
 func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, error) {
 	switch metadata.Type {
 	case Uint256:
-		return decodeUint256(row.StorageValue.Bytes()), nil
+		return decodeInteger(row.StorageValue.Bytes()), nil
 	case Uint48:
-		return decodeUint48(row.StorageValue.Bytes()), nil
+		return decodeInteger(row.StorageValue.Bytes()), nil
+	case Uint128:
+		return decodeInteger(row.StorageValue.Bytes()), nil
 	case Address:
 		return decodeAddress(row.StorageValue.Bytes()), nil
 	case Bytes32:
 		return row.StorageValue.Hex(), nil
+	case PackedSlot:
+		return decodePackedSlot(row.StorageValue.Bytes(), metadata.PackedTypes), nil
 	default:
 		panic(fmt.Sprintf("can't decode unknown type: %d", metadata.Type))
 	}
 }
 
-func decodeUint256(raw []byte) string {
-	n := big.NewInt(0).SetBytes(raw)
-	return n.String()
-}
-
-func decodeUint48(raw []byte) string {
+func decodeInteger(raw []byte) string {
 	n := big.NewInt(0).SetBytes(raw)
 	return n.String()
 }
 
 func decodeAddress(raw []byte) string {
 	return common.BytesToAddress(raw).Hex()
 }
+
+func decodePackedSlot(raw []byte, packedTypes map[int]ValueType) map[int]string {
+	storageSlotData := raw
+	decodedStorageSlotItems := map[int]string{}
+	numberOfTypes := len(packedTypes)
+
+	for position := 0; position < numberOfTypes; position++ {
+		//get length of remaining storage date
+		lengthOfStorageData := len(storageSlotData)
+
+		//get item details (type, length, starting index, value bytes)
+		itemType := packedTypes[position]
+		lengthOfItem := getNumberOfBytes(itemType)
+		itemStartingIndex := lengthOfStorageData - lengthOfItem
+		itemValueBytes := storageSlotData[itemStartingIndex:]
+
+		//decode item's bytes and set in results map
+		decodedValue := decodeIndividualItems(itemValueBytes, itemType)
+		decodedStorageSlotItems[position] = decodedValue
+
+		//pop last item off raw slot data before moving on
+		storageSlotData = storageSlotData[0:itemStartingIndex]
+	}
+
+	return decodedStorageSlotItems
+}
+
+func decodeIndividualItems(itemBytes []byte, valueType ValueType) string {
+	switch valueType {
+	case Uint48:
+		return decodeInteger(itemBytes)
+	case Uint128:
+		return decodeInteger(itemBytes)
+	default:
+		panic(fmt.Sprintf("can't decode unknown type: %d", valueType))
+	}
+}
+
+func getNumberOfBytes(valueType ValueType) int {
+	switch valueType {
+	case Uint48:
+		return 48 / bitsPerByte
+	case Uint128:
+		return 128 / bitsPerByte
+	default:
+		panic(fmt.Sprintf("ValueType %d not recognized", valueType))
+	}
+}

libraries/shared/storage/utils/decoder_test.go

@@ -38,6 +38,17 @@ var _ = Describe("Storage decoder", func() {
 		Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
 	})
 
+	It("decodes uint128", func() {
+		fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000011123")
+		row := utils.StorageDiffRow{StorageValue: fakeInt}
+		metadata := utils.StorageValueMetadata{Type: utils.Uint128}
+
+		result, err := utils.Decode(row, metadata)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
+	})
+
 	It("decodes uint48", func() {
 		fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000123")
 		row := utils.StorageDiffRow{StorageValue: fakeInt}
@@ -59,4 +70,81 @@ var _ = Describe("Storage decoder", func() {
 		Expect(err).NotTo(HaveOccurred())
 		Expect(result).To(Equal(fakeAddress.Hex()))
 	})
+
+	Describe("when there are multiple items packed in the storage slot", func() {
+		It("decodes uint48 items", func() {
+			//this is a real storage data example
+			packedStorage := common.HexToHash("000000000000000000000000000000000000000000000002a300000000002a30")
+			row := utils.StorageDiffRow{StorageValue: packedStorage}
+			packedTypes := map[int]utils.ValueType{}
+			packedTypes[0] = utils.Uint48
+			packedTypes[1] = utils.Uint48
+
+			metadata := utils.StorageValueMetadata{
+				Type:        utils.PackedSlot,
+				PackedTypes: packedTypes,
+			}
+
+			result, err := utils.Decode(row, metadata)
+			decodedValues := result.(map[int]string)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(decodedValues[0]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a30").Bytes()).String()))
+			Expect(decodedValues[1]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a300").Bytes()).String()))
+		})
+
+		It("decodes 5 uint48 items", func() {
+			//TODO: this packedStorageHex was generated by hand, it would be nice to test this against
+			//real storage data that has several items packed into it
+			packedStorageHex := "0000000A5D1AFFFFFFFFFFFE00000009F3C600000002A300000000002A30"
+
+			packedStorage := common.HexToHash(packedStorageHex)
+			row := utils.StorageDiffRow{StorageValue: packedStorage}
+			packedTypes := map[int]utils.ValueType{}
+			packedTypes[0] = utils.Uint48
+			packedTypes[1] = utils.Uint48
+			packedTypes[2] = utils.Uint48
+			packedTypes[3] = utils.Uint48
+			packedTypes[4] = utils.Uint48
+
+			metadata := utils.StorageValueMetadata{
+				Type:        utils.PackedSlot,
+				PackedTypes: packedTypes,
+			}
+
+			result, err := utils.Decode(row, metadata)
+			decodedValues := result.(map[int]string)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(decodedValues[0]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a30").Bytes()).String()))
+			Expect(decodedValues[1]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a300").Bytes()).String()))
+			Expect(decodedValues[2]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("9F3C6").Bytes()).String()))
+			Expect(decodedValues[3]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("FFFFFFFFFFFE").Bytes()).String()))
+			Expect(decodedValues[4]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("A5D1A").Bytes()).String()))
+		})
+
+		It("decodes 2 uint128 items", func() {
+			//TODO: this packedStorageHex was generated by hand, it would be nice to test this against
+			//real storage data that has several items packed into it
+			packedStorageHex := "000000038D7EA4C67FF8E502B6730000" +
+				"0000000000000000AB54A98CEB1F0AD2"
+			packedStorage := common.HexToHash(packedStorageHex)
+			row := utils.StorageDiffRow{StorageValue: packedStorage}
+			packedTypes := map[int]utils.ValueType{}
+			packedTypes[0] = utils.Uint128
+			packedTypes[1] = utils.Uint128
+
+			metadata := utils.StorageValueMetadata{
+				Type:        utils.PackedSlot,
+				PackedTypes: packedTypes,
+			}
+
+			result, err := utils.Decode(row, metadata)
+			decodedValues := result.(map[int]string)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(decodedValues[0]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("AB54A98CEB1F0AD2").Bytes()).String()))
+			Expect(decodedValues[1]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("38D7EA4C67FF8E502B6730000").Bytes()).String()))
+		})
+	})
 })

libraries/shared/storage/utils/value.go

@@ -16,27 +16,54 @@
 
 package utils
 
+import "fmt"
+
 type ValueType int
 
 const (
 	Uint256 ValueType = iota
 	Uint48
+	Uint128
 	Bytes32
 	Address
+	PackedSlot
 )
 
 type Key string
 
 type StorageValueMetadata struct {
-	Name string
-	Keys map[Key]string
-	Type ValueType
+	Name        string
+	Keys        map[Key]string
+	Type        ValueType
+	PackedNames map[int]string    //zero indexed position in map => name of packed item
+	PackedTypes map[int]ValueType //zero indexed position in map => type of packed item
+}
+
+func GetStorageValueMetadata(name string, keys map[Key]string, valueType ValueType) StorageValueMetadata {
+	return getMetadata(name, keys, valueType, nil, nil)
+}
+
+func GetStorageValueMetadataForPackedSlot(name string, keys map[Key]string, valueType ValueType, packedNames map[int]string, packedTypes map[int]ValueType) StorageValueMetadata {
+	return getMetadata(name, keys, valueType, packedNames, packedTypes)
 }
 
-func GetStorageValueMetadata(name string, keys map[Key]string, t ValueType) StorageValueMetadata {
+func getMetadata(name string, keys map[Key]string, valueType ValueType, packedNames map[int]string, packedTypes map[int]ValueType) StorageValueMetadata {
+	assertPackedSlotArgs(valueType, packedNames, packedTypes)
+
 	return StorageValueMetadata{
-		Name: name,
-		Keys: keys,
-		Type: t,
+		Name:        name,
+		Keys:        keys,
+		Type:        valueType,
+		PackedNames: packedNames,
+		PackedTypes: packedTypes,
 	}
 }
+
+func assertPackedSlotArgs(valueType ValueType, packedNames map[int]string, packedTypes map[int]ValueType) {
+	if valueType == PackedSlot && (packedTypes == nil || packedNames == nil) {
+		panic(fmt.Sprintf("ValueType is PackedSlot. Expected PackedNames and PackedTypes to not be nil, but got PackedNames = %v and PackedTypes = %v", packedNames, packedTypes))
+	} else if (packedNames != nil && packedTypes != nil) && valueType != PackedSlot {
+		panic(fmt.Sprintf("PackedNames and PackedTypes passed in. Expected ValueType to equal PackedSlot (%v), but got %v.", PackedSlot, valueType))
+	}
+
+}

libraries/shared/storage/utils/value_test.go

@@ -7,7 +7,7 @@ import (
 )
 
 var _ = Describe("Storage value metadata getter", func() {
-	It("returns a storage value metadata instance with corresponding fields assigned", func() {
+	It("returns storage value metadata for a single storage variable", func() {
 		metadataName := "fake_name"
 		metadataKeys := map[utils.Key]string{"key": "value"}
 		metadataType := utils.Uint256
@@ -19,4 +19,60 @@ var _ = Describe("Storage value metadata getter", func() {
 		}
 		Expect(utils.GetStorageValueMetadata(metadataName, metadataKeys, metadataType)).To(Equal(expectedMetadata))
 	})
+
+	Describe("metadata for a packed storaged slot", func() {
+		It("returns metadata for multiple storage variables", func() {
+			metadataName := "fake_name"
+			metadataKeys := map[utils.Key]string{"key": "value"}
+			metadataType := utils.PackedSlot
+			metadataPackedNames := map[int]string{0: "name"}
+			metadataPackedTypes := map[int]utils.ValueType{0: utils.Uint48}
+
+			expectedMetadata := utils.StorageValueMetadata{
+				Name:        metadataName,
+				Keys:        metadataKeys,
+				Type:        metadataType,
+				PackedTypes: metadataPackedTypes,
+				PackedNames: metadataPackedNames,
+			}
+			Expect(utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, metadataPackedNames, metadataPackedTypes)).To(Equal(expectedMetadata))
+		})
+
+		It("panics if PackedTypes are nil when the type is PackedSlot", func() {
+			metadataName := "fake_name"
+			metadataKeys := map[utils.Key]string{"key": "value"}
+			metadataType := utils.PackedSlot
+			metadataPackedNames := map[int]string{0: "name"}
+
+			getMetadata := func() {
+				utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, metadataPackedNames, nil)
+			}
+			Expect(getMetadata).To(Panic())
+		})
+
+		It("panics if PackedNames are nil when the type is PackedSlot", func() {
+			metadataName := "fake_name"
+			metadataKeys := map[utils.Key]string{"key": "value"}
+			metadataType := utils.PackedSlot
+			metadataPackedTypes := map[int]utils.ValueType{0: utils.Uint48}
+
+			getMetadata := func() {
+				utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, nil, metadataPackedTypes)
+			}
+			Expect(getMetadata).To(Panic())
+		})
+
+		It("panics if valueType is not PackedSlot if PackedNames is populated", func() {
+			metadataName := "fake_name"
+			metadataKeys := map[utils.Key]string{"key": "value"}
+			metadataType := utils.Uint48
+			metadataPackedNames := map[int]string{0: "name"}
+			metadataPackedTypes := map[int]utils.ValueType{0: utils.Uint48}
+
+			getMetadata := func() {
+				utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, metadataPackedNames, metadataPackedTypes)
+			}
+			Expect(getMetadata).To(Panic())
+		})
+	})
 })