feat(lib/runtime): add extra required runtime imports for parachain v…

…alidation (#3254)

This PR adds following import calls:
    ext_default_child_storage_clear_prefix_version_2
    ext_default_child_storage_root_version_2
    ext_offchain_index_clear_version_1

lib/runtime/constants.go

@@ -5,9 +5,11 @@ package runtime
 
 const (
 	// v0.9 test API wasm
+	// This wasm is generated using https://github.com/ChainSafe/polkadot-spec.
 	HOST_API_TEST_RUNTIME     = "hostapi_runtime"
 	HOST_API_TEST_RUNTIME_FP  = "hostapi_runtime.compact.wasm"
-	HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/4d190603d21d4431888bcb1ec546c4dc03b7bf93/test/runtimes/hostapi/hostapi_runtime.compact.wasm?raw=true" //nolint:lll
+	HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/raw/master/test/" +
+		"runtimes/hostapi/hostapi_runtime.compact.wasm"
 
 	// v0.9.29 polkadot
 	POLKADOT_RUNTIME_v0929     = "polkadot_runtime-v929"

lib/runtime/interfaces.go

@@ -24,6 +24,7 @@ type Storage interface {
 	ClearChildStorage(keyToChild, key []byte) error
 	NextKey([]byte) []byte
 	ClearPrefixInChild(keyToChild, prefix []byte) error
+	ClearPrefixInChildWithLimit(keyToChild, prefix []byte, limit uint32) (uint32, bool, error)
 	GetChildNextKey(keyToChild, key []byte) ([]byte, error)
 	GetChild(keyToChild []byte) (*trie.Trie, error)
 	ClearPrefix(prefix []byte) (err error)

lib/runtime/storage/trie.go

@@ -259,6 +259,18 @@ func (s *TrieState) ClearPrefixInChild(keyToChild, prefix []byte) error {
 	return nil
 }
 
+func (s *TrieState) ClearPrefixInChildWithLimit(keyToChild, prefix []byte, limit uint32) (uint32, bool, error) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	child, err := s.t.GetChild(keyToChild)
+	if err != nil || child == nil {
+		return 0, false, err
+	}
+
+	return child.ClearPrefixLimit(prefix, limit)
+}
+
 // GetChildNextKey returns the next lexicographical larger key from child storage. If it does not exist, it returns nil.
 func (s *TrieState) GetChildNextKey(keyToChild, key []byte) ([]byte, error) {
 	s.lock.RLock()

lib/runtime/wasmer/imports.go

@@ -48,11 +48,13 @@ package wasmer
 // extern int64_t ext_default_child_storage_next_key_version_1(void *context, int64_t a, int64_t b);
 // extern int64_t ext_default_child_storage_read_version_1(void *context, int64_t a, int64_t b, int64_t c, int32_t d);
 // extern int64_t ext_default_child_storage_root_version_1(void *context, int64_t a);
+// extern int64_t ext_default_child_storage_root_version_2(void *context, int64_t a, int32_t b);
 // extern void ext_default_child_storage_set_version_1(void *context, int64_t a, int64_t b, int64_t c);
 // extern void ext_default_child_storage_storage_kill_version_1(void *context, int64_t a);
 // extern int32_t ext_default_child_storage_storage_kill_version_2(void *context, int64_t a, int64_t b);
 // extern int64_t ext_default_child_storage_storage_kill_version_3(void *context, int64_t a, int64_t b);
 // extern void ext_default_child_storage_clear_prefix_version_1(void *context, int64_t a, int64_t b);
+// extern int64_t ext_default_child_storage_clear_prefix_version_2(void *context, int64_t a, int64_t b, int64_t c);
 // extern int32_t ext_default_child_storage_exists_version_1(void *context, int64_t a, int64_t b);
 //
 // extern void ext_allocator_free_version_1(void *context, int32_t a);
@@ -66,6 +68,7 @@ package wasmer
 // extern int32_t ext_hashing_twox_128_version_1(void *context, int64_t a);
 // extern int32_t ext_hashing_twox_64_version_1(void *context, int64_t a);
 //
+// extern void ext_offchain_index_clear_version_1(void *context, int64_t a);
 // extern void ext_offchain_index_set_version_1(void *context, int64_t a, int64_t b);
 // extern int32_t ext_offchain_is_validator_version_1(void *context);
 // extern void ext_offchain_local_storage_clear_version_1(void *context, int32_t a, int64_t b);
@@ -1060,6 +1063,74 @@ func ext_default_child_storage_clear_prefix_version_1(context unsafe.Pointer, ch
 	}
 }
 
+// NewDigestItem returns a new VaryingDataType to represent a DigestItem
+func NewKillStorageResult(deleted uint32, allDeleted bool) scale.VaryingDataType {
+	killStorageResult := scale.MustNewVaryingDataType(new(noneRemain), new(someRemain))
+
+	var err error
+	if allDeleted {
+		err = killStorageResult.Set(noneRemain(deleted))
+	} else {
+		err = killStorageResult.Set(someRemain(deleted))
+	}
+
+	if err != nil {
+		panic(err)
+	}
+	return killStorageResult
+}
+
+//export ext_default_child_storage_clear_prefix_version_2
+func ext_default_child_storage_clear_prefix_version_2(context unsafe.Pointer, childStorageKey, prefixSpan,
+	limitSpan C.int64_t) C.int64_t {
+	logger.Debug("executing...")
+
+	instanceContext := wasm.IntoInstanceContext(context)
+	ctx := instanceContext.Data().(*runtime.Context)
+	storage := ctx.Storage
+
+	keyToChild := asMemorySlice(instanceContext, childStorageKey)
+	prefix := asMemorySlice(instanceContext, prefixSpan)
+
+	limitBytes := asMemorySlice(instanceContext, limitSpan)
+
+	var limit []byte
+	err := scale.Unmarshal(limitBytes, &limit)
+	if err != nil {
+		logger.Warnf("failed scale decoding limit: %s", err)
+		return mustToWasmMemoryNil(instanceContext)
+	}
+
+	if len(limit) == 0 {
+		// limit is None, set limit to max
+		limit = []byte{0xff, 0xff, 0xff, 0xff}
+	}
+
+	limitUint := binary.LittleEndian.Uint32(limit)
+
+	deleted, allDeleted, err := storage.ClearPrefixInChildWithLimit(
+		keyToChild, prefix, limitUint)
+	if err != nil {
+		logger.Errorf("failed to clear prefix in child with limit: %s", err)
+	}
+
+	killStorageResult := NewKillStorageResult(deleted, allDeleted)
+
+	encodedKillStorageResult, err := scale.Marshal(killStorageResult)
+	if err != nil {
+		logger.Errorf("failed to encode result: %s", err)
+		return 0
+	}
+
+	resultSpan, err := toWasmMemoryOptional(instanceContext, encodedKillStorageResult)
+	if err != nil {
+		logger.Errorf("failed to allocate: %s", err)
+		return 0
+	}
+
+	return C.int64_t(resultSpan)
+}
+
 //export ext_default_child_storage_exists_version_1
 func ext_default_child_storage_exists_version_1(context unsafe.Pointer,
 	childStorageKey, key C.int64_t) C.int32_t {
@@ -1158,6 +1229,13 @@ func ext_default_child_storage_root_version_1(context unsafe.Pointer,
 	return C.int64_t(root)
 }
 
+//export ext_default_child_storage_root_version_2
+func ext_default_child_storage_root_version_2(context unsafe.Pointer,
+	childStorageKey C.int64_t, stateVersion C.int32_t) (ptrSize C.int64_t) {
+	// TODO: Implement this after we have storage trie version 1 implemented #2418
+	return ext_default_child_storage_root_version_1(context, childStorageKey)
+}
+
 //export ext_default_child_storage_set_version_1
 func ext_default_child_storage_set_version_1(context unsafe.Pointer,
 	childStorageKeySpan, keySpan, valueSpan C.int64_t) {
@@ -1503,6 +1581,22 @@ func ext_offchain_index_set_version_1(context unsafe.Pointer, keySpan, valueSpan
 	}
 }
 
+//export ext_offchain_index_clear_version_1
+func ext_offchain_index_clear_version_1(context unsafe.Pointer, keySpan C.int64_t) {
+	// Remove a key and its associated value from the Offchain DB.
+	// https://github.com/paritytech/substrate/blob/4d608f9c42e8d70d835a748fa929e59a99497e90/primitives/io/src/lib.rs#L1213
+	logger.Trace("executing...")
+
+	instanceContext := wasm.IntoInstanceContext(context)
+	runtimeCtx := instanceContext.Data().(*runtime.Context)
+
+	storageKey := asMemorySlice(instanceContext, keySpan)
+	err := runtimeCtx.NodeStorage.BaseDB.Del(storageKey)
+	if err != nil {
+		logger.Errorf("failed to set value in raw storage: %s", err)
+	}
+}
+
 //export ext_offchain_local_storage_clear_version_1
 func ext_offchain_local_storage_clear_version_1(context unsafe.Pointer, kind C.int32_t, key C.int64_t) {
 	logger.Trace("executing...")
@@ -2116,12 +2210,14 @@ func importsNodeRuntime() (imports *wasm.Imports, err error) {
 		{"ext_crypto_sr25519_verify_version_2", ext_crypto_sr25519_verify_version_2, C.ext_crypto_sr25519_verify_version_2},
 		{"ext_crypto_start_batch_verify_version_1", ext_crypto_start_batch_verify_version_1, C.ext_crypto_start_batch_verify_version_1},
 		{"ext_default_child_storage_clear_prefix_version_1", ext_default_child_storage_clear_prefix_version_1, C.ext_default_child_storage_clear_prefix_version_1},
+		{"ext_default_child_storage_clear_prefix_version_2", ext_default_child_storage_clear_prefix_version_2, C.ext_default_child_storage_clear_prefix_version_2},
 		{"ext_default_child_storage_clear_version_1", ext_default_child_storage_clear_version_1, C.ext_default_child_storage_clear_version_1},
 		{"ext_default_child_storage_exists_version_1", ext_default_child_storage_exists_version_1, C.ext_default_child_storage_exists_version_1},
 		{"ext_default_child_storage_get_version_1", ext_default_child_storage_get_version_1, C.ext_default_child_storage_get_version_1},
 		{"ext_default_child_storage_next_key_version_1", ext_default_child_storage_next_key_version_1, C.ext_default_child_storage_next_key_version_1},
 		{"ext_default_child_storage_read_version_1", ext_default_child_storage_read_version_1, C.ext_default_child_storage_read_version_1},
 		{"ext_default_child_storage_root_version_1", ext_default_child_storage_root_version_1, C.ext_default_child_storage_root_version_1},
+		{"ext_default_child_storage_root_version_2", ext_default_child_storage_root_version_2, C.ext_default_child_storage_root_version_2},
 		{"ext_default_child_storage_set_version_1", ext_default_child_storage_set_version_1, C.ext_default_child_storage_set_version_1},
 		{"ext_default_child_storage_storage_kill_version_1", ext_default_child_storage_storage_kill_version_1, C.ext_default_child_storage_storage_kill_version_1},
 		{"ext_default_child_storage_storage_kill_version_2", ext_default_child_storage_storage_kill_version_2, C.ext_default_child_storage_storage_kill_version_2},
@@ -2139,6 +2235,7 @@ func importsNodeRuntime() (imports *wasm.Imports, err error) {
 		{"ext_misc_print_num_version_1", ext_misc_print_num_version_1, C.ext_misc_print_num_version_1},
 		{"ext_misc_print_utf8_version_1", ext_misc_print_utf8_version_1, C.ext_misc_print_utf8_version_1},
 		{"ext_misc_runtime_version_version_1", ext_misc_runtime_version_version_1, C.ext_misc_runtime_version_version_1},
+		{"ext_offchain_index_clear_version_1", ext_offchain_index_clear_version_1, C.ext_offchain_index_clear_version_1},
 		{"ext_offchain_http_request_add_header_version_1", ext_offchain_http_request_add_header_version_1, C.ext_offchain_http_request_add_header_version_1},
 		{"ext_offchain_http_request_start_version_1", ext_offchain_http_request_start_version_1, C.ext_offchain_http_request_start_version_1},
 		{"ext_offchain_index_set_version_1", ext_offchain_index_set_version_1, C.ext_offchain_index_set_version_1},

lib/runtime/wasmer/imports_test.go

@@ -33,6 +33,26 @@ var testChildKey = []byte("childKey")
 var testKey = []byte("key")
 var testValue = []byte("value")
 
+func Test_ext_offchain_index_clear_version_1(t *testing.T) {
+	inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME)
+
+	err := inst.ctx.NodeStorage.BaseDB.Put(testKey, testValue)
+	require.NoError(t, err)
+
+	value, err := inst.ctx.NodeStorage.BaseDB.Get(testKey)
+	require.NoError(t, err)
+	require.Equal(t, testValue, value)
+
+	encKey, err := scale.Marshal(testKey)
+	require.NoError(t, err)
+
+	_, err = inst.Exec("rtm_ext_offchain_index_clear_version_1", encKey)
+	require.NoError(t, err)
+
+	_, err = inst.ctx.NodeStorage.BaseDB.Get(testKey)
+	require.ErrorIs(t, err, chaindb.ErrKeyNotFound)
+}
+
 func Test_ext_offchain_timestamp_version_1(t *testing.T) {
 	inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME)
 	runtimeFunc, ok := inst.vm.Exports["rtm_ext_offchain_timestamp_version_1"]
@@ -1283,6 +1303,58 @@ func Test_ext_default_child_storage_clear_version_1(t *testing.T) {
 	require.Nil(t, val)
 }
 
+func Test_ext_default_child_storage_clear_prefix_version_2(t *testing.T) {
+	inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME)
+
+	prefix := []byte("key")
+
+	testKeyValuePair := []struct {
+		key   []byte
+		value []byte
+	}{
+		{[]byte("keyOne"), []byte("value1")},
+		{[]byte("keyTwo"), []byte("value2")},
+		{[]byte("keyThree"), []byte("value3")},
+	}
+
+	err := inst.ctx.Storage.SetChild(testChildKey, trie.NewEmptyTrie())
+	require.NoError(t, err)
+
+	for _, kv := range testKeyValuePair {
+		err = inst.ctx.Storage.SetChildStorage(testChildKey, kv.key, kv.value)
+		require.NoError(t, err)
+	}
+
+	// Confirm if value is set
+	keys, err := inst.ctx.Storage.(*storage.TrieState).GetKeysWithPrefixFromChild(testChildKey, prefix)
+	require.NoError(t, err)
+	require.Equal(t, 3, len(keys))
+
+	encChildKey, err := scale.Marshal(testChildKey)
+	require.NoError(t, err)
+
+	encPrefix, err := scale.Marshal(prefix)
+	require.NoError(t, err)
+
+	testLimit := uint32(1)
+	testLimitBytes := make([]byte, 4)
+	binary.LittleEndian.PutUint32(testLimitBytes, testLimit)
+
+	encLimit, err := scale.Marshal(&testLimitBytes)
+	require.NoError(t, err)
+
+	data := append(encChildKey, encPrefix...)
+	data = append(data, encLimit...)
+
+	_, err = inst.Exec("rtm_ext_default_child_storage_clear_prefix_version_2", data)
+	require.NoError(t, err)
+
+	keys, err = inst.ctx.Storage.(*storage.TrieState).GetKeysWithPrefixFromChild(testChildKey, prefix)
+	require.NoError(t, err)
+	// since one key is removed, there will be two remaining.
+	require.Equal(t, 2, len(keys))
+}
+
 func Test_ext_default_child_storage_clear_prefix_version_1(t *testing.T) {
 	t.Parallel()
 	inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME)

lib/runtime/wazero/imports.go

@@ -1111,6 +1111,73 @@ func ext_default_child_storage_clear_prefix_version_1(
 	}
 }
 
+// NewDigestItem returns a new VaryingDataType to represent a DigestItem
+func NewKillStorageResult(deleted uint32, allDeleted bool) scale.VaryingDataType {
+	killStorageResult := scale.MustNewVaryingDataType(new(noneRemain), new(someRemain))
+
+	var err error
+	if allDeleted {
+		err = killStorageResult.Set(noneRemain(deleted))
+	} else {
+		err = killStorageResult.Set(someRemain(deleted))
+	}
+
+	if err != nil {
+		panic(err)
+	}
+	return killStorageResult
+}
+
+//export ext_default_child_storage_clear_prefix_version_2
+func ext_default_child_storage_clear_prefix_version_2(ctx context.Context, m api.Module,
+	childStorageKey, prefixSpan, limitSpan uint64) uint64 {
+
+	rtCtx := ctx.Value(runtimeContextKey).(*runtime.Context)
+	if rtCtx == nil {
+		panic("nil runtime context")
+	}
+	storage := rtCtx.Storage
+
+	keyToChild := read(m, childStorageKey)
+	prefix := read(m, prefixSpan)
+	limitBytes := read(m, limitSpan)
+
+	var limit []byte
+	err := scale.Unmarshal(limitBytes, &limit)
+	if err != nil {
+		logger.Warnf("failed scale decoding limit: %s", err)
+		panic(err)
+	}
+
+	if len(limit) == 0 {
+		// limit is None, set limit to max
+		limit = []byte{0xff, 0xff, 0xff, 0xff}
+	}
+
+	limitUint := binary.LittleEndian.Uint32(limit)
+
+	deleted, allDeleted, err := storage.ClearPrefixInChildWithLimit(
+		keyToChild, prefix, limitUint)
+	if err != nil {
+		logger.Errorf("failed to clear prefix in child with limit: %s", err)
+	}
+
+	killStorageResult := NewKillStorageResult(deleted, allDeleted)
+
+	encodedKillStorageResult, err := scale.Marshal(killStorageResult)
+	if err != nil {
+		logger.Errorf("failed to encode result: %s", err)
+		return 0
+	}
+
+	resultSpan, err := write(m, rtCtx.Allocator, scale.MustMarshal(&encodedKillStorageResult))
+	if err != nil {
+		panic(err)
+	}
+
+	return resultSpan
+}
+
 func ext_default_child_storage_exists_version_1(ctx context.Context, m api.Module, childStorageKey, key uint64) uint32 {
 	rtCtx := ctx.Value(runtimeContextKey).(*runtime.Context)
 	if rtCtx == nil {
@@ -1203,6 +1270,13 @@ func ext_default_child_storage_root_version_1(
 	return ret
 }
 
+//export ext_default_child_storage_root_version_2
+func ext_default_child_storage_root_version_2(ctx context.Context, m api.Module, childStorageKey uint64,
+	stateVersion uint32) (ptrSize uint64) {
+	// TODO: Implement this after we have storage trie version 1 implemented #2418
+	return ext_default_child_storage_root_version_1(ctx, m, childStorageKey)
+}
+
 func ext_default_child_storage_storage_kill_version_1(ctx context.Context, m api.Module, childStorageKeySpan uint64) {
 	rtCtx := ctx.Value(runtimeContextKey).(*runtime.Context)
 	if rtCtx == nil {
@@ -1519,6 +1593,23 @@ func ext_offchain_index_set_version_1(ctx context.Context, m api.Module, keySpan
 	}
 }
 
+//export ext_offchain_index_clear_version_1
+func ext_offchain_index_clear_version_1(ctx context.Context, m api.Module, keySpan uint64) {
+	// Remove a key and its associated value from the Offchain DB.
+	// https://github.com/paritytech/substrate/blob/4d608f9c42e8d70d835a748fa929e59a99497e90/primitives/io/src/lib.rs#L1213
+
+	rtCtx := ctx.Value(runtimeContextKey).(*runtime.Context)
+	if rtCtx == nil {
+		panic("nil runtime context")
+	}
+
+	storageKey := read(m, keySpan)
+	err := rtCtx.NodeStorage.BaseDB.Del(storageKey)
+	if err != nil {
+		logger.Errorf("failed to set value in raw storage: %s", err)
+	}
+}
+
 func ext_offchain_local_storage_clear_version_1(ctx context.Context, m api.Module, kind uint32, key uint64) {
 	rtCtx := ctx.Value(runtimeContextKey).(*runtime.Context)
 	if rtCtx == nil {