WIP rpc: implement iterator sessions

docs/node-configuration.md

@@ -136,6 +136,8 @@ RPC:
   MaxFindResultItems: 100
   MaxNEP11Tokens: 100
   Port: 10332
+  SessionEnabled: true
+  SessionExpirationTime: 60
   StartWhenSynchronized: false
   TLSConfig:
     Address: ""
@@ -159,6 +161,15 @@ where:
 - `MaxNEP11Tokens` - limit for the number of tokens returned from
   `getnep11balances` call.
 - `Port` is an RPC server port it should be bound to.
+- `SessionEnabled` denotes whether session-based iterator JSON-RPC API is enabled.
+  If true, then all iterators got from `invoke*` calls will be stored as sessions
+  on the server side available for further traverse. `traverseiterator` and
+  `terminatesession` JSON-RPC calls will be handled by the server. It is not
+  recommended to enable this setting on public networks due to possible DoS
+  attack. Set to `false` by default.
+- `SessionExpirationTime` is a lifetime of iterator session in seconds. It is set
+  to `60` seconds by default and is relevant only if `SessionEnabled` is set to
+  `true`.
 - `StartWhenSynchronized` controls when RPC server will be started, by default
   (`false` setting) it's started immediately and RPC is availabe during node
   synchronization. Setting it to `true` will make the node start RPC service only

docs/rpc.md

@@ -71,6 +71,8 @@ which would yield the response:
 | `sendrawtransaction` |
 | `submitblock` |
 | `submitoracleresponse` |
+| `terminatesession` |
+| `traverseiterator` |
 | `validateaddress` |
 | `verifyproof` |
 

go.mod

@@ -5,6 +5,7 @@ require (
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e
 	github.com/coreos/go-semver v0.3.0
 	github.com/davecgh/go-spew v1.1.1
+	github.com/google/uuid v1.2.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/holiman/uint256 v1.2.0

pkg/config/config.go

@@ -16,6 +16,9 @@ const (
 	UserAgentPrefix = "NEO-GO:"
 	// UserAgentFormat is a formatted string used to generate user agent string.
 	UserAgentFormat = UserAgentWrapper + UserAgentPrefix + "%s" + UserAgentWrapper
+	// DefaultMaxIteratorResultItems is the default upper bound of traversed
+	// iterator items per JSON-RPC response.
+	DefaultMaxIteratorResultItems = 100
 )
 
 // Version is the version of the node, set at the build time.
@@ -56,9 +59,10 @@ func LoadFile(configPath string) (Config, error) {
 			PingInterval: 30,
 			PingTimeout:  90,
 			RPC: rpc.Config{
-				MaxIteratorResultItems: 100,
+				MaxIteratorResultItems: DefaultMaxIteratorResultItems,
 				MaxFindResultItems:     100,
 				MaxNEP11Tokens:         100,
+				SessionExpirationTime:  60,
 			},
 		},
 	}

pkg/core/interop/iterator/interop.go

@@ -36,14 +36,15 @@ func IsIterator(item stackitem.Item) bool {
 	return ok
 }
 
-// Values returns an array of up to `max` iterator values. The second
-// return parameter denotes whether iterator is truncated.
-func Values(item stackitem.Item, max int) ([]stackitem.Item, bool) {
+// Values returns an array of up to `max` iterator values. The provided
+// iterator can safely be reused to retrieve the rest of its values in the
+// subsequent calls to Values.
+func Values(item stackitem.Item, max int) []stackitem.Item {
 	var result []stackitem.Item
 	arr := item.Value().(iterator)
-	for arr.Next() && max > 0 {
+	for max > 0 && arr.Next() {
 		result = append(result, arr.Value())
 		max--
 	}
-	return result, arr.Next()
+	return result
 }

pkg/rpc/client/doc.go

@@ -52,6 +52,8 @@ Supported methods
 	sendrawtransaction
 	submitblock
 	submitoracleresponse
+	terminatesession
+	traverseiterator
 	validateaddress
 
 Extensions:

pkg/rpc/client/helper.go

@@ -2,13 +2,22 @@ package client
 
 import (
 	"crypto/elliptic"
+	"encoding/json"
 	"errors"
 	"fmt"
 
+	"github.com/nspcc-dev/neo-go/pkg/core/interop/interopnames"
+	"github.com/nspcc-dev/neo-go/pkg/core/transaction"
 	"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
+	"github.com/nspcc-dev/neo-go/pkg/io"
 	"github.com/nspcc-dev/neo-go/pkg/rpc/client/nns"
+	"github.com/nspcc-dev/neo-go/pkg/rpc/request"
 	"github.com/nspcc-dev/neo-go/pkg/rpc/response/result"
+	"github.com/nspcc-dev/neo-go/pkg/smartcontract"
+	"github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag"
 	"github.com/nspcc-dev/neo-go/pkg/util"
+	"github.com/nspcc-dev/neo-go/pkg/vm/emit"
+	"github.com/nspcc-dev/neo-go/pkg/vm/opcode"
 	"github.com/nspcc-dev/neo-go/pkg/vm/stackitem"
 )
 
@@ -97,33 +106,107 @@ func topMapFromStack(st []stackitem.Item) (*stackitem.Map, error) {
 	return st[index].(*stackitem.Map), nil
 }
 
-// topIterableFromStack returns top list of elements of `resultItemType` type from the stack.
-func topIterableFromStack(st []stackitem.Item, resultItemType interface{}) ([]interface{}, error) {
-	index := len(st) - 1 // top stack element is last in the array
-	if t := st[index].Type(); t != stackitem.InteropT {
-		return nil, fmt.Errorf("invalid return stackitem type: %s (InteropInterface expected)", t.String())
+func (c *Client) invokeAndPackIteratorResults(contract util.Uint160, operation string, params []smartcontract.Parameter, signers []transaction.Signer) (*result.Invoke, error) {
+	// We can't use TraverseIterator for it, because this functionality is disabled
+	// on the mainnet/testnet, see https://github.com/neo-project/neo-modules/pull/715#discussion_r897641254
+	// Thus, we'll create a short program that traverses iterator got from System.Contract.Call,
+	// packs all its values into array and stores it on stack.
+	script := io.NewBufBinWriter()
+	emit.Instruction(script.BinWriter, opcode.INITSLOT, // Initialize local slot...
+		[]byte{
+			2, // with 2 local variables (0-th for iterator, 1-th for the resulting array)...
+			0, // and 0 arguments.
+		})
+	// Pack arguments for System.Contract.Call.
+	if len(params) == 0 {
+		emit.Opcodes(script.BinWriter, opcode.NEWARRAY0)
+	} else {
+		bytes, err := json.Marshal(params)
+		if err != nil {
+			return nil, fmt.Errorf("failed to convert operation parameters into System.Contract.Call arguments: marshalling error %w", err)
+		}
+		var slice request.Params
+		err = json.Unmarshal(bytes, &slice)
+		if err != nil {
+			return nil, fmt.Errorf("failed to convert operation parameters into System.Contract.Call arguments: unmarshalling error %w", err)
+		}
+		err = request.ExpandArrayIntoScript(script.BinWriter, slice)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create function invocation script: %w", err)
+		}
+		emit.Int(script.BinWriter, int64(len(slice)))
+		emit.Opcodes(script.BinWriter, opcode.PACK)
+	}
+	emit.AppCallNoArgs(script.BinWriter, contract, operation, callflag.All) // The System.Contract.Call itself, it will push Iterator on estack.
+	emit.Opcodes(script.BinWriter, opcode.STLOC0,                           // Pop the result of System.Contract.Call (the iterator) from estack and store it inside the 0-th cell of the local slot.
+		opcode.NEWARRAY0, // Push new empty array to estack. This array will store iterator's elements.
+		opcode.STLOC1)    // Pop the empty array from estack and store it inside the 1-th cell of the local slot.
+
+	// Start the iterator traversal cycle.
+	iteratorTraverseCycleStartOffset := script.Len()
+	emit.Opcodes(script.BinWriter, opcode.LDLOC0)                   // Load iterator from the 0-th cell of the local slot and push it on estack.
+	emit.Syscall(script.BinWriter, interopnames.SystemIteratorNext) // Call System.Iterator.Next, it will pop the iterator from estack and push `true` or `false` to estack.
+	jmpIfNotOffset := script.Len()
+	emit.Instruction(script.BinWriter, opcode.JMPIFNOT, // Pop boolean value (from the previous step) from estack, if `false`, then iterator has no more items => jump to the end of program.
+		[]byte{
+			0x00, // jump to loadResultOffset, but we'll fill this byte after script creation.
+		})
+	emit.Opcodes(script.BinWriter, opcode.LDLOC1, // Load the resulting array from 1-th cell of local slot and push it to estack.
+		opcode.DUP, // Create a copy of the resulting array and push it to estack.
+		// TODO: this DUP actually is not needed if we remove STLOC1 a few lines below; array is reference type, it will be updated anyway.
+		opcode.LDLOC0) // Load iterator from the 0-th cell of local slot and push it to estack.
+	emit.Syscall(script.BinWriter, interopnames.SystemIteratorValue) // Call System.Iterator.Value, it will pop the iterator from estack and push its current value to estack.
+	emit.Opcodes(script.BinWriter, opcode.APPEND,                    // Pop iterator value and the copy of the resulting array from estack. Append value to the resulting array. We still have the resulting array on the estack.
+		opcode.STLOC1) // Pop the resulting array from estack and store it inside the 1-th cell of the local slot (update the array value).
+	jmpOffset := script.Len()
+	emit.Instruction(script.BinWriter, opcode.JMP, // Jump to the start of iterator traverse cycle start.
+		[]byte{
+			uint8(iteratorTraverseCycleStartOffset - jmpOffset), // jump to iteratorTraverseCycleStartOffset; offset is relative to JMP position.
+		})
+
+	// End of the program: push the result on stack and return.
+	loadResultOffset := script.Len()
+	emit.Opcodes(script.BinWriter, opcode.LDLOC1, // Load the resulting array from 1-th cell of local slot and push it to estack.
+		opcode.RET) // Return.
+	if err := script.Err; err != nil {
+		return nil, fmt.Errorf("failed to create iterator unwrapper script: %w", err)
+	}
+
+	// Fill in JMPIFNOT instruction parameter.
+	bytes := script.Bytes()
+	bytes[jmpIfNotOffset+1] = uint8(loadResultOffset - jmpIfNotOffset) // +1 is for JMPIFNOT itself; offset is relative to JMPIFNOT position.
+
+	return c.InvokeScript(bytes, signers)
+}
+
+// unwrapTopStackItem returns the list of elements of `resultItemType` type from the top element
+// of the provided stack. The top element is expected to be an Array, otherwise an error is returned.
+func unwrapTopStackItem(st []stackitem.Item, resultItemType interface{}) ([]interface{}, error) {
+	index := len(st) - 1 // top stack element is the last in the array
+	if t := st[index].Type(); t != stackitem.ArrayT {
+		return nil, fmt.Errorf("invalid return stackitem type: %s (Array expected)", t.String())
 	}
-	iter, ok := st[index].Value().(result.Iterator)
+	items, ok := st[index].Value().([]stackitem.Item)
 	if !ok {
 		return nil, fmt.Errorf("failed to deserialize iterable from interop stackitem: invalid value type (Array expected)")
 	}
-	result := make([]interface{}, len(iter.Values))
-	for i := range iter.Values {
+	result := make([]interface{}, len(items))
+	for i := range items {
 		switch resultItemType.(type) {
 		case []byte:
-			bytes, err := iter.Values[i].TryBytes()
+			bytes, err := items[i].TryBytes()
 			if err != nil {
 				return nil, fmt.Errorf("failed to deserialize []byte from stackitem #%d: %w", i, err)
 			}
 			result[i] = bytes
 		case string:
-			bytes, err := iter.Values[i].TryBytes()
+			bytes, err := items[i].TryBytes()
 			if err != nil {
 				return nil, fmt.Errorf("failed to deserialize string from stackitem #%d: %w", i, err)
 			}
 			result[i] = string(bytes)
 		case util.Uint160:
-			bytes, err := iter.Values[i].TryBytes()
+			bytes, err := items[i].TryBytes()
 			if err != nil {
 				return nil, fmt.Errorf("failed to deserialize uint160 from stackitem #%d: %w", i, err)
 			}
@@ -132,7 +215,7 @@ func topIterableFromStack(st []stackitem.Item, resultItemType interface{}) ([]in
 				return nil, fmt.Errorf("failed to decode uint160 from stackitem #%d: %w", i, err)
 			}
 		case nns.RecordState:
-			rs, ok := iter.Values[i].Value().([]stackitem.Item)
+			rs, ok := items[i].Value().([]stackitem.Item)
 			if !ok {
 				return nil, fmt.Errorf("failed to decode RecordState from stackitem #%d: not a struct", i)
 			}

pkg/rpc/client/native.go

@@ -118,7 +118,7 @@ func (c *Client) NNSIsAvailable(nnsHash util.Uint160, name string) (bool, error)
 
 // NNSGetAllRecords returns all records for a given name from NNS service.
 func (c *Client) NNSGetAllRecords(nnsHash util.Uint160, name string) ([]nns.RecordState, error) {
-	result, err := c.InvokeFunction(nnsHash, "getAllRecords", []smartcontract.Parameter{
+	result, err := c.invokeAndPackIteratorResults(nnsHash, "getAllRecords", []smartcontract.Parameter{
 		{
 			Type:  smartcontract.StringType,
 			Value: name,
@@ -132,7 +132,7 @@ func (c *Client) NNSGetAllRecords(nnsHash util.Uint160, name string) ([]nns.Reco
 		return nil, err
 	}
 
-	arr, err := topIterableFromStack(result.Stack, nns.RecordState{})
+	arr, err := unwrapTopStackItem(result.Stack, nns.RecordState{})
 	if err != nil {
 		return nil, fmt.Errorf("failed to get token IDs from stack: %w", err)
 	}

pkg/rpc/client/nep11.go

@@ -84,7 +84,7 @@ func (c *Client) CreateNEP11TransferTx(acc *wallet.Account, tokenHash util.Uint1
 
 // NEP11TokensOf returns an array of token IDs for the specified owner of the specified NFT token.
 func (c *Client) NEP11TokensOf(tokenHash util.Uint160, owner util.Uint160) ([][]byte, error) {
-	result, err := c.InvokeFunction(tokenHash, "tokensOf", []smartcontract.Parameter{
+	result, err := c.invokeAndPackIteratorResults(tokenHash, "tokensOf", []smartcontract.Parameter{
 		{
 			Type:  smartcontract.Hash160Type,
 			Value: owner,
@@ -98,7 +98,7 @@ func (c *Client) NEP11TokensOf(tokenHash util.Uint160, owner util.Uint160) ([][]
 		return nil, err
 	}
 
-	arr, err := topIterableFromStack(result.Stack, []byte{})
+	arr, err := unwrapTopStackItem(result.Stack, []byte{})
 	if err != nil {
 		return nil, fmt.Errorf("failed to get token IDs from stack: %w", err)
 	}
@@ -161,7 +161,7 @@ func (c *Client) NEP11DBalanceOf(tokenHash, owner util.Uint160, tokenID []byte)
 
 // NEP11DOwnerOf returns list of the specified NEP-11 divisible token owners.
 func (c *Client) NEP11DOwnerOf(tokenHash util.Uint160, tokenID []byte) ([]util.Uint160, error) {
-	result, err := c.InvokeFunction(tokenHash, "ownerOf", []smartcontract.Parameter{
+	result, err := c.invokeAndPackIteratorResults(tokenHash, "ownerOf", []smartcontract.Parameter{
 		{
 			Type:  smartcontract.ByteArrayType,
 			Value: tokenID,
@@ -175,7 +175,7 @@ func (c *Client) NEP11DOwnerOf(tokenHash util.Uint160, tokenID []byte) ([]util.U
 		return nil, err
 	}
 
-	arr, err := topIterableFromStack(result.Stack, util.Uint160{})
+	arr, err := unwrapTopStackItem(result.Stack, util.Uint160{})
 	if err != nil {
 		return nil, fmt.Errorf("failed to get token IDs from stack: %w", err)
 	}
@@ -210,7 +210,7 @@ func (c *Client) NEP11Properties(tokenHash util.Uint160, tokenID []byte) (*stack
 
 // NEP11Tokens returns list of the tokens minted by the contract.
 func (c *Client) NEP11Tokens(tokenHash util.Uint160) ([][]byte, error) {
-	result, err := c.InvokeFunction(tokenHash, "tokens", []smartcontract.Parameter{}, nil)
+	result, err := c.invokeAndPackIteratorResults(tokenHash, "tokens", []smartcontract.Parameter{}, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -219,7 +219,7 @@ func (c *Client) NEP11Tokens(tokenHash util.Uint160) ([][]byte, error) {
 		return nil, err
 	}
 
-	arr, err := topIterableFromStack(result.Stack, []byte{})
+	arr, err := unwrapTopStackItem(result.Stack, []byte{})
 	if err != nil {
 		return nil, fmt.Errorf("failed to get token IDs from stack: %w", err)
 	}

pkg/rpc/client/rpc.go

@@ -3,9 +3,12 @@ package client
 import (
 	"encoding/base64"
 	"encoding/hex"
+	"encoding/json"
 	"errors"
 	"fmt"
 
+	"github.com/google/uuid"
+	"github.com/nspcc-dev/neo-go/pkg/config"
 	"github.com/nspcc-dev/neo-go/pkg/config/netmode"
 	"github.com/nspcc-dev/neo-go/pkg/core/block"
 	"github.com/nspcc-dev/neo-go/pkg/core/fee"
@@ -24,6 +27,7 @@ import (
 	"github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger"
 	"github.com/nspcc-dev/neo-go/pkg/util"
 	"github.com/nspcc-dev/neo-go/pkg/vm/opcode"
+	"github.com/nspcc-dev/neo-go/pkg/vm/stackitem"
 	"github.com/nspcc-dev/neo-go/pkg/wallet"
 )
 
@@ -1129,3 +1133,51 @@ func (c *Client) GetNativeContractHash(name string) (util.Uint160, error) {
 	c.cacheLock.Unlock()
 	return cs.Hash, nil
 }
+
+// TraverseIterator returns a set of iterator values (maxItemsCount at max) for
+// the specified iterator and session. If result contains no elements, then either
+// Iterator has no elements or session was expired and terminated by the server.
+// If maxItemsCount is non-positive number, then the full set of iterator values
+// will be returned using several `traverseiterator` calls if needed.
+func (c *Client) TraverseIterator(sessionID, iteratorID uuid.UUID, maxItemsCount int) ([]stackitem.Item, error) {
+	var traverseAll bool
+	if maxItemsCount <= 0 {
+		maxItemsCount = config.DefaultMaxIteratorResultItems
+		traverseAll = true
+	}
+	var (
+		result []stackitem.Item
+		params = request.NewRawParams(sessionID.String(), iteratorID.String(), maxItemsCount)
+	)
+	for {
+		var resp = new([]json.RawMessage)
+		if err := c.performRequest("traverseiterator", params, resp); err != nil {
+			return nil, err
+		}
+		for i, iBytes := range *resp {
+			itm, err := stackitem.FromJSONWithTypes(iBytes)
+			if err != nil {
+				return nil, fmt.Errorf("failed to unmarshal %d-th iterator value: %w", i, err)
+			}
+			result = append(result, itm)
+		}
+		if len(*resp) < maxItemsCount || !traverseAll {
+			break
+		}
+	}
+
+	return result, nil
+}
+
+// TerminateSession tries to terminate the specified session and returns whether
+// termination was performed. If `false` is returned, then the specified session
+// does not exist or has already been terminated.
+func (c *Client) TerminateSession(sessionID uuid.UUID) (bool, error) {
+	var resp bool
+	params := request.NewRawParams(sessionID.String())
+	if err := c.performRequest("terminatesession", params, &resp); err != nil {
+		return false, err
+	}
+
+	return resp, nil
+}