Add TokenVM example

.github/workflows/tokenvm-static-analysis.yml

@@ -0,0 +1,27 @@
+# Copyright (C) 2023, Ava Labs, Inc. All rights reserved.
+# See the file LICENSE for licensing terms.
+
+name: TokenVM Static Analysis
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  Lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+          check-latest: true
+      - name: Run static analysis tests
+        working-directory: ./examples/tokenvm
+        shell: bash
+        run: scripts/tests.lint.sh

.github/workflows/tokenvm-sync-tests.yml

@@ -0,0 +1,32 @@
+# Copyright (C) 2023, Ava Labs, Inc. All rights reserved.
+# See the file LICENSE for licensing terms.
+
+name: TokenVM Sync Tests
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - "*"
+  pull_request:
+
+jobs:
+  tests:
+    runs-on:
+      labels: ubuntu-latest-16-cores
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.19
+      - name: Run sync tests
+        working-directory: ./examples/tokenvm
+        shell: bash
+        run: scripts/run.sh
+        env:
+          MODE: "full-test"

.github/workflows/tokenvm-unit-tests.yml

@@ -0,0 +1,39 @@
+# Copyright (C) 2023, Ava Labs, Inc. All rights reserved.
+# See the file LICENSE for licensing terms.
+
+name: TokenVM Unit Tests
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - "*"
+  pull_request:
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.19
+      - name: Run unit tests
+        working-directory: ./examples/tokenvm
+        shell: bash
+        run: scripts/tests.unit.sh
+      - name: Run integration tests
+        working-directory: ./examples/tokenvm
+        shell: bash
+        run: scripts/tests.integration.sh
+      - name: Run e2e tests
+        working-directory: ./examples/tokenvm
+        shell: bash
+        run: scripts/run.sh
+        env:
+          MODE: "test"

.gitignore

@@ -54,3 +54,4 @@ genesis.json
 *.test
 
 dist/
+*.pk

.golangci.yml

@@ -8,7 +8,6 @@ run:
   skip-files:
     - ".*\\.pb\\.go$"
     - ".*mock.*"
-    - "merkledb/*"
 
 issues:
   # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.

README.md

@@ -105,10 +105,14 @@ happen.
 `hypersdk` transactions must specify the keys they will touch in state (read
 or write) during execution and authentication so that all relevant data can be
 pre-fetched before block execution starts, which ensures all data accessed during
-verification of a block is done so in memory). This restriction also enables
-transactions to be processed in parallel as distinct, ordered transaction sets
-can be trivially formed by looking at the overlap of keys that transactions
-will touch.
+verification of a block is done so in memory). Notably, the keys specified here
+are not keys in a merkle trie (which may be quite volatile) but are instead the
+actual keys used to access data by the storage engine (like your address, which
+is much less volatile and not as cumbersome of a UX barrier).
+
+This restriction also enables transactions to be processed in parallel as distinct,
+ordered transaction sets can be trivially formed by looking at the overlap of keys
+that transactions will touch.
 
 _Parallel transaction execution was originally included in `hypersdk` but
 removed because the overhead of the naïve mechanism used to group transactions
@@ -212,11 +216,30 @@ and are stored alongside all other runtime logs. The unification of all of
 these functions with avalanchego means existing avalanchego monitoring tools
 work out of the box on your `hypervm`.
 
-## Example: `indexvm`
-We created the [`indexvm`](https://github.com/ava-labs/indexvm) while
-building the `hypersdk` to test out our design decisions and abstractions.
-We recommend taking a look at this `hypervm` to gain an in-depth understanding
-of how you can build a complex runtime on top of the `hypersdk`.
+## Examples
+### Beginner: `tokenvm`
+We created the [`tokenvm`](./examples/tokenvm) to showcase how to use the
+`hypersdk` in an application most readers are already familiar with, token minting
+and token trading. The `tokenvm` lets anyone create any asset, mint more of
+their asset, modify the metadata of their asset (if they reveal some info), and
+burn their asset. Additionally, there is an embedded on-chain exchange that
+allows anyone to create orders and fill (partial) orders of anyone else. To
+make this example easy to play with, the `tokenvm` also bundles a powerful CLI
+tool and serves RPC requests for trades out of an in-memory order book it
+maintains by syncing blocks. If you are interested in the intersection of
+exchanges and blockchains, it is definitely worth a read (the logic for filling
+orders is < 100 lines of code!).
+
+To ensure the `hypersdk` stays reliable as we optimize and evolve the codebase,
+we also run E2E tests in the `tokenvm` on each PR to the `hypersdk` core modules.
+
+### Expert: `indexvm`
+The [`indexvm`](https://github.com/ava-labs/indexvm) is much more complex than
+the `tokenvm` (more elaborate mechanisms and a new use case you may not be
+familiar with). It was built during the design of the `hypersdk` to test out the
+limits of the abstractions for building complex on-chain mechanisms. We recommend
+taking a look at this `hypervm` once you already have familiarity with the `hypersdk` to gain an
+even deeper understanding of how you can build a complex runtime on top of the `hypersdk`.
 
 The `indexvm` is dedicated to increasing the usefulness of the world's
 content-addressable data (like IPFS) by enabling anyone to "index it" by
@@ -238,7 +261,7 @@ a small part in this movement by making it easier for anyone to generate
 world-class recommendations for anyone on the internet, even if you've never
 interacted with them before.
 
-We'll use this example to explain how to use the `hypersdk` below.
+We'll use both of these `hypervms` to explain how to use the `hypersdk` below.
 
 ## How It Works
 To use the `hypersdk`, you must import it into your own `hypervm` and implement the
@@ -283,8 +306,8 @@ structures utilized by the `hypersdk` and handles both `Accepted` and
 `Gossiper`, `Handlers`, and `Database` packages so this is typically a lot of
 boilerplate code.
 
-You can view what this looks like in the `indexvm` by clicking this
-[link](https://github.com/ava-labs/indexvm/blob/main/controller/controller.go).
+You can view what this looks like in the `tokenvm` by clicking this
+[link](./examples/tokenvm/controller/controller.go).
 
 ### Genesis
 ```golang
@@ -300,8 +323,8 @@ start of the network (fee price, enabled txs, etc.). The serialized genesis of
 any `hyperchain` is persisted on the P-Chain for anyone to see when the network
 is created.
 
-You can view what this looks like in the `indexvm` by clicking this
-[link](https://github.com/ava-labs/indexvm/blob/main/genesis/genesis.go).
+You can view what this looks like in the `tokenvm` by clicking this
+[link](./examples/tokenvm/genesis/genesis.go).
 
 ### Action
 ```golang
@@ -321,8 +344,8 @@ the blockchain runtime. Specifically, they are "user-defined" element of
 any `hypersdk` transaction that is processed by all participants of any
 `hyperchain`.
 
-You can view what a simple transfer `Action` looks like [here](https://github.com/ava-labs/indexvm/blob/main/actions/transfer.go)
-and what a more complex "index" `Action` looks like [here](https://github.com/ava-labs/indexvm/blob/main/actions/index.go).
+You can view what a simple transfer `Action` looks like [here](./examples/tokenvm/actions/transfer.go)
+and what a more complex "fill order" `Action` looks like [here](./examples/tokenvm/actions/fill_order.go).
 
 ### Auth
 ```golang
@@ -407,8 +430,6 @@ out on the Avalanche Discord._
 * Use pre-specified state keys to process transactions in parallel (txs with no
   overlap can be processed at the same time, create conflict sets on-the-fly
   instead of before execution)
-* Perform E2E tests and publish nightly performance benchmarks using a simple
-  hypervm that just performs simple transfers (`dummyvm`)
 * Add support for Avalanche Warp Messaging (AWM) so any deployed hypervms
   (hyperchains) can communicate with each other ([see ava-labs/xsvm](https://github.com/ava-labs/xsvm))
 * Add a WASM runtime module to allow developers to embed smart contract
@@ -431,3 +452,9 @@ out on the Avalanche Discord._
 * Implement support for S3 and PostgreSQL storage backends
 * Provide optional auto-serialization/deserialization of `Actions` and `Auth`
   if only certain types are used in their definition
+* Add a module that could be used to track the location of various pieces
+  of data across a network ([see consistent
+  hasher](https://github.com/ava-labs/avalanchego/tree/master/utils/hashing/consistent))
+  of `hypervm` participants (even better if this is made abstract to any implementer
+  such that they can just register and request data from it and it is automatically
+  handled by the network layer)

chain/builder.go

@@ -145,7 +145,7 @@ func BuildBlock(ctx context.Context, vm VM, preferred ids.ID) (snowman.Block, er
 
 			// Update block with new transaction
 			b.Txs = append(b.Txs, next)
-			b.UnitsConsumed += nextUnits
+			b.UnitsConsumed += result.Units
 			surplusFee += (next.Base.UnitPrice - b.UnitPrice) * result.Units
 			results = append(results, result)
 			return len(b.Txs) < r.GetMaxBlockTxs(), false, false, nil

chain/dependencies.go

@@ -99,8 +99,11 @@ type Action interface {
 
 	// Auth may contain an [Actor] that performs a transaction
 	//
-	// if attempt to reference missing key, error...it is ok to not use all keys (conditional logic based on state)
-	StateKeys(Auth) [][]byte
+	// We provide the [txID] here because different actions like to use this as
+	// a unique identifier for things created in an action.
+	//
+	// If attempt to reference missing key, error...it is ok to not use all keys (conditional logic based on state)
+	StateKeys(auth Auth, txID ids.ID) [][]byte
 
 	// Key distinction with "Auth" is the payment of fees. All non-fee payments
 	// occur in Execute but Auth handles fees.

chain/transaction.go

@@ -96,8 +96,10 @@ func (t *Transaction) Expiry() int64 { return t.Base.Timestamp }
 func (t *Transaction) UnitPrice() uint64 { return t.Base.UnitPrice }
 
 // It is ok to have duplicate ReadKeys...the processor will skip them
+//
+// TODO: verify the invariant that [t.id] is set by this point
 func (t *Transaction) StateKeys() [][]byte {
-	return append(t.Action.StateKeys(t.Auth), t.Auth.StateKeys()...)
+	return append(t.Action.StateKeys(t.Auth, t.ID()), t.Auth.StateKeys()...)
 }
 
 // Units is charged whether or not a transaction is successful because state

client/helpers.go

@@ -12,7 +12,7 @@ import (
 )
 
 const (
-	waitSleep = 3 * time.Second
+	waitSleep = 500 * time.Millisecond
 )
 
 type Modifier interface {
@@ -65,6 +65,7 @@ func (cli *Client) GenerateTransaction(
 }
 
 func Wait(ctx context.Context, check func(ctx context.Context) (bool, error)) error {
+	time.Sleep(500 * time.Millisecond) // usually a small delay here helps a lot
 	for ctx.Err() == nil {
 		exit, err := check(ctx)
 		if err != nil {

codec/optional_packer.go

@@ -86,7 +86,7 @@ func (o *OptionalPacker) PackPublicKey(pk crypto.PublicKey) {
 
 func (o *OptionalPacker) UnpackPublicKey(dest *crypto.PublicKey) {
 	if o.checkBit() {
-		o.ip.UnpackPublicKey(dest)
+		o.ip.UnpackPublicKey(true, dest)
 	}
 }
 

codec/packer.go

@@ -104,9 +104,9 @@ func (p *Packer) PackPublicKey(src crypto.PublicKey) {
 	p.p.PackFixedBytes(src[:])
 }
 
-func (p *Packer) UnpackPublicKey(dest *crypto.PublicKey) {
+func (p *Packer) UnpackPublicKey(required bool, dest *crypto.PublicKey) {
 	copy((*dest)[:], p.p.UnpackFixedBytes(crypto.PublicKeyLen))
-	if *dest == crypto.EmptyPublicKey {
+	if required && *dest == crypto.EmptyPublicKey {
 		p.p.Errs.Add(fmt.Errorf("%w: PublicKey field is not populated", ErrFieldNotPopulated))
 	}
 }

consts/consts.go

@@ -10,6 +10,7 @@ const (
 	MaxUint        = ^uint(0)
 	MaxInt         = int(MaxUint >> 1)
 	IntLen         = 4
+	Uint16Len      = 2
 	Uint64Len      = 8
 	MaxUint64      = ^uint64(0)
 )