feat: gnovm benchmarking tool

benchmarking/Makefile

@@ -0,0 +1,18 @@
+.PHONY: opcode storage build_opcode build_storage
+
+# Build target
+build_opcode:
+ go build -tags "benchmarkingops" -o build/gnobench ./cmd
+
+# Build target
+build_storage:
+ go build -tags "benchmarkingstorage" -o build/gnobench ./cmd
+
+
+# Run target
+opcode: build_opcode
+ ./build/gnobench
+
+# Run target
+storage: build_storage
+ ./build/gnobench

benchmarking/README.md

@@ -0,0 +1,90 @@
+# `gnobench` the time consumed for GnoVM OpCode execution and store access
+
+`gnobench` benchmarks the time consumed for each VM CPU OpCode and persistent access to the store, including marshalling and unmarshalling of realm objects.
+
+## Usage
+
+### Simple mode
+
+The benchmark only involves the GnoVM and the persistent store. It benchmarks the bare minimum components, and the results are isolated from other components. We use standardize gno contract to perform the benchmarking.
+
+This mode is the best for benchmarking each major release and/or changes in GnoVM.
+
+ make opcode
+ make storage
+
+### Production mode
+
+It benchmarks the node in the production environment with minimum overhead.
+We can not only benchmark with standardize the contract but also capture the live usage in production environment.
+It gives us a complete picture of the node perform.
+
+
+ 1. Build the production node with benchmarking flags:
+
+ `go build -tags "benchmarkingstorage benchmarkingops" gno.land/cmd/gnoland`
+
+ 2. Run the node in the production environment. It will dump benchmark data to a benchmarks.bin file.
+
+ 3. call the realm contracts at `gno.land/r/x/benchmark/opcodes` and `gno.land/r/x/benchmark/storage`
+
+ 4. Stop the server after the benchmarking session is complete.
+
+ 5. Run the following command to convert the binary dump:
+
+ `gnobench -bin path_to_benchmarks.bin`
+
+ it converts the binary dump to results.csv and results_stats.csv.
+
+
+## Results
+
+The benchmarking results are stored in two files:
+ 1. The raw results are saved in results.csv.
+
+ | Operation | Elapsed Time | Disk IO Bytes |
+ |-----------------|--------------|---------------|
+ | OpEval | 40333 | 0 |
+ | OpPopBlock | 208 | 0 |
+ | OpHalt | 167 | 0 |
+ | OpEval | 500 | 0 |
+ | OpInterfaceType | 458 | 0 |
+ | OpPopBlock | 166 | 0 |
+ | OpHalt | 125 | 0 |
+ | OpInterfaceType | 21125 | 0 |
+ | OpEval | 541 | 0 |
+ | OpEval | 209 | 0 |
+ | OpInterfaceType | 334 | 0 |
+
+
+
+ 2. The averages and standard deviations are summarized in results_stats.csv.
+
+ | Operation | Avg Time | Avg Size | Time Std Dev | Count |
+|----------------|----------|----------|--------------|-------|
+| OpAdd | 101 | 0 | 45 | 300 |
+| OpAddAssign | 309 | 0 | 1620 | 100 |
+| OpArrayLit | 242 | 0 | 170 | 700 |
+| OpArrayType | 144 | 0 | 100 | 714 |
+| OpAssign | 136 | 0 | 95 | 2900 |
+| OpBand | 92 | 0 | 30 | 100 |
+| OpBandAssign | 127 | 0 | 62 | 100 |
+| OpBandn | 97 | 0 | 54 | 100 |
+| OpBandnAssign | 125 | 0 | 113 | 100 |
+| OpBinary1 | 128 | 0 | 767 | 502 |
+| OpBody | 127 | 0 | 145 | 13700 |
+
+## Design consideration
+
+### Minimum Overhead and Footprint
+
+- Constant build flags enable benchmarking.
+- Encode operations and measurements in binary.
+- Dump to a local file in binary.
+- No logging, printout, or network access involved.
+
+### Accurate
+
+- Pause the timer for storage access while performing VM opcode benchmarking.
+- Measure each OpCode execution in nanoseconds.
+- Store access includes the duration for Amino marshalling and unmarshalling.

benchmarking/cmd/main.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+ "flag"
+ "log"
+ "os"
+ "path/filepath"
+
+ bm "github.com/gnolang/gno/benchmarking"
+)
+
+var (
+ outFlag = flag.String("out", "results.csv", "the out put file")
+ benchFlag = flag.String("bench", "./gno", "the path to the benchmark contract")
+ binFlag = flag.String("bin", "", "interpret the existing benchmarking file.")
+)
+
+// We dump the benchmark in bytes for speed and minimal overhead.
+const tmpFile = "benchmark.bin"
+
+func main() {
+ flag.Parse()
+ if *binFlag != "" {
+ binFile, err := filepath.Abs(*binFlag)
+ if err != nil {
+ log.Fatal("unable to get absolute path for the file", err)
+ }
+ stats(binFile)
+ return
+ }
+ bm.Init(tmpFile)
+ bstore := benchmarkDiskStore()
+
+ dir, err := filepath.Abs(*benchFlag)
+ if err != nil {
+ log.Fatal("unable to get absolute path for storage directory.", err)
+ }
+
+ // load stdlibs
+ loadStdlibs(bstore)
+
+ if bm.OpsEnabled {
+ benchmarkOpCodes(bstore, dir)
+ }
+ if bm.StorageEnabled {
+ benchmarkStorage(bstore, dir)
+ }
+ bm.Finish()
+ stats(tmpFile)
+ err = os.Remove(tmpFile)
+ if err != nil {
+ log.Printf("Error removing tmp file: %v", err)
+ }
+
+}

benchmarking/cmd/opcode_test.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+ "testing"
+
+ gno "github.com/gnolang/gno/gnovm/pkg/gnolang"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestLoadOpcodesPackage(t *testing.T) {
+ dir := "../gno/opcodes"
+ bstore := benchmarkDiskStore()
+ pv := addPackage(bstore, dir, opcodesPkgPath)
+ pb := pv.GetBlock(bstore)
+
+ assert := assert.New(t)
+ require := require.New(t)
+
+ declTypes := []string{
+ "foo",
+ "dog",
+ "foofighter",
+ }
+ for i := 0; i < len(declTypes); i++ {
+ tv := pb.Values[i]
+ v, ok := tv.V.(gno.TypeValue)
+ require.True(ok, "it should be a TypeValue")
+ dtv, ok2 := v.Type.(*gno.DeclaredType)
+ tn := declTypes[i]
+
+ require.True(ok2, "it should be a DeclaredType")
+ assert.Equal(tn, string(dtv.Name), "the declared type name should be "+tn)
+ }
+
+ // These are the functions used to benchmark the OpCode in the benchmarking contract.
+ // We call each to benchmark a group of OpCodes.
+ funcValues := []string{
+ "OpDecl",
+ "OpEvalInt",
+ "OpEvalFloat",
+ "StmtOps",
+ "ControlOps",
+ "OpDefer",
+ "OpUnary",
+ "OpBinary",
+ "ExprOps",
+ "OpLor",
+ "OpLand",
+ "OpPanic",
+ "OpTypeSwitch",
+ "OpCallDeferNativeBody",
+ "OpRange",
+ "OpForLoop",
+ "OpTypes",
+ "OpOpValues",
+ }
+
+ for i := 3; i < 3+len(funcValues); i++ {
+ j := i - 3
+ tv := pb.Values[i]
+ fv, ok := tv.V.(*gno.FuncValue)
+ require.True(ok, "it should be a FuncValue")
+ fn := funcValues[j]
+ assert.Equal(fn, string(fv.Name), "the declared type name should be "+fn)
+ }
+}

benchmarking/cmd/run.go

@@ -0,0 +1,138 @@
+package main
+
+import (
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/gnolang/gno/gnovm/pkg/gnoenv"
+ gno "github.com/gnolang/gno/gnovm/pkg/gnolang"
+ "github.com/gnolang/gno/gnovm/stdlibs"
+ osm "github.com/gnolang/gno/tm2/pkg/os"
+)
+
+const (
+ opcodesPkgPath = "gno.land/r/x/benchmark/opcodes"
+ rounds = 100
+)
+
+func benchmarkOpCodes(bstore gno.Store, dir string) {
+ opcodesPkgDir := filepath.Join(dir, "opcodes")
+
+ pv := addPackage(bstore, opcodesPkgDir, opcodesPkgPath)
+ for i := 0; i < rounds; i++ {
+ callOpsBench(bstore, pv)
+ }
+}
+
+func callOpsBench(bstore gno.Store, pv *gno.PackageValue) {
+ // start
+ pb := pv.GetBlock(bstore)
+ for _, tv := range pb.Values {
+ if fv, ok := tv.V.(*gno.FuncValue); ok {
+ cx := gno.Call(fv.Name)
+ callFunc(bstore, pv, cx)
+ }
+ }
+}
+
+const storagePkgPath = "gno.land/r/x/benchmark/storage"
+
+func benchmarkStorage(bstore gno.Store, dir string) {
+ avlPkgDir := filepath.Join(dir, "avl")
+ addPackage(bstore, avlPkgDir, "gno.land/p/demo/avl")
+
+ storagePkgDir := filepath.Join(dir, "storage")
+ pv := addPackage(bstore, storagePkgDir, storagePkgPath)
+ benchStoreSet(bstore, pv)
+ benchStoreGet(bstore, pv)
+}
+
+func benchStoreSet(bstore gno.Store, pv *gno.PackageValue) {
+ title := "1KB content"
+ content := strings.Repeat("a", 1024)
+
+ // in forum.gno: func AddPost(title, content string)
+ // one AddPost will be added to three different boards in the forum.gno contract
+
+ for i := 0; i < rounds; i++ {
+ cx := gno.Call("AddPost", gno.Str(title), gno.Str(content))
+ callFunc(bstore, pv, cx)
+ }
+}
+
+func benchStoreGet(bstore gno.Store, pv *gno.PackageValue) {
+ // in forum.gno: func GetPost(boardId, postId int) string in forum.gno
+ // there are three different boards on the benchmarking forum contract
+ for i := 0; i < 3; i++ {
+ for j := 0; j < rounds; j++ {
+ cx := gno.Call("GetPost", gno.X(i), gno.X(j))
+ callFunc(bstore, pv, cx)
+ }
+ }
+}
+
+func callFunc(bstore gno.Store, pv *gno.PackageValue, cx gno.Expr) []gno.TypedValue {
+ m := gno.NewMachineWithOptions(
+ gno.MachineOptions{
+ PkgPath: pv.PkgPath,
+ Output: os.Stdout, // XXX
+ Store: bstore,
+ })
+
+ defer m.Release()
+
+ m.SetActivePackage(pv)
+ return m.Eval(cx)
+}
+
+// addPacakge
+
+func addPackage(bstore gno.Store, dir string, pkgPath string) *gno.PackageValue {
+ // load benchmark contract
+ m := gno.NewMachineWithOptions(
+ gno.MachineOptions{
+ PkgPath: "",
+ Output: os.Stdout,
+ Store: bstore,
+ })
+ defer m.Release()
+
+ memPkg := gno.ReadMemPackage(dir, pkgPath)
+
+ // pare the file, create pn, pv and save the values in m.store
+ _, pv := m.RunMemPackage(memPkg, true)
+
+ return pv
+}
+
+// load stdlibs
+func loadStdlibs(bstore gno.Store) {
+ // copied from vm/builtin.go
+ getPackage := func(pkgPath string, newStore gno.Store) (pn *gno.PackageNode, pv *gno.PackageValue) {
+ stdlibDir := filepath.Join(gnoenv.RootDir(), "gnovm", "stdlibs")
+ stdlibPath := filepath.Join(stdlibDir, pkgPath)
+ if !osm.DirExists(stdlibPath) {
+ // does not exist.
+ return nil, nil
+ }
+
+ memPkg := gno.ReadMemPackage(stdlibPath, pkgPath)
+ if memPkg.IsEmpty() {
+ // no gno files are present, skip this package
+ return nil, nil
+ }
+
+ m2 := gno.NewMachineWithOptions(gno.MachineOptions{
+ PkgPath: "gno.land/r/stdlibs/" + pkgPath,
+ // PkgPath: pkgPath,
+ Output: os.Stdout,
+ Store: newStore,
+ })
+ defer m2.Release()
+ return m2.RunMemPackage(memPkg, true)
+ }
+
+ bstore.SetPackageGetter(getPackage)
+ bstore.SetNativeStore(stdlibs.NativeStore)
+}