fix(gno.land): pre-load all standard libraries in vm.Initialize

gno.land/cmd/gnoland/start.go

@@ -234,17 +234,17 @@ func execStart(ctx context.Context, c *startCfg, io commands.IO) error {
 		return fmt.Errorf("unable to initialize telemetry, %w", err)
 	}
 
+	// Print the starting graphic
+	if c.logFormat != string(log.JSONFormat) {
+		io.Println(startGraphic)
+	}
+
 	// Create application and node
 	cfg.LocalApp, err = gnoland.NewApp(nodeDir, c.skipFailingGenesisTxs, logger, c.genesisMaxVMCycles)
 	if err != nil {
 		return fmt.Errorf("unable to create the Gnoland app, %w", err)
 	}
 
-	// Print the starting graphic
-	if c.logFormat != string(log.JSONFormat) {
-		io.Println(startGraphic)
-	}
-
 	// Create a default node, with the given setup
 	gnoNode, err := node.DefaultNewNode(cfg, genesisPath, logger)
 	if err != nil {

gno.land/pkg/gnoland/app.go

@@ -33,6 +33,9 @@
 	GenesisTxHandler GenesisTxHandler
 	Logger           *slog.Logger
 	MaxCycles        int64
+	// Whether to cache the result of loading the standard libraries.
+	// This is useful if you have to start many nodes, like in testing.
+	CacheStdlibLoad bool
 }
 
 func NewAppOptions() *AppOptions {
@@ -121,7 +124,7 @@
 
 	// Initialize the VMKeeper.
 	ms := baseApp.GetCacheMultiStore()
-	vmKpr.Initialize(ms)
+	vmKpr.Initialize(cfg.Logger, ms, cfg.CacheStdlibLoad)
 	ms.MultiWrite() // XXX why was't this needed?
 
 	return baseApp, nil
@@ -157,7 +160,12 @@
 }
 
 // InitChainer returns a function that can initialize the chain with genesis.
-func InitChainer(baseApp *sdk.BaseApp, acctKpr auth.AccountKeeperI, bankKpr bank.BankKeeperI, resHandler GenesisTxHandler) func(sdk.Context, abci.RequestInitChain) abci.ResponseInitChain {
+func InitChainer(
+	baseApp *sdk.BaseApp,
+	acctKpr auth.AccountKeeperI,
+	bankKpr bank.BankKeeperI,
+	resHandler GenesisTxHandler,
+) func(sdk.Context, abci.RequestInitChain) abci.ResponseInitChain {
 	return func(ctx sdk.Context, req abci.RequestInitChain) abci.ResponseInitChain {
 		if req.AppState != nil {
 			// Get genesis state

gno.land/pkg/gnoland/node_inmemory.go

@@ -94,6 +94,7 @@
 		GenesisTxHandler: cfg.GenesisTxHandler,
 		MaxCycles:        cfg.GenesisMaxVMCycles,
 		DB:               memdb.NewMemDB(),
+		CacheStdlibLoad:  true,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("error initializing new app: %w", err)

gno.land/pkg/sdk/vm/builtins.go

@@ -1,47 +1,11 @@
 package vm
 
 import (
-	"os"
-	"path/filepath"
-
-	gno "github.com/gnolang/gno/gnovm/pkg/gnolang"
 	"github.com/gnolang/gno/tm2/pkg/crypto"
-	osm "github.com/gnolang/gno/tm2/pkg/os"
 	"github.com/gnolang/gno/tm2/pkg/sdk"
 	"github.com/gnolang/gno/tm2/pkg/std"
 )
 
-// NOTE: this function may add loaded dependencies to store if they don't
-// already exist, including mem packages. If this happens during a transaction
-// with the tx context store, the transaction caller will pay for operations.
-// NOTE: native functions/methods added here must be quick operations, or
-// account for gas before operation.
-// TODO: define criteria for inclusion, and solve gas calculations(???).
-func (vm *VMKeeper) getPackage(pkgPath string, store gno.Store) (pn *gno.PackageNode, pv *gno.PackageValue) {
-	// otherwise, built-in package value.
-	// first, load from filepath.
-	stdlibPath := filepath.Join(vm.stdlibsDir, pkgPath)
-	if !osm.DirExists(stdlibPath) {
-		// does not exist.
-		return nil, nil
-	}
-	memPkg := gno.ReadMemPackage(stdlibPath, pkgPath)
-	if memPkg.IsEmpty() {
-		// no gno files are present, skip this package
-		return nil, nil
-	}
-
-	m2 := gno.NewMachineWithOptions(gno.MachineOptions{
-		PkgPath: "gno.land/r/stdlibs/" + pkgPath,
-		// PkgPath: pkgPath,
-		Output: os.Stdout,
-		Store:  store,
-	})
-	defer m2.Release()
-	pn, pv = m2.RunMemPackage(memPkg, true)
-	return
-}
-
 // ----------------------------------------
 // SDKBanker
 

gno.land/pkg/sdk/vm/common_test.go

@@ -25,6 +25,14 @@ type testEnv struct {
 }
 
 func setupTestEnv() testEnv {
+	return _setupTestEnv(true)
+}
+
+func setupTestEnvCold() testEnv {
+	return _setupTestEnv(false)
+}
+
+func _setupTestEnv(cacheStdlibs bool) testEnv {
 	db := memdb.NewMemDB()
 
 	baseCapKey := store.NewStoreKey("baseCapKey")
@@ -41,7 +49,9 @@ func setupTestEnv() testEnv {
 	stdlibsDir := filepath.Join("..", "..", "..", "..", "gnovm", "stdlibs")
 	vmk := NewVMKeeper(baseCapKey, iavlCapKey, acck, bank, stdlibsDir, 100_000_000)
 
-	vmk.Initialize(ms.MultiCacheWrap())
+	mcw := ms.MultiCacheWrap()
+	vmk.Initialize(log.NewNoopLogger(), mcw, cacheStdlibs)
+	mcw.MultiWrite()
 
 	return testEnv{ctx: ctx, vmk: vmk, bank: bank, acck: acck}
 }

gno.land/pkg/sdk/vm/gas_test.go

@@ -67,7 +67,7 @@ func TestAddPkgDeliverTx(t *testing.T) {
 	gasDeliver := gctx.GasMeter().GasConsumed()
 
 	assert.True(t, res.IsOK())
-	assert.Equal(t, int64(87929), gasDeliver)
+	assert.Equal(t, int64(87965), gasDeliver)
 }
 
 // Enough gas for a failed transaction.

gno.land/pkg/sdk/vm/keeper.go

@@ -6,17 +6,25 @@
 	"bytes"
 	"context"
 	"fmt"
+	"log/slog"
 	"os"
+	"path/filepath"
 	"strings"
+	"sync"
+	"time"
 
 	gno "github.com/gnolang/gno/gnovm/pkg/gnolang"
 	"github.com/gnolang/gno/gnovm/stdlibs"
+	"github.com/gnolang/gno/tm2/pkg/db/memdb"
 	"github.com/gnolang/gno/tm2/pkg/errors"
+	osm "github.com/gnolang/gno/tm2/pkg/os"
 	"github.com/gnolang/gno/tm2/pkg/sdk"
 	"github.com/gnolang/gno/tm2/pkg/sdk/auth"
 	"github.com/gnolang/gno/tm2/pkg/sdk/bank"
 	"github.com/gnolang/gno/tm2/pkg/std"
 	"github.com/gnolang/gno/tm2/pkg/store"
+	"github.com/gnolang/gno/tm2/pkg/store/dbadapter"
+	"github.com/gnolang/gno/tm2/pkg/store/types"
 	"github.com/gnolang/gno/tm2/pkg/telemetry"
 	"github.com/gnolang/gno/tm2/pkg/telemetry/metrics"
 	"go.opentelemetry.io/otel/attribute"
@@ -73,20 +81,38 @@
 	return vmk
 }
 
-func (vm *VMKeeper) Initialize(ms store.MultiStore) {
+func (vm *VMKeeper) Initialize(
+	logger *slog.Logger,
+	ms store.MultiStore,
+	cacheStdlibLoad bool,
+) {
 	if vm.gnoStore != nil {
 		panic("should not happen")
 	}
-	alloc := gno.NewAllocator(maxAllocTx)
 	baseSDKStore := ms.GetStore(vm.baseKey)
 	iavlSDKStore := ms.GetStore(vm.iavlKey)
+
+	if cacheStdlibLoad {
+		vm.gnoStore = CachedStdlibLoad(vm.stdlibsDir, baseSDKStore, iavlSDKStore)
+		return
+	}
+
+	alloc := gno.NewAllocator(maxAllocTx)
 	vm.gnoStore = gno.NewStore(alloc, baseSDKStore, iavlSDKStore)
-	vm.gnoStore.SetPackageGetter(vm.getPackage)
 	vm.gnoStore.SetNativeStore(stdlibs.NativeStore)
-	if vm.gnoStore.NumMemPackages() > 0 {
+	if !cacheStdlibLoad && vm.gnoStore.NumMemPackages() == 0 {
+		// No packages in the store; set up the stdlibs.
+		start := time.Now()
+
+		loadStdlib(vm.stdlibsDir, vm.gnoStore)
+
+		logger.Debug("Standard libraries initialized",
+			"elapsed", time.Since(start))
+	} else {
 		// for now, all mem packages must be re-run after reboot.
 		// TODO remove this, and generally solve for in-mem garbage collection
 		// and memory management across many objects/types/nodes/packages.
+		start := time.Now()
 		m2 := gno.NewMachineWithOptions(
 			gno.MachineOptions{
 				PkgPath: "",
@@ -97,7 +123,79 @@
 		gno.DisableDebug()
 		m2.PreprocessAllFilesAndSaveBlockNodes()
 		gno.EnableDebug()
+		logger.Debug("GnoVM packages preprocessed",
+			"elapsed", time.Since(start))
+	}
+}
+
+func CachedStdlibLoad(stdlibsDir string, baseStore, iavlStore store.Store) gno.Store {
+	cachedStdlibOnce.Do(func() {
+		cachedStdlibBase = memdb.NewMemDB()
+		cachedStdlibIavl = memdb.NewMemDB()
+
+		cachedGnoStore = gno.NewStore(nil,
+			dbadapter.StoreConstructor(cachedStdlibBase, types.StoreOptions{}),
+			dbadapter.StoreConstructor(cachedStdlibIavl, types.StoreOptions{}))
+		cachedGnoStore.SetNativeStore(stdlibs.NativeStore)
+		loadStdlib(stdlibsDir, cachedGnoStore)
+	})
+
+	itr := cachedStdlibBase.Iterator(nil, nil)
+	for ; itr.Valid(); itr.Next() {
+		baseStore.Set(itr.Key(), itr.Value())
 	}
+
+	itr = cachedStdlibIavl.Iterator(nil, nil)
+	for ; itr.Valid(); itr.Next() {
+		iavlStore.Set(itr.Key(), itr.Value())
+	}
+
+	alloc := gno.NewAllocator(maxAllocTx)
+	gs := gno.NewStore(alloc, baseStore, iavlStore)
+	gs.SetNativeStore(stdlibs.NativeStore)
+	gno.CopyCachesFromStore(gs, cachedGnoStore)
+	return gs
+}
+
+var (
+	cachedStdlibOnce sync.Once
+	cachedStdlibBase *memdb.MemDB
+	cachedStdlibIavl *memdb.MemDB
+	cachedGnoStore   gno.Store
+)
+
+func loadStdlib(stdlibsDir string, store gno.Store) {
+	stdlibInitList := stdlibs.InitOrder()
+	for _, lib := range stdlibInitList {
+		if lib == "testing" {
+			// XXX: testing is skipped for now while it uses testing-only packages
+			// like fmt and encoding/json
+			continue
+		}
+		loadStdlibPackage(lib, stdlibsDir, store)
+	}
+}
+
+func loadStdlibPackage(pkgPath, stdlibsDir string, store gno.Store) {
+	stdlibPath := filepath.Join(stdlibsDir, pkgPath)
+	if !osm.DirExists(stdlibPath) {
+		// does not exist.
+		panic(fmt.Sprintf("failed loading stdlib %q: does not exist", pkgPath))
+	}
+	memPkg := gno.ReadMemPackage(stdlibPath, pkgPath)
+	if memPkg.IsEmpty() {
+		// no gno files are present
+		panic(fmt.Sprintf("failed loading stdlib %q: not a valid MemPackage", pkgPath))
+	}
+
+	m := gno.NewMachineWithOptions(gno.MachineOptions{
+		PkgPath: "gno.land/r/stdlibs/" + pkgPath,
+		// PkgPath: pkgPath, XXX why?
+		Output: os.Stdout,
+		Store:  store,
+	})
+	defer m.Release()
+	m.RunMemPackage(memPkg, true)
 }
 
 func (vm *VMKeeper) getGnoStore(ctx sdk.Context) gno.Store {

gno.land/pkg/sdk/vm/keeper_test.go

@@ -366,6 +366,16 @@
 // Call Run with stdlibs.
 func TestVMKeeperRunImportStdlibs(t *testing.T) {
 	env := setupTestEnv()
+	testVMKeeperRunImportStdlibs(t, env)
+}
+
+// Call Run with stdlibs, "cold" loading the standard libraries
+func TestVMKeeperRunImportStdlibsColdStdlibLoad(t *testing.T) {
+	env := setupTestEnvCold()
+	testVMKeeperRunImportStdlibs(t, env)
+}
+
+func testVMKeeperRunImportStdlibs(t *testing.T, env testEnv) {
 	ctx := env.ctx
 
 	// Give "addr1" some gnots.

gnovm/pkg/gnolang/store.go

@@ -112,6 +112,16 @@
 	return ds
 }
 
+// CopyCachesFromStore allows to copy a store's internal object, type and
+// BlockNode cache into the dst store.
+// This is mostly useful for testing, where many stores have to be initialized.
+func CopyCachesFromStore(dst, src Store) {
+	ds, ss := dst.(*defaultStore), src.(*defaultStore)
+	ds.cacheObjects = maps.Clone(ss.cacheObjects)
+	ds.cacheTypes = maps.Clone(ss.cacheTypes)
+	ds.cacheNodes = maps.Clone(ss.cacheNodes)
+}
+
 func (ds *defaultStore) GetAllocator() *Allocator {
 	return ds.alloc
 }
@@ -562,7 +572,7 @@
 		// implementations works by running Machine.RunMemPackage with save = true,
 		// which would add the package to the store after running.
 		// Some packages may never be persisted, thus why we only attempt this twice.
-		if !isRetry {
+		if !isRetry && ds.pkgGetter != nil {
 			if pv := ds.GetPackage(path, false); pv != nil {
 				return ds.getMemPackage(path, true)
 			}

gnovm/stdlibs/native.go → gnovm/stdlibs/generated.go

@@ -1018,3 +1018,51 @@
 		},
 	},
 }
+
+var initOrder = [...]string{
+	"errors",
+	"internal/bytealg",
+	"io",
+	"unicode",
+	"unicode/utf8",
+	"bytes",
+	"strings",
+	"bufio",
+	"encoding/binary",
+	"math/bits",
+	"math",
+	"crypto/chacha20/chacha",
+	"crypto/cipher",
+	"crypto/chacha20",
+	"strconv",
+	"crypto/chacha20/rand",
+	"crypto/ed25519",
+	"crypto/sha256",
+	"encoding",
+	"encoding/base64",
+	"encoding/hex",
+	"hash",
+	"hash/adler32",
+	"math/overflow",
+	"math/rand",
+	"path",
+	"sort",
+	"net/url",
+	"regexp/syntax",
+	"regexp",
+	"std",
+	"testing",
+	"time",
+	"unicode/utf16",
+}
+
+// InitOrder returns the initialization order of the standard libraries.
+// This is calculated starting from the list of all standard libraries and
+// iterating through each: if a package depends on an unitialized package, that
+// is processed first, and so on recursively; matching the behaviour of Go's
+// [program initialization].
+//
+// [program initialization]: https://go.dev/ref/spec#Program_initialization
+func InitOrder() []string {
+	return initOrder[:]
+}