Problem: there's no compact historical state storage

Closes: crypto-org-chain#704
Solution:
- Integration version store and streaming service.

multiple backends

db size benchmark

support mdbx backend through tm-db

store latest version

support GetLatestVersion

query multistore

test versiondb streamer

fix lint

fix state listener

temp

fix state listening

optimize common case

fix lint

try to fix mdbx build

update cosmos-sdk

fix clone append

add test case

fix subkey problem in history index

revert chunking, hard to work with variable length key

support iterator

check future height

fix lint

new state listener

fix latest state in query

fix integration test

fix prune node test

update dependency

add utility to read from file streamer

Update versiondb/multistore.go

Signed-off-by: yihuang <huang@crypto.com>

add unit test

create common backend test cases

update dependency

update with new file streamer format

Problem: python3.10 is not used in integration tests

Solution:
- start using python3.10, prepare to later PRs which need the new features
- update nixpkgs nesserary for the nix stuff to work.

python-roaring64

remove debug log

add test cases, improve coverage

Makefile

@@ -74,6 +74,10 @@ ifeq (boltdb,$(findstring boltdb,$(COSMOS_BUILD_OPTIONS)))
  BUILD_TAGS += boltdb
 endif
 
+ifeq (mdbx,$(findstring mdbx,$(COSMOS_BUILD_OPTIONS)))
+ BUILD_TAGS += mdbx
+endif
+
 ifeq (,$(findstring nostrip,$(COSMOS_BUILD_OPTIONS)))
  ldflags += -w -s
 endif

app/app.go

@@ -5,12 +5,15 @@ import (
  "net/http"
  "os"
  "path/filepath"
+ "strings"
  "sync"
 
+ "github.com/crypto-org-chain/cronos/x/cronos"
  "github.com/crypto-org-chain/cronos/x/cronos/middleware"
 
  "github.com/cosmos/cosmos-sdk/client"
  "github.com/cosmos/cosmos-sdk/codec/types"
+ "github.com/cosmos/cosmos-sdk/server"
  "github.com/gorilla/mux"
  "github.com/rakyll/statik/fs"
  "github.com/spf13/cast"
@@ -122,7 +125,8 @@ import (
 
  // this line is used by starport scaffolding # stargate/app/moduleImport
  cronosappclient "github.com/crypto-org-chain/cronos/client"
- "github.com/crypto-org-chain/cronos/x/cronos"
+ "github.com/crypto-org-chain/cronos/versiondb"
+ "github.com/crypto-org-chain/cronos/versiondb/tmdb"
  cronosclient "github.com/crypto-org-chain/cronos/x/cronos/client"
  cronoskeeper "github.com/crypto-org-chain/cronos/x/cronos/keeper"
  evmhandlers "github.com/crypto-org-chain/cronos/x/cronos/keeper/evmhandlers"
@@ -350,7 +354,8 @@ func New(
  // configure state listening capabilities using AppOptions
  // we are doing nothing with the returned streamingServices and waitGroup in this case
  // Only support file streamer right now.
- if cast.ToString(appOpts.Get(cronosappclient.FlagStreamers)) == "file" {
+ streamers := cast.ToString(appOpts.Get(cronosappclient.FlagStreamers))
+ if strings.Contains(streamers, "file") {
  streamingDir := filepath.Join(cast.ToString(appOpts.Get(flags.FlagHome)), "data", FileStreamerDirectory)
  if err := os.MkdirAll(streamingDir, os.ModePerm); err != nil {
  panic(err)
@@ -361,7 +366,7 @@ func New(
  for _, storeKey := range keys {
  exposeStoreKeys = append(exposeStoreKeys, storeKey)
  }
- service, err := file.NewStreamingService(streamingDir, "", exposeStoreKeys, appCodec)
+ service, err := file.NewStreamingService(streamingDir, "", exposeStoreKeys, appCodec, false)
  if err != nil {
  panic(err)
  }
@@ -373,6 +378,40 @@ func New(
  }
  }
 
+ if strings.Contains(streamers, "versiondb") {
+ rootDir := cast.ToString(appOpts.Get(flags.FlagHome))
+ dataDir := filepath.Join(rootDir, "data", "versiondb")
+ if err := os.MkdirAll(dataDir, os.ModePerm); err != nil {
+ panic(err)
+ }
+ backendType := server.GetAppDBBackend(appOpts)
+ plainDB, err := dbm.NewDB("plain", backendType, dataDir)
+ if err != nil {
+ panic(err)
+ }
+ historyDB, err := dbm.NewDB("history", backendType, dataDir)
+ if err != nil {
+ panic(err)
+ }
+ changesetDB, err := dbm.NewDB("changeset", backendType, dataDir)
+ if err != nil {
+ panic(err)
+ }
+ versionDB := tmdb.NewStore(plainDB, historyDB, changesetDB)
+
+ // default to exposing all
+ exposeStoreKeys := make([]storetypes.StoreKey, 0, len(keys))
+ for _, storeKey := range keys {
+ exposeStoreKeys = append(exposeStoreKeys, storeKey)
+ }
+ service := versiondb.NewStreamingService(versionDB, exposeStoreKeys)
+ bApp.SetStreamingService(service)
+ qms := versiondb.NewMultiStore(versionDB, exposeStoreKeys)
+ qms.MountTransientStores(tkeys)
+ qms.MountMemoryStores(memKeys)
+ bApp.SetQueryMultiStore(qms)
+ }
+
  app := &App{
  BaseApp: bApp,
  cdc: cdc,

default.nix

@@ -8,7 +8,7 @@
 let
  version = "v0.9.0";
  pname = "cronosd";
- tags = [ "ledger" "netgo" network ]
+ tags = [ "ledger" "netgo" network "mdbx" ]
  ++ lib.lists.optionals (rocksdb != null) [ "rocksdb" "rocksdb_build" ];
  ldflags = lib.concatStringsSep "\n" ([
  "-X github.com/cosmos/cosmos-sdk/version.Name=cronos"
@@ -27,6 +27,7 @@ buildGoApplication rec {
  "!/app/"
  "!/cmd/"
  "!/client/"
+ "!/versiondb/"
  "!go.mod"
  "!go.sum"
  "!gomod2nix.toml"

go.mod

@@ -4,6 +4,7 @@ go 1.18
 
 require (
  cosmossdk.io/math v1.0.0-beta.3
+ github.com/RoaringBitmap/roaring v1.2.1
  github.com/armon/go-metrics v0.4.1
  github.com/cosmos/cosmos-sdk v0.46.3
  github.com/cosmos/ibc-go/v5 v5.0.0
@@ -45,10 +46,12 @@ require (
  github.com/beorn7/perks v1.0.1 // indirect
  github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
  github.com/bgentry/speakeasy v0.1.0 // indirect
+ github.com/bits-and-blooms/bitset v1.2.0 // indirect
  github.com/btcsuite/btcd v0.22.1 // indirect
  github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
  github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect
  github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce // indirect
+ github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b // indirect
  github.com/cenkalti/backoff/v4 v4.1.3 // indirect
  github.com/cespare/xxhash v1.1.0 // indirect
  github.com/cespare/xxhash/v2 v2.1.2 // indirect
@@ -69,9 +72,8 @@ require (
  github.com/deckarep/golang-set v1.8.0 // indirect
  github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
  github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect
- github.com/dgraph-io/badger/v2 v2.2007.4 // indirect
+ github.com/dgraph-io/badger/v3 v3.2103.2 // indirect
  github.com/dgraph-io/ristretto v0.1.0 // indirect
- github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
  github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect
  github.com/dop251/goja v0.0.0-20220405120441-9037c2b61cbf // indirect
  github.com/dustin/go-humanize v1.0.0 // indirect
@@ -91,7 +93,8 @@ require (
  github.com/golang/glog v1.0.0 // indirect
  github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
  github.com/golang/snappy v0.0.4 // indirect
- github.com/google/btree v1.0.1 // indirect
+ github.com/google/btree v1.1.2 // indirect
+ github.com/google/flatbuffers v2.0.0+incompatible // indirect
  github.com/google/go-cmp v0.5.8 // indirect
  github.com/google/orderedcode v0.0.1 // indirect
  github.com/google/uuid v1.3.0 // indirect
@@ -135,7 +138,9 @@ require (
  github.com/mitchellh/go-homedir v1.1.0 // indirect
  github.com/mitchellh/go-testing-interface v1.0.0 // indirect
  github.com/mitchellh/mapstructure v1.5.0 // indirect
+ github.com/mschoch/smat v0.2.0 // indirect
  github.com/mtibben/percent v0.2.1 // indirect
+ github.com/natefinch/atomic v1.0.1 // indirect
  github.com/olekukonko/tablewriter v0.0.5 // indirect
  github.com/pelletier/go-toml v1.9.5 // indirect
  github.com/pelletier/go-toml/v2 v2.0.5 // indirect
@@ -165,6 +170,7 @@ require (
  github.com/tendermint/go-amino v0.16.0 // indirect
  github.com/tklauser/go-sysconf v0.3.10 // indirect
  github.com/tklauser/numcpus v0.4.0 // indirect
+ github.com/torquem-ch/mdbx-go v0.26.0 // indirect
  github.com/tyler-smith/go-bip39 v1.1.0 // indirect
  github.com/ulikunitz/xz v0.5.8 // indirect
  github.com/zondax/hid v0.9.1-0.20220302062450-5552068d2266 // indirect
@@ -189,7 +195,7 @@ require (
 )
 
 replace (
- github.com/cosmos/cosmos-sdk => github.com/cosmos/cosmos-sdk v0.46.2
+ github.com/cosmos/cosmos-sdk => github.com/yihuang/cosmos-sdk v0.43.0-beta1.0.20221014023203-2c6b9d06b12d
  github.com/ethereum/go-ethereum => github.com/ethereum/go-ethereum v1.10.19
 
  // Fix upstream GHSA-h395-qcrw-5vmq vulnerability.
@@ -199,6 +205,11 @@ replace (
 
  github.com/peggyjv/gravity-bridge/module/v2 => github.com/crypto-org-chain/gravity-bridge/module/v2 v2.0.1-0.20221027085649-2107c6bd6bc4
 
+ // https://github.com/tendermint/tm-db/pull/297
+ github.com/tendermint/tm-db => github.com/yihuang/tm-db v0.0.0-20221006023748-f6214ae9454d
+
+ github.com/torquem-ch/mdbx-go => github.com/yihuang/mdbx-go v0.0.0-20221010042614-b72b4f091d88
+
  // TODO: remove after fixed https://github.com/cosmos/cosmos-sdk/issues/11364
  github.com/zondax/hid => github.com/zondax/hid v0.9.0
 )

gomod2nix.toml

@@ -31,6 +31,9 @@ schema = 3
  [mod."github.com/ChainSafe/go-schnorrkel"]
  version = "v0.0.0-20200405005733-88cbf1b4c40d"
  hash = "sha256-i8RXZemJGlSjBT35oPm0SawFiBoIU5Pkq5xp4n/rzCY="
+ [mod."github.com/RoaringBitmap/roaring"]
+ version = "v1.2.1"
+ hash = "sha256-0/R956wrCW71eOE36CbxGJJRuQjKwvvIQ/D8QTn2A6w="
  [mod."github.com/StackExchange/wmi"]
  version = "v1.2.1"
  hash = "sha256-1BoEeWAWyebH+1mMuyPhWZut8nWHb6r73MgcqlGuUEY="
@@ -58,6 +61,9 @@ schema = 3
  [mod."github.com/bgentry/speakeasy"]
  version = "v0.1.0"
  hash = "sha256-Gt1vj6CFovLnO6wX5u2O4UfecY9V2J9WGw1ez4HMrgk="
+ [mod."github.com/bits-and-blooms/bitset"]
+ version = "v1.2.0"
+ hash = "sha256-IxNmtELycM+XVzg4qBv04hAJUT3nSWuyP9R+8zc9LmU="
  [mod."github.com/btcsuite/btcd"]
  version = "v0.22.1"
  hash = "sha256-hBU+roIELcmbW2Gz7eGZzL9qNA1bakq5wNxqCgs4TKc="
@@ -70,6 +76,9 @@ schema = 3
  [mod."github.com/btcsuite/btcutil"]
  version = "v1.0.3-0.20201208143702-a53e38424cce"
  hash = "sha256-4kasJReFcj25JRHx9dJMct3yDkHqVoHGUx5cu45Msfo="
+ [mod."github.com/c2h5oh/datasize"]
+ version = "v0.0.0-20220606134207-859f65c6625b"
+ hash = "sha256-1uH+D3w0Y/B3poXm545XGrT4S4c+msTbj7gKgu9pbPM="
  [mod."github.com/cenkalti/backoff/v4"]
  version = "v4.1.3"
  hash = "sha256-u6MEDopHoTWAZoVvvXOKnAg++xre53YgQx0gmf6t2KU="
@@ -98,12 +107,15 @@ schema = 3
  version = "v1.0.0-alpha7"
  hash = "sha256-2wCH+toTF2A6MfFjOa13muEH5oBCcxAhZEqirNOrBA0="
  [mod."github.com/cosmos/cosmos-sdk"]
- version = "v0.46.2"
- hash = "sha256-Lgn4+Vd5PUUkfHc+lTdK2G6/nymZekFVTe1FxWRqh2w="
- replaced = "github.com/cosmos/cosmos-sdk"
+ version = "v0.43.0-beta1.0.20221014023203-2c6b9d06b12d"
+ hash = "sha256-t/QEOJgATr82KFXes+AzNlNL6w2tGHfiwZAE4PdO5GE="
+ replaced = "github.com/yihuang/cosmos-sdk"
  [mod."github.com/cosmos/go-bip39"]
  version = "v1.0.0"
  hash = "sha256-Qm2aC2vaS8tjtMUbHmlBSagOSqbduEEDwc51qvQaBmA="
+ [mod."github.com/cosmos/gogoproto"]
+ version = "v1.4.2"
+ hash = "sha256-hOY+mhPDYWcSYSdth2AW7IONdgicqQir0z/1XrXt9NY="
  [mod."github.com/cosmos/gorocksdb"]
  version = "v1.2.0"
  hash = "sha256-209TcVuXc5s/TcOvNlaQ1HEJAUDTEK3nxPhs+d8TEcY="
@@ -137,15 +149,12 @@ schema = 3
  [mod."github.com/desertbit/timer"]
  version = "v0.0.0-20180107155436-c41aec40b27f"
  hash = "sha256-abLOtEcomAqCWLphd2X6WkD/ED764w6sa6unox4BXss="
- [mod."github.com/dgraph-io/badger/v2"]
- version = "v2.2007.4"
- hash = "sha256-+KwqZJZpViv8S3TqUVvPXrFoMgWFyS3NoLsi4RR5fGk="
+ [mod."github.com/dgraph-io/badger/v3"]
+ version = "v3.2103.2"
+ hash = "sha256-F6pvsaSKwXOl9RfnUQFqAl6xpCVu9+rthQgOxhKVk1g="
  [mod."github.com/dgraph-io/ristretto"]
  version = "v0.1.0"
  hash = "sha256-01jneg1+1x8tTfUTBZ+6mHkQaqXVnPYxLJyJhJQcvt4="
- [mod."github.com/dgryski/go-farm"]
- version = "v0.0.0-20200201041132-a6ae2369ad13"
- hash = "sha256-aOMlPwFY36bLiiIx4HonbCYRAhagk5N6HAWN7Ygif+E="
  [mod."github.com/dlclark/regexp2"]
  version = "v1.4.1-0.20201116162257-a2a8dda75c91"
  hash = "sha256-VNNMZIc7NkDg3DVLnqeJNM/KZqkkaZu2/HTLBL8X2xE="
@@ -164,6 +173,7 @@ schema = 3
  [mod."github.com/ethereum/go-ethereum"]
  version = "v1.10.19"
  hash = "sha256-7FPnTGcCb8Xd1QVR+6PmGTaHdTY1mm/8osFTW1JLuG8="
+ replaced = "github.com/ethereum/go-ethereum"
  [mod."github.com/evmos/ethermint"]
  version = "v0.6.1-0.20221003153722-491c3da7ebd7"
  hash = "sha256-vnfjk57gYa+F8nn0LByX/B1LV8PY2Jvm8vXV6be4ufc="
@@ -217,8 +227,11 @@ schema = 3
  version = "v0.0.4"
  hash = "sha256-Umx+5xHAQCN/Gi4HbtMhnDCSPFAXSsjVbXd8n5LhjAA="
  [mod."github.com/google/btree"]
- version = "v1.0.1"
- hash = "sha256-1PIeFGgUL4BK/StL/D12pg9bEQ5HfMT/fMLdus4pZTs="
+ version = "v1.1.2"
+ hash = "sha256-K7V2obq3pLM71Mg0vhhHtZ+gtaubwXPQx3xcIyZDCjM="
+ [mod."github.com/google/flatbuffers"]
+ version = "v2.0.0+incompatible"
+ hash = "sha256-4Db9FdOL60Da4H1+K4Qv02w4omxdsh3uzpmY1vtqHeA="
  [mod."github.com/google/go-cmp"]
  version = "v0.5.8"
  hash = "sha256-8zkIo+Sr1NXMnj3PNmvjX2sZKnAKWXOFvmnX7D9bwxQ="
@@ -354,9 +367,15 @@ schema = 3
  [mod."github.com/mitchellh/mapstructure"]
  version = "v1.5.0"
  hash = "sha256-ztVhGQXs67MF8UadVvG72G3ly0ypQW0IRDdOOkjYwoE="
+ [mod."github.com/mschoch/smat"]
+ version = "v0.2.0"
+ hash = "sha256-DZvUJXjIcta3U+zxzgU3wpoGn/V4lpBY7Xme8aQUi+E="
  [mod."github.com/mtibben/percent"]
  version = "v0.2.1"
  hash = "sha256-Zj1lpCP6mKQ0UUTMs2By4LC414ou+iJzKkK+eBHfEcc="
+ [mod."github.com/natefinch/atomic"]
+ version = "v1.0.1"
+ hash = "sha256-fbOVHCwRNI8PFjC4o0YXpKZO0JU2aWTfH5c7WXXKMHg="
  [mod."github.com/olekukonko/tablewriter"]
  version = "v0.0.5"
  hash = "sha256-/5i70IkH/qSW5KjGzv8aQNKh9tHoz98tqtL0K2DMFn4="
@@ -461,14 +480,19 @@ schema = 3
  version = "v0.34.22"
  hash = "sha256-4p4cpyCWjBbNQUpYN2gDJvnyj+Pov9hw5uRjHrrO++Y="
  [mod."github.com/tendermint/tm-db"]
- version = "v0.6.7"
- hash = "sha256-hl/3RrBrpkk2zA6dmrNlIYKs1/GfqegSscDSkA5Pjlo="
+ version = "v0.0.0-20221006023748-f6214ae9454d"
+ hash = "sha256-mtTVR3f3A9CmcyJBXTeurQuHGiVA5hIzqlxskz1M1qk="
+ replaced = "github.com/yihuang/tm-db"
  [mod."github.com/tklauser/go-sysconf"]
  version = "v0.3.10"
  hash = "sha256-Zf2NsgM9+HeM949vCce4HQtSbfUiFpeiQ716yKcFyx4="
  [mod."github.com/tklauser/numcpus"]
  version = "v0.4.0"
  hash = "sha256-ndE82nOb3agubhEV7aRzEqqTlN4DPbKFHEm2+XZLn8k="
+ [mod."github.com/torquem-ch/mdbx-go"]
+ version = "v0.0.0-20221010042614-b72b4f091d88"
+ hash = "sha256-HWsrhzSGoYgEUUziQPHEtQGwBQ/upg3/f1fY2NGiC0g="
+ replaced = "github.com/yihuang/mdbx-go"
  [mod."github.com/tyler-smith/go-bip39"]
  version = "v1.1.0"
  hash = "sha256-3YhWBtSwRLGwm7vNwqumphZG3uLBW1vwT9QkQ8JuSjU="
@@ -478,6 +502,7 @@ schema = 3
  [mod."github.com/zondax/hid"]
  version = "v0.9.0"
  hash = "sha256-PvXtxXo/3C+DS9ZeGBlr4zXbIpaYNtMqLzxYhusFXNY="
+ replaced = "github.com/zondax/hid"
  [mod."go.etcd.io/bbolt"]
  version = "v1.3.6"
  hash = "sha256-DenVAmyN22xUiivk6fdJp4C9ZnUJXCMDUf8E0goRRV4="

integration_tests/configs/default.jsonnet

@@ -2,7 +2,7 @@
  dotenv: '../../scripts/.env',
  'cronos_777-1': {
  cmd: 'cronosd',
- 'start-flags': '--trace --streamers file',
+ 'start-flags': '--trace --streamers versiondb,file',
  config: {
  mempool: {
  version: 'v1',

integration_tests/configs/pruned-node.jsonnet

@@ -2,6 +2,8 @@ local config = import 'default.jsonnet';
 
 config {
  'cronos_777-1'+: {
+ // don't enable versiondb, since it don't do pruning right now
+ 'start-flags': '--trace --streamers file',
  'app-config'+: {
  pruning: 'everything',
  'state-sync'+: {

integration_tests/configs/state_benchmark.jsonnet

@@ -0,0 +1,36 @@
+local config = import 'default.jsonnet';
+
+config {
+ 'cronos_777-1'+: {
+ 'start-flags': '--trace --streamers file,versiondb',
+ 'app-config'+: {
+ 'app-db-backend': 'rocksdb',
+ 'state-sync'+: {
+ 'snapshot-interval': 0,
+ },
+ },
+ validators: [
+ super.validators[0],
+ super.validators[1] {
+ 'app-config'+: {
+ pruning: 'everything',
+ },
+ },
+ ] + super.validators[2:],
+ genesis+: {
+ consensus_params+: {
+ block+: {
+ max_gas: '163000000',
+ },
+ },
+ app_state+: {
+ feemarket+: {
+ params+: {
+ no_base_fee: true,
+ min_gas_multiplier: '0',
+ },
+ },
+ },
+ },
+ },
+}