refactor(schema)!: move view interfaces from testing to schema (#21204)

schema/indexer/indexer.go

@@ -5,6 +5,7 @@ import (
 
  "cosmossdk.io/schema/appdata"
  "cosmossdk.io/schema/logutil"
+ "cosmossdk.io/schema/view"
 )
 
 // Config species the configuration passed to an indexer initialization function.
@@ -55,10 +56,11 @@ type InitParams struct {
  Config Config
 
  // Context is the context that the indexer should use to listen for a shutdown signal via Context.Done(). Other
- // parameters may also be passed through context from the app if necessary.
+ // parameters may also be passed through context from the app if necessary. It is expected to be non-nil.
  Context context.Context
 
- // Logger is a logger the indexer can use to write log messages.
+ // Logger is a logger the indexer can use to write log messages. It may be nil if the indexer does not need
+ // to write logs.
  Logger logutil.Logger
 }
 
@@ -67,12 +69,13 @@ type InitResult struct {
  // Listener is the indexer's app data listener.
  Listener appdata.Listener
 
- // LastBlockPersisted indicates the last block that the indexer persisted (if it is persisting data). It
- // should be 0 if the indexer has no data stored and wants to start syncing state. It should be -1 if the indexer
- // does not care to persist state at all and is just listening for some other streaming purpose. If the indexer
- // has persisted state and has missed some blocks, a runtime error will occur to prevent the indexer from continuing
- // in an invalid state. If an indexer starts indexing after a chain's genesis (returning 0), the indexer manager
- // will attempt to perform a catch-up sync of state. Historical events will not be replayed, but an accurate
+ // View is a view of indexed data that indexers can provide. It is optional and may be nil.
+ // If it is provided it can be used for automated testing and other purposes.
+ // At indexer start-up, the block number returned by the view will be used to determine the
+ // starting block for the indexer. If the block number is 0, the indexer manager will attempt
+ // to perform a catch-up sync of state. Historical events will not be replayed, but an accurate
  // representation of the current state at the height at which indexing began can be reproduced.
- LastBlockPersisted int64
+ // If the block number is non-zero but does not match the current chain height, a runtime error
+ // will occur because this is an unsafe condition that indicates lost data.
+ View view.AppData
 }

schema/testing/appdatasim/app_data.go

@@ -9,6 +9,7 @@ import (
  "cosmossdk.io/schema"
  "cosmossdk.io/schema/appdata"
  "cosmossdk.io/schema/testing/statesim"
+ "cosmossdk.io/schema/view"
 )
 
 // Options are the options for creating an app data simulator.
@@ -151,11 +152,11 @@ func (a *Simulator) ProcessPacket(packet appdata.Packet) error {
 }
 
 // AppState returns the current app state backing the simulator.
-func (a *Simulator) AppState() statesim.AppState {
+func (a *Simulator) AppState() view.AppState {
  return a.state
 }
 
 // BlockNum returns the current block number of the simulator.
-func (a *Simulator) BlockNum() uint64 {
- return a.blockNum
+func (a *Simulator) BlockNum() (uint64, error) {
+ return a.blockNum, nil
 }

schema/testing/appdatasim/diff.go

@@ -4,34 +4,36 @@ import (
  "fmt"
 
  "cosmossdk.io/schema/testing/statesim"
+ "cosmossdk.io/schema/view"
 )
 
-// HasAppData defines an interface for things that hold app data include app state.
-// If an indexer implements this then DiffAppData can be used to compare it with
-// the Simulator state which also implements this.
-type HasAppData interface {
- // AppState returns the app state.
- AppState() statesim.AppState
-
- // BlockNum returns the latest block number.
- BlockNum() uint64
-}
-
 // DiffAppData compares the app data of two objects that implement HasAppData.
 // This can be used by indexer to compare their state with the Simulator state
 // if the indexer implements HasAppData.
 // It returns a human-readable diff if the app data differs and the empty string
 // if they are the same.
-func DiffAppData(expected, actual HasAppData) string {
+func DiffAppData(expected, actual view.AppData) string {
  res := ""
 
  if stateDiff := statesim.DiffAppStates(expected.AppState(), actual.AppState()); stateDiff != "" {
  res += "App State Diff:\n"
  res += stateDiff
  }
 
- if expected.BlockNum() != actual.BlockNum() {
- res += fmt.Sprintf("BlockNum: expected %d, got %d\n", expected.BlockNum(), actual.BlockNum())
+ expectedBlock, err := expected.BlockNum()
+ if err != nil {
+ res += fmt.Sprintf("ERROR getting expected block num: %s\n", err)
+ return res
+ }
+
+ actualBlock, err := actual.BlockNum()
+ if err != nil {
+ res += fmt.Sprintf("ERROR getting actual block num: %s\n", err)
+ return res
+ }
+
+ if expectedBlock != actualBlock {
+ res += fmt.Sprintf("BlockNum: expected %d, got %d\n", expectedBlock, actualBlock)
  }
 
  return res

schema/testing/appdatasim/testdata/app_sim_example_schema.txt

@@ -78,7 +78,7 @@ StartBlock: {6 <nil> <nil>}
 OnObjectUpdate: {"ModuleName":"test_cases","Updates":[{"TypeName":"TwoKeys","Key":["A𞥟",981],"Value":null,"Delete":false},{"TypeName":"ManyValues","Key":"ᵕ؏­􏿽A","Value":{"Value1":-317,"Value2":"AA==","Value3":-37.62890625,"Value4":232},"Delete":false}]}
 OnObjectUpdate: {"ModuleName":"all_kinds","Updates":[{"TypeName":"test_address","Key":"AACHBAjyAgFHOQAABo+PGAK3Bj7TwwBb/wAB3gE=","Value":{"valNotNull":"HBwBHAY6AAKO+UwDKRICAT0lgRRvCRvHFFoNAigBAUEDHoQUfB2qApRB/z41AAubARsBATQg3gCppQMAAQwHAQ=="},"Delete":false}]}
 OnObjectUpdate: {"ModuleName":"all_kinds","Updates":[{"TypeName":"test_decimal","Key":"9.5E+8","Value":["-2","88111430.0122412446"],"Delete":false}]}
-OnObjectUpdate: {"ModuleName":"all_kinds","Updates":[{"TypeName":"test_uint8","Key":7,"Value":null,"Delete":true},{"TypeName":"test_time","Key":"1970-01-01T00:59:59.999999999+01:00","Value":["1970-01-01T01:00:00.000000001+01:00",null],"Delete":false}]}
+OnObjectUpdate: {"ModuleName":"all_kinds","Updates":[{"TypeName":"test_uint8","Key":7,"Value":null,"Delete":true},{"TypeName":"test_time","Key":"1969-12-31T18:59:59.999999999-05:00","Value":["1969-12-31T19:00:00.000000001-05:00",null],"Delete":false}]}
 OnObjectUpdate: {"ModuleName":"test_cases","Updates":[{"TypeName":"RetainDeletions","Key":"൴~𝔶ٞ蹯a_ ᛮ!؋aض©-?","Value":{"Value2":""},"Delete":false}]}
 OnObjectUpdate: {"ModuleName":"all_kinds","Updates":[{"TypeName":"test_uint8","Key":14,"Value":{"valNotNull":116},"Delete":false},{"TypeName":"test_duration","Key":100403021838,"Value":[1547,null],"Delete":false}]}
 OnObjectUpdate: {"ModuleName":"all_kinds","Updates":[{"TypeName":"test_int64","Key":-34196421,"Value":[56,224549431],"Delete":false},{"TypeName":"test_bool","Key":true,"Value":{"valNotNull":true,"valNullable":null},"Delete":false}]}

schema/testing/appdatasim/testdata/diff_example.txt

@@ -60,10 +60,10 @@ Module all_kinds
  Object key=š℠¼々¢~;-Ⱥ!˃a[ʰᾌ?{ᪧ৵%ᾯ¦〈: NOT FOUND
  Object Collection test_time
  OBJECT COUNT ERROR: expected 4, got 3
- Object key=1970-01-01 01:00:00.000000005 +0100 CET: NOT FOUND
- Object key=1970-01-01 01:00:00.001598687 +0100 CET
- valNotNull: expected 1970-01-01 01:00:00.007727197 +0100 CET, got 1970-01-01 01:00:00.034531678 +0100 CET
- valNullable: expected 1970-01-01 01:00:00.000000484 +0100 CET, got 1970-01-01 01:00:00.000000033 +0100 CET
+ Object key=1969-12-31 19:00:00.000000005 -0500 EST: NOT FOUND
+ Object key=1969-12-31 19:00:00.001598687 -0500 EST
+ valNotNull: expected 1969-12-31 19:00:00.007727197 -0500 EST, got 1969-12-31 19:00:00.034531678 -0500 EST
+ valNullable: expected 1969-12-31 19:00:00.000000484 -0500 EST, got 1969-12-31 19:00:00.000000033 -0500 EST
  Object Collection test_uint16
  OBJECT COUNT ERROR: expected 4, got 3
  Object key=23712: NOT FOUND
@@ -75,5 +75,5 @@ Module all_kinds
  Object Collection test_uint8
  OBJECT COUNT ERROR: expected 3, got 2
  Object key=1: NOT FOUND
- Module test_cases: NOT FOUND
+ Module test_cases: actual module NOT FOUND
 BlockNum: expected 2, got 1

schema/testing/statesim/app.go

@@ -9,6 +9,7 @@ import (
 
  "cosmossdk.io/schema"
  "cosmossdk.io/schema/appdata"
+ "cosmossdk.io/schema/view"
 )
 
 // App is a collection of simulated module states corresponding to an app's schema for testing purposes.
@@ -27,7 +28,7 @@ func NewApp(appSchema map[string]schema.ModuleSchema, options Options) *App {
  }
 
  for moduleName, moduleSchema := range appSchema {
- moduleState := NewModule(moduleSchema, options)
+ moduleState := NewModule(moduleName, moduleSchema, options)
  app.moduleStates.Set(moduleName, moduleState)
  }
 
@@ -62,7 +63,7 @@ func (a *App) InitializeModule(data appdata.ModuleInitializationData) error {
  return fmt.Errorf("module %s already initialized", data.ModuleName)
  }
 
- a.moduleStates.Set(data.ModuleName, NewModule(data.Schema, a.options))
+ a.moduleStates.Set(data.ModuleName, NewModule(data.ModuleName, data.Schema, a.options))
  return nil
 }
 
@@ -91,18 +92,22 @@ func (a *App) UpdateGen() *rapid.Generator[appdata.ObjectUpdateData] {
 }
 
 // GetModule returns the module state for the given module name.
-func (a *App) GetModule(moduleName string) (ModuleState, bool) {
- return a.moduleStates.Get(moduleName)
+func (a *App) GetModule(moduleName string) (view.ModuleState, error) {
+ mod, ok := a.moduleStates.Get(moduleName)
+ if !ok {
+ return nil, nil
+ }
+ return mod, nil
 }
 
 // Modules iterates over all the module state instances in the app.
-func (a *App) Modules(f func(moduleName string, modState ModuleState) bool) {
+func (a *App) Modules(f func(modState view.ModuleState, err error) bool) {
  a.moduleStates.Scan(func(key string, value *Module) bool {
- return f(key, value)
+ return f(value, nil)
  })
 }
 
 // NumModules returns the number of modules in the app.
-func (a *App) NumModules() int {
- return a.moduleStates.Len()
+func (a *App) NumModules() (int, error) {
+ return a.moduleStates.Len(), nil
 }

schema/testing/statesim/app_diff.go

@@ -1,32 +1,46 @@
 package statesim
 
-import "fmt"
+import (
+ "fmt"
 
-// AppState defines an interface for things that represent application state in schema format.
-type AppState interface {
- // GetModule returns the module state for the given module name.
- GetModule(moduleName string) (ModuleState, bool)
-
- // Modules iterates over all the module state instances in the app.
- Modules(f func(moduleName string, modState ModuleState) bool)
-
- // NumModules returns the number of modules in the app.
- NumModules() int
-}
+ "cosmossdk.io/schema/view"
+)
 
 // DiffAppStates compares the app state of two objects that implement AppState and returns a string with a diff if they
 // are different or the empty string if they are the same.
-func DiffAppStates(expected, actual AppState) string {
+func DiffAppStates(expected, actual view.AppState) string {
  res := ""
 
- if expected.NumModules() != actual.NumModules() {
- res += fmt.Sprintf("MODULE COUNT ERROR: expected %d, got %d\n", expected.NumModules(), actual.NumModules())
+ expectNumModules, err := expected.NumModules()
+ if err != nil {
+ res += fmt.Sprintf("ERROR getting expected num modules: %s\n", err)
+ return res
+ }
+
+ actualNumModules, err := actual.NumModules()
+ if err != nil {
+ res += fmt.Sprintf("ERROR getting actual num modules: %s\n", err)
+ return res
+ }
+
+ if expectNumModules != actualNumModules {
+ res += fmt.Sprintf("MODULE COUNT ERROR: expected %d, got %d\n", expectNumModules, actualNumModules)
  }
 
- expected.Modules(func(moduleName string, expectedMod ModuleState) bool {
- actualMod, found := actual.GetModule(moduleName)
- if !found {
- res += fmt.Sprintf("Module %s: NOT FOUND\n", moduleName)
+ expected.Modules(func(expectedMod view.ModuleState, err error) bool {
+ if err != nil {
+ res += fmt.Sprintf("ERROR getting expected module: %s\n", err)
+ return true
+ }
+
+ moduleName := expectedMod.ModuleName()
+ actualMod, err := actual.GetModule(moduleName)
+ if err != nil {
+ res += fmt.Sprintf("ERROR getting actual module: %s\n", err)
+ return true
+ }
+ if actualMod == nil {
+ res += fmt.Sprintf("Module %s: actual module NOT FOUND\n", moduleName)
  return true
  }
 

schema/testing/statesim/module.go

@@ -8,17 +8,19 @@ import (
  "pgregory.net/rapid"
 
  "cosmossdk.io/schema"
+ "cosmossdk.io/schema/view"
 )
 
 // Module is a collection of object collections corresponding to a module's schema for testing purposes.
 type Module struct {
+ name string
  moduleSchema schema.ModuleSchema
  objectCollections *btree.Map[string, *ObjectCollection]
  updateGen *rapid.Generator[schema.ObjectUpdate]
 }
 
 // NewModule creates a new Module for the given module schema.
-func NewModule(moduleSchema schema.ModuleSchema, options Options) *Module {
+func NewModule(name string, moduleSchema schema.ModuleSchema, options Options) *Module {
  objectCollections := &btree.Map[string, *ObjectCollection]{}
  var objectTypeNames []string
 
@@ -39,6 +41,7 @@ func NewModule(moduleSchema schema.ModuleSchema, options Options) *Module {
  })
 
  return &Module{
+ name: name,
  moduleSchema: moduleSchema,
  updateGen: updateGen,
  objectCollections: objectCollections,
@@ -61,24 +64,33 @@ func (o *Module) UpdateGen() *rapid.Generator[schema.ObjectUpdate] {
  return o.updateGen
 }
 
+// ModuleName returns the name of the module.
+func (o *Module) ModuleName() string {
+ return o.name
+}
+
 // ModuleSchema returns the module schema for the module.
 func (o *Module) ModuleSchema() schema.ModuleSchema {
  return o.moduleSchema
 }
 
 // GetObjectCollection returns the object collection for the given object type.
-func (o *Module) GetObjectCollection(objectType string) (ObjectCollectionState, bool) {
- return o.objectCollections.Get(objectType)
+func (o *Module) GetObjectCollection(objectType string) (view.ObjectCollection, error) {
+ obj, ok := o.objectCollections.Get(objectType)
+ if !ok {
+ return nil, nil
+ }
+ return obj, nil
 }
 
 // ObjectCollections iterates over all object collections in the module.
-func (o *Module) ObjectCollections(f func(value ObjectCollectionState) bool) {
+func (o *Module) ObjectCollections(f func(value view.ObjectCollection, err error) bool) {
  o.objectCollections.Scan(func(key string, value *ObjectCollection) bool {
- return f(value)
+ return f(value, nil)
  })
 }
 
 // NumObjectCollections returns the number of object collections in the module.
-func (o *Module) NumObjectCollections() int {
- return o.objectCollections.Len()
+func (o *Module) NumObjectCollections() (int, error) {
+ return o.objectCollections.Len(), nil
 }