Implement in-memory DB to run Yorkie without MongoDB (#276)

README.md

@@ -17,7 +17,7 @@ Yorkie is an open source document store for building collaborative editing appli
 Yorkie consists of three main components: Client, Document and Agent.
 
  ```
-  Client "A" (Go)                 Agent                        Mongo DB
+  Client "A" (Go)                 Agent                        MemDB or MongoDB
 ┌───────────────────┐           ┌────────────────────────┐   ┌───────────┐
 │  Document "D-1"   │◄─Changes─►│  Collection "C-1"      │   │ Changes   │
 │  { a: 1, b: {} }  │           │ ┌───────────────────┐  │◄─►│ Snapshots │
@@ -29,8 +29,8 @@ Yorkie consists of three main components: Client, Document and Agent.
 └───────────────────┘           │ └───────────────────┘  │
   Client "C" (Admin)             │                        │
 ┌────────────────────┐          └────────────────────────┘
-│  Query "Q-1"       │               ▲
-│ db[c-1].find({a:2})├─MongoDB Query─┘
+│  Query "Q-1"       │              ▲
+│ db[c-1].find({a:2})├───DB Query───┘
 └────────────────────┘
  ```
 
@@ -59,8 +59,6 @@ Yorkie: 0.1.8
 ...
 ```
 
-Yorkie uses MongoDB to store its data. To start MongoDB, type `docker-compose -f docker/docker-compose.yml up -d`.
-
 Next, let's start a Yorkie agent. Agent runs until they're told to quit and handle the communication of maintenance tasks of Agent. and start the agent:
 
 ```

go.mod

@@ -9,11 +9,13 @@ require (
 	github.com/google/uuid v1.1.2
 	github.com/grpc-ecosystem/go-grpc-middleware v1.2.2
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
+	github.com/hashicorp/go-memdb v1.3.2
 	github.com/moby/locker v1.0.1
 	github.com/prometheus/client_golang v1.11.0
 	github.com/rs/xid v1.2.1
 	github.com/spf13/cobra v1.1.3
 	github.com/stretchr/testify v1.7.0
+	go.etcd.io/etcd/api/v3 v3.5.1
 	go.etcd.io/etcd/client/v3 v3.5.1
 	go.mongodb.org/mongo-driver v1.5.1
 	go.uber.org/zap v1.17.0
@@ -77,7 +79,9 @@ require (
 	github.com/gostaticanalysis/forcetypeassert v0.0.0-20200621232751-01d4955beaa5 // indirect
 	github.com/gostaticanalysis/nilerr v0.1.1 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
+	github.com/hashicorp/go-immutable-radix v1.3.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jgautheron/goconst v1.5.1 // indirect
@@ -150,7 +154,6 @@ require (
 	github.com/xdg-go/stringprep v1.0.2 // indirect
 	github.com/yeya24/promlinter v0.1.0 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
-	go.etcd.io/etcd/api/v3 v3.5.1 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.1 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect

go.sum

@@ -355,6 +355,10 @@ github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/U
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-immutable-radix v1.3.0 h1:8exGP7ego3OmkfksihtSouGMZ+hQrhxx+FVELeXpVPE=
+github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-memdb v1.3.2 h1:RBKHOsnSszpU6vxq80LzC2BaQjuuvoyaQbkLTf7V7g8=
+github.com/hashicorp/go-memdb v1.3.2/go.mod h1:Mluclgwib3R93Hk5fxEfiRhB+6Dar64wWh71LpNSe3g=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
@@ -363,10 +367,12 @@ github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=

internal/cli/agent.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/yorkie-team/yorkie/internal/log"
 	"github.com/yorkie-team/yorkie/yorkie"
+	"github.com/yorkie-team/yorkie/yorkie/backend/db/mongo"
 	"github.com/yorkie-team/yorkie/yorkie/backend/sync/etcd"
 )
 
@@ -37,7 +38,9 @@ var (
 var (
 	flagConfPath string
 
+	mongoConnectionURI     string
 	mongoConnectionTimeout time.Duration
+	mongoYorkieDatabase    string
 	mongoPingTimeout       time.Duration
 
 	authWebhookMaxWaitInterval time.Duration
@@ -58,12 +61,19 @@ func newAgentCmd() *cobra.Command {
 		Use:   "agent [options]",
 		Short: "Starts yorkie agent",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			conf.Mongo.ConnectionTimeout = mongoConnectionTimeout.String()
-			conf.Mongo.PingTimeout = mongoPingTimeout.String()
 			conf.Backend.AuthWebhookMaxWaitInterval = authWebhookMaxWaitInterval.String()
 			conf.Backend.AuthWebhookCacheAuthTTL = authWebhookCacheAuthTTL.String()
 			conf.Backend.AuthWebhookCacheUnauthTTL = authWebhookCacheUnauthTTL.String()
 
+			if mongoConnectionURI != "" {
+				conf.Mongo = &mongo.Config{
+					ConnectionURI:     mongoConnectionURI,
+					ConnectionTimeout: mongoConnectionTimeout.String(),
+					YorkieDatabase:    mongoYorkieDatabase,
+					PingTimeout:       mongoPingTimeout.String(),
+				}
+			}
+
 			if etcdEndpoints != nil {
 				conf.ETCD = &etcd.Config{
 					Endpoints:     etcdEndpoints,
@@ -188,20 +198,20 @@ func init() {
 		false,
 		"Enable runtime profiling data via HTTP server.",
 	)
+	cmd.Flags().StringVar(
+		&mongoConnectionURI,
+		"mongo-connection-uri",
+		"",
+		"MongoDB's connection URI",
+	)
 	cmd.Flags().DurationVar(
 		&mongoConnectionTimeout,
 		"mongo-connection-timeout",
 		yorkie.DefaultMongoConnectionTimeout,
 		"Mongo DB's connection timeout",
 	)
 	cmd.Flags().StringVar(
-		&conf.Mongo.ConnectionURI,
-		"mongo-connection-uri",
-		yorkie.DefaultMongoConnectionURI,
-		"MongoDB's connection URI",
-	)
-	cmd.Flags().StringVar(
-		&conf.Mongo.YorkieDatabase,
+		&mongoYorkieDatabase,
 		"mongo-yorkie-database",
 		yorkie.DefaultMongoYorkieDatabase,
 		"Yorkie's database name in MongoDB",

pkg/document/document.go

@@ -129,6 +129,11 @@ func (d *Document) ApplyChangePack(pack *change.Pack) error {
 	return nil
 }
 
+// InternalDocument returns the internal document.
+func (d *Document) InternalDocument() *InternalDocument {
+	return d.doc
+}
+
 // Key returns the key of this document.
 func (d *Document) Key() *key.Key {
 	return d.doc.key

yorkie/backend/backend.go

@@ -26,10 +26,11 @@ import (
 	"github.com/yorkie-team/yorkie/internal/log"
 	"github.com/yorkie-team/yorkie/pkg/cache"
 	"github.com/yorkie-team/yorkie/yorkie/backend/db"
+	memdb "github.com/yorkie-team/yorkie/yorkie/backend/db/memory"
 	"github.com/yorkie-team/yorkie/yorkie/backend/db/mongo"
 	"github.com/yorkie-team/yorkie/yorkie/backend/sync"
 	"github.com/yorkie-team/yorkie/yorkie/backend/sync/etcd"
-	"github.com/yorkie-team/yorkie/yorkie/backend/sync/memory"
+	memsync "github.com/yorkie-team/yorkie/yorkie/backend/sync/memory"
 	"github.com/yorkie-team/yorkie/yorkie/profiling/prometheus"
 )
 
@@ -77,9 +78,17 @@ func New(
 		UpdatedAt: time.Now(),
 	}
 
-	mongoClient, err := mongo.Dial(mongoConf)
-	if err != nil {
-		return nil, err
+	var database db.DB
+	if mongoConf != nil {
+		database, err = mongo.Dial(mongoConf)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		database, err = memdb.New()
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	var coordinator sync.Coordinator
@@ -94,13 +103,19 @@ func New(
 
 		coordinator = etcdClient
 	} else {
-		coordinator = memory.NewCoordinator(agentInfo)
+		coordinator = memsync.NewCoordinator(agentInfo)
+	}
+
+	dbInfo := "memory"
+	if mongoConf != nil {
+		dbInfo = mongoConf.ConnectionURI
 	}
 
 	log.Logger.Infof(
-		"backend created: id: %s, rpc: %s",
+		"backend created: id: %s, rpc: %s: db: %s",
 		agentInfo.ID,
 		agentInfo.RPCAddr,
+		dbInfo,
 	)
 
 	lruCache, err := cache.NewLRUExpireCache(authWebhookCacheSize)
@@ -111,7 +126,7 @@ func New(
 	return &Backend{
 		Config:           conf,
 		agentInfo:        agentInfo,
-		DB:               mongoClient,
+		DB:               database,
 		Coordinator:      coordinator,
 		Metrics:          metrics,
 		AuthWebhookCache: lruCache,

yorkie/backend/db/change_info.go

@@ -29,6 +29,7 @@ import (
 
 // ChangeInfo is a structure representing information of a change.
 type ChangeInfo struct {
+	ID         ID       `bson:"_id"`
 	DocID      ID       `bson:"doc_id"`
 	ServerSeq  uint64   `bson:"server_seq"`
 	ClientSeq  uint32   `bson:"client_seq"`

yorkie/backend/db/client_info.go

@@ -146,6 +146,31 @@ func (i *ClientInfo) EnsureDocumentAttached(docID ID) error {
 	return nil
 }
 
+// DeepCopy returns a deep copy of this client info.
+func (i *ClientInfo) DeepCopy() *ClientInfo {
+	if i == nil {
+		return nil
+	}
+
+	documents := make(map[ID]*ClientDocInfo, len(i.Documents))
+	for k, v := range i.Documents {
+		documents[k] = &ClientDocInfo{
+			Status:    v.Status,
+			ServerSeq: v.ServerSeq,
+			ClientSeq: v.ClientSeq,
+		}
+	}
+
+	return &ClientInfo{
+		ID:        i.ID,
+		Key:       i.Key,
+		Status:    i.Status,
+		Documents: documents,
+		CreatedAt: i.CreatedAt,
+		UpdatedAt: i.UpdatedAt,
+	}
+}
+
 func (i *ClientInfo) hasDocument(docID ID) bool {
 	return i.Documents != nil && i.Documents[docID] != nil
 }

yorkie/backend/db/db.go

@@ -33,12 +33,12 @@ func (id ID) String() string {
 }
 
 // Bytes returns bytes of decoded hexadecimal string representation of this ID.
-func (id ID) Bytes() []byte {
+func (id ID) Bytes() ([]byte, error) {
 	decoded, err := hex.DecodeString(id.String())
 	if err != nil {
-		return nil
+		return nil, err
 	}
-	return decoded
+	return decoded, nil
 }
 
 // IDFromBytes returns ID represented by the encoded hexadecimal string from bytes.
@@ -74,17 +74,14 @@ type DB interface {
 		createDocIfNotExist bool,
 	) (*DocInfo, error)
 
-	// StoreChangeInfos stores the given changes then updates the given docInfo.
-	StoreChangeInfos(
+	// CreateChangeInfos stores the given changes then updates the given docInfo.
+	CreateChangeInfos(
 		ctx context.Context,
 		docInfo *DocInfo,
 		initialServerSeq uint64,
 		changes []*change.Change,
 	) error
 
-	// CreateSnapshotInfo stores the snapshot of the given document.
-	CreateSnapshotInfo(ctx context.Context, docID ID, doc *document.InternalDocument) error
-
 	// FindChangesBetweenServerSeqs returns the changes between two server sequences.
 	FindChangesBetweenServerSeqs(
 		ctx context.Context,
@@ -101,6 +98,12 @@ type DB interface {
 		to uint64,
 	) ([]*ChangeInfo, error)
 
+	// CreateSnapshotInfo stores the snapshot of the given document.
+	CreateSnapshotInfo(ctx context.Context, docID ID, doc *document.InternalDocument) error
+
+	// FindLastSnapshotInfo finds the last snapshot of the given document.
+	FindLastSnapshotInfo(ctx context.Context, docID ID) (*SnapshotInfo, error)
+
 	// UpdateAndFindMinSyncedTicket updates the given serverSeq of the given client
 	// and returns the min synced ticket.
 	UpdateAndFindMinSyncedTicket(
@@ -109,7 +112,4 @@ type DB interface {
 		docID ID,
 		serverSeq uint64,
 	) (*time.Ticket, error)
-
-	// FindLastSnapshotInfo finds the last snapshot of the given document.
-	FindLastSnapshotInfo(ctx context.Context, docID ID) (*SnapshotInfo, error)
 }

yorkie/backend/db/db_test.go

@@ -18,6 +18,8 @@ func TestID(t *testing.T) {
 	t.Run("get ID from bytes test", func(t *testing.T) {
 		bytes := make([]byte, 12)
 		ID := db.IDFromBytes(bytes)
-		assert.Equal(t, bytes, ID.Bytes())
+		bytesID, err := ID.Bytes()
+		assert.NoError(t, err)
+		assert.Equal(t, bytes, bytesID)
 	})
 }

yorkie/backend/db/doc_info.go

@@ -48,3 +48,20 @@ func (info *DocInfo) GetKey() (*key.Key, error) {
 
 	return docKey, nil
 }
+
+// DeepCopy creates a deep copy of this DocInfo.
+func (info *DocInfo) DeepCopy() *DocInfo {
+	if info == nil {
+		return nil
+	}
+
+	return &DocInfo{
+		ID:         info.ID,
+		Key:        info.Key,
+		ServerSeq:  info.ServerSeq,
+		Owner:      info.Owner,
+		CreatedAt:  info.CreatedAt,
+		AccessedAt: info.AccessedAt,
+		UpdatedAt:  info.UpdatedAt,
+	}
+}