Skip to content

Commit

Permalink
FAB-3047 Change attachment arrays to use pointers
Browse files Browse the repository at this point in the history 
This is change 3 of 4 for FAB-2725 CouchDB optimizations

Motivation for this change:
Interactions with CouchDB are currently done individually.
Need to switch to using bulk operations to get optimal performance
from CouchDB. Need to performance test and stress test.

- Change CouchDoc to use pointers for attachments

Change-Id: I8dcbb29432073514c20033e4cdc1204ecbabc0ef
Signed-off-by: Chris Elder <chris.elder@us.ibm.com>
  • Loading branch information
Chris Elder committed Apr 10, 2017
1 parent a932b54 commit 7984725
Show file tree
Hide file tree
Showing 3 changed files with 61 additions and 56 deletions.
8 changes: 5 additions & 3 deletions core/ledger/kvledger/txmgmt/statedb/statecouchdb/statecouchdb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ func (vdb *VersionedDB) GetState(namespace string, key string) (*statedb.Version
return &statedb.VersionedValue{Value: returnValue, Version: &returnVersion}, nil
}

func removeDataWrapper(wrappedValue []byte, attachments []couchdb.Attachment) ([]byte, version.Height) {
func removeDataWrapper(wrappedValue []byte, attachments []*couchdb.Attachment) ([]byte, version.Height) {

//initialize the return value
returnValue := []byte{} // TODO: empty byte or nil
Expand Down Expand Up @@ -264,12 +264,14 @@ func (vdb *VersionedDB) ApplyUpdates(batch *statedb.UpdateBatch, height *version
// Handle it as json
couchDoc.JSONValue = addVersionAndChainCodeID(vv.Value, ns, vv.Version)
} else { // if the data is not JSON, save as binary attachment in Couch
//Create an attachment structure and load the bytes

attachment := &couchdb.Attachment{}
attachment.AttachmentBytes = vv.Value
attachment.ContentType = "application/octet-stream"
attachment.Name = binaryWrapper
couchDoc.Attachments = append(couchDoc.Attachments, *attachment)
attachments := append([]*couchdb.Attachment{}, attachment)

couchDoc.Attachments = attachments
couchDoc.JSONValue = addVersionAndChainCodeID(nil, ns, vv.Version)
}

Expand Down
13 changes: 8 additions & 5 deletions core/ledger/util/couchdb/couchdb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ type DocID struct {
type QueryResult struct {
ID string
Value []byte
Attachments []Attachment
Attachments []*Attachment
}

//CouchConnectionDef contains parameters
Expand Down Expand Up @@ -175,7 +175,7 @@ type FileDetails struct {
//CouchDoc defines the structure for a JSON document value
type CouchDoc struct {
JSONValue []byte
Attachments []Attachment
Attachments []*Attachment
}

//BatchRetrieveDocMedatadataResponse is used for processing REST batch responses from CouchDB
Expand Down Expand Up @@ -597,6 +597,7 @@ func getRevisionHeader(resp *http.Response) (string, error) {
//ReadDoc method provides function to retrieve a document from the database by id
func (dbclient *CouchDatabase) ReadDoc(id string) (*CouchDoc, string, error) {
var couchDoc CouchDoc
attachments := []*Attachment{}

logger.Debugf("Entering ReadDoc() id=[%s]", id)
if !utf8.ValidString(id) {
Expand Down Expand Up @@ -668,7 +669,7 @@ func (dbclient *CouchDatabase) ReadDoc(id string) (*CouchDoc, string, error) {
default:

//Create an attachment structure and load it
attachment := Attachment{}
attachment := &Attachment{}
attachment.ContentType = p.Header.Get("Content-Type")
contentDispositionParts := strings.Split(p.Header.Get("Content-Disposition"), ";")
if strings.TrimSpace(contentDispositionParts[0]) == "attachment" {
Expand All @@ -689,7 +690,7 @@ func (dbclient *CouchDatabase) ReadDoc(id string) (*CouchDoc, string, error) {
logger.Debugf("Retrieved attachment data")
attachment.AttachmentBytes = respBody
attachment.Name = p.FileName()
couchDoc.Attachments = append(couchDoc.Attachments, attachment)
attachments = append(attachments, attachment)

default:

Expand All @@ -701,13 +702,15 @@ func (dbclient *CouchDatabase) ReadDoc(id string) (*CouchDoc, string, error) {
logger.Debugf("Retrieved attachment data")
attachment.AttachmentBytes = partdata
attachment.Name = p.FileName()
couchDoc.Attachments = append(couchDoc.Attachments, attachment)
attachments = append(attachments, attachment)

} // end content-encoding switch
} // end if attachment
} // end content-type switch
} // for all multiparts

couchDoc.Attachments = attachments

return &couchDoc, revision, nil
}

Expand Down
96 changes: 48 additions & 48 deletions core/ledger/util/couchdb/couchdb_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ func TestDBBadConnectionDef(t *testing.T) {

func TestDBCreateSaveWithoutRevision(t *testing.T) {

if ledgerconfig.IsCouchDBEnabled() == true {
if ledgerconfig.IsCouchDBEnabled() {

database := "testdbcreatesavewithoutrevision"
err := cleanup(database)
Expand All @@ -117,7 +117,7 @@ func TestDBCreateSaveWithoutRevision(t *testing.T) {

func TestDBCreateEnsureFullCommit(t *testing.T) {

if ledgerconfig.IsCouchDBEnabled() == true {
if ledgerconfig.IsCouchDBEnabled() {

database := "testdbensurefullcommit"
err := cleanup(database)
Expand Down Expand Up @@ -148,7 +148,7 @@ func TestDBCreateEnsureFullCommit(t *testing.T) {

func TestDBBadDatabaseName(t *testing.T) {

if ledgerconfig.IsCouchDBEnabled() == true {
if ledgerconfig.IsCouchDBEnabled() {

//create a new instance and database object using a valid database name mixed case
couchInstance, err := CreateCouchInstance(connectURL, username, password)
Expand Down Expand Up @@ -187,7 +187,7 @@ func TestDBBadConnection(t *testing.T) {
// TODO Re-enable once configurable retry logic is introduced
t.Skip()

if ledgerconfig.IsCouchDBEnabled() == true {
if ledgerconfig.IsCouchDBEnabled() {

//create a new instance and database object
_, err := CreateCouchInstance(badConnectURL, username, password)
Expand All @@ -197,7 +197,7 @@ func TestDBBadConnection(t *testing.T) {

func TestDBCreateDatabaseAndPersist(t *testing.T) {

if ledgerconfig.IsCouchDBEnabled() == true {
if ledgerconfig.IsCouchDBEnabled() {

database := "testdbcreatedatabaseandpersist"
err := cleanup(database)
Expand Down Expand Up @@ -286,7 +286,7 @@ func TestDBCreateDatabaseAndPersist(t *testing.T) {

func TestDBBadJSON(t *testing.T) {

if ledgerconfig.IsCouchDBEnabled() == true {
if ledgerconfig.IsCouchDBEnabled() {

database := "testdbbadjson"
err := cleanup(database)
Expand Down Expand Up @@ -322,7 +322,7 @@ func TestDBBadJSON(t *testing.T) {
}

func TestPrefixScan(t *testing.T) {
if !ledgerconfig.IsCouchDBEnabled() == true {
if !ledgerconfig.IsCouchDBEnabled() {
return
}
database := "testprefixscan"
Expand Down Expand Up @@ -384,7 +384,7 @@ func TestPrefixScan(t *testing.T) {

func TestDBSaveAttachment(t *testing.T) {

if ledgerconfig.IsCouchDBEnabled() == true {
if ledgerconfig.IsCouchDBEnabled() {

database := "testdbsaveattachment"
err := cleanup(database)
Expand All @@ -395,12 +395,12 @@ func TestDBSaveAttachment(t *testing.T) {

byteText := []byte(`This is a test document. This is only a test`)

attachment := Attachment{}
attachment := &Attachment{}
attachment.AttachmentBytes = byteText
attachment.ContentType = "text/plain"
attachment.Name = "valueBytes"

attachments := []Attachment{}
attachments := []*Attachment{}
attachments = append(attachments, attachment)

//create a new instance and database object
Expand Down Expand Up @@ -428,7 +428,7 @@ func TestDBSaveAttachment(t *testing.T) {

func TestDBDeleteDocument(t *testing.T) {

if ledgerconfig.IsCouchDBEnabled() == true {
if ledgerconfig.IsCouchDBEnabled() {

database := "testdbdeletedocument"
err := cleanup(database)
Expand Down Expand Up @@ -466,7 +466,7 @@ func TestDBDeleteDocument(t *testing.T) {

func TestDBDeleteNonExistingDocument(t *testing.T) {

if ledgerconfig.IsCouchDBEnabled() == true {
if ledgerconfig.IsCouchDBEnabled() {

database := "testdbdeletenonexistingdocument"
err := cleanup(database)
Expand Down Expand Up @@ -508,7 +508,7 @@ func TestCouchDBVersion(t *testing.T) {

func TestRichQuery(t *testing.T) {

if ledgerconfig.IsCouchDBEnabled() == true {
if ledgerconfig.IsCouchDBEnabled() {

byteJSON01 := []byte(`{"asset_name":"marble01","color":"blue","size":1,"owner":"jerry"}`)
byteJSON02 := []byte(`{"asset_name":"marble02","color":"red","size":2,"owner":"tom"}`)
Expand All @@ -527,85 +527,85 @@ func TestRichQuery(t *testing.T) {
attachment1.AttachmentBytes = []byte(`marble01 - test attachment`)
attachment1.ContentType = "application/octet-stream"
attachment1.Name = "data"
attachments1 := []Attachment{}
attachments1 = append(attachments1, *attachment1)
attachments1 := []*Attachment{}
attachments1 = append(attachments1, attachment1)

attachment2 := &Attachment{}
attachment2.AttachmentBytes = []byte(`marble02 - test attachment`)
attachment2.ContentType = "application/octet-stream"
attachment2.Name = "data"
attachments2 := []Attachment{}
attachments2 = append(attachments2, *attachment2)
attachments2 := []*Attachment{}
attachments2 = append(attachments2, attachment2)

attachment3 := &Attachment{}
attachment3.AttachmentBytes = []byte(`marble03 - test attachment`)
attachment3.ContentType = "application/octet-stream"
attachment3.Name = "data"
attachments3 := []Attachment{}
attachments3 = append(attachments3, *attachment3)
attachments3 := []*Attachment{}
attachments3 = append(attachments3, attachment3)

attachment4 := &Attachment{}
attachment4.AttachmentBytes = []byte(`marble04 - test attachment`)
attachment4.ContentType = "application/octet-stream"
attachment4.Name = "data"
attachments4 := []Attachment{}
attachments4 = append(attachments4, *attachment4)
attachments4 := []*Attachment{}
attachments4 = append(attachments4, attachment4)

attachment5 := &Attachment{}
attachment5.AttachmentBytes = []byte(`marble05 - test attachment`)
attachment5.ContentType = "application/octet-stream"
attachment5.Name = "data"
attachments5 := []Attachment{}
attachments5 = append(attachments5, *attachment5)
attachments5 := []*Attachment{}
attachments5 = append(attachments5, attachment5)

attachment6 := &Attachment{}
attachment6.AttachmentBytes = []byte(`marble06 - test attachment`)
attachment6.ContentType = "application/octet-stream"
attachment6.Name = "data"
attachments6 := []Attachment{}
attachments6 = append(attachments6, *attachment6)
attachments6 := []*Attachment{}
attachments6 = append(attachments6, attachment6)

attachment7 := &Attachment{}
attachment7.AttachmentBytes = []byte(`marble07 - test attachment`)
attachment7.ContentType = "application/octet-stream"
attachment7.Name = "data"
attachments7 := []Attachment{}
attachments7 = append(attachments7, *attachment7)
attachments7 := []*Attachment{}
attachments7 = append(attachments7, attachment7)

attachment8 := &Attachment{}
attachment8.AttachmentBytes = []byte(`marble08 - test attachment`)
attachment8.ContentType = "application/octet-stream"
attachment7.Name = "data"
attachments8 := []Attachment{}
attachments8 = append(attachments8, *attachment8)
attachments8 := []*Attachment{}
attachments8 = append(attachments8, attachment8)

attachment9 := &Attachment{}
attachment9.AttachmentBytes = []byte(`marble09 - test attachment`)
attachment9.ContentType = "application/octet-stream"
attachment9.Name = "data"
attachments9 := []Attachment{}
attachments9 = append(attachments9, *attachment9)
attachments9 := []*Attachment{}
attachments9 = append(attachments9, attachment9)

attachment10 := &Attachment{}
attachment10.AttachmentBytes = []byte(`marble10 - test attachment`)
attachment10.ContentType = "application/octet-stream"
attachment10.Name = "data"
attachments10 := []Attachment{}
attachments10 = append(attachments10, *attachment10)
attachments10 := []*Attachment{}
attachments10 = append(attachments10, attachment10)

attachment11 := &Attachment{}
attachment11.AttachmentBytes = []byte(`marble11 - test attachment`)
attachment11.ContentType = "application/octet-stream"
attachment11.Name = "data"
attachments11 := []Attachment{}
attachments11 = append(attachments11, *attachment11)
attachments11 := []*Attachment{}
attachments11 = append(attachments11, attachment11)

attachment12 := &Attachment{}
attachment12.AttachmentBytes = []byte(`marble12 - test attachment`)
attachment12.ContentType = "application/octet-stream"
attachment12.Name = "data"
attachments12 := []Attachment{}
attachments12 = append(attachments12, *attachment12)
attachments12 := []*Attachment{}
attachments12 = append(attachments12, attachment12)

database := "testrichquery"
err := cleanup(database)
Expand Down Expand Up @@ -779,7 +779,7 @@ func TestRichQuery(t *testing.T) {

func TestBatchBatchOperations(t *testing.T) {

if ledgerconfig.IsCouchDBEnabled() == true {
if ledgerconfig.IsCouchDBEnabled() {

byteJSON01 := []byte(`{"_id":"marble01","asset_name":"marble01","color":"blue","size":"1","owner":"jerry"}`)
byteJSON02 := []byte(`{"_id":"marble02","asset_name":"marble02","color":"red","size":"2","owner":"tom"}`)
Expand All @@ -791,36 +791,36 @@ func TestBatchBatchOperations(t *testing.T) {
attachment1.AttachmentBytes = []byte(`marble01 - test attachment`)
attachment1.ContentType = "application/octet-stream"
attachment1.Name = "data"
attachments1 := []Attachment{}
attachments1 = append(attachments1, *attachment1)
attachments1 := []*Attachment{}
attachments1 = append(attachments1, attachment1)

attachment2 := &Attachment{}
attachment2.AttachmentBytes = []byte(`marble02 - test attachment`)
attachment2.ContentType = "application/octet-stream"
attachment2.Name = "data"
attachments2 := []Attachment{}
attachments2 = append(attachments2, *attachment2)
attachments2 := []*Attachment{}
attachments2 = append(attachments2, attachment2)

attachment3 := &Attachment{}
attachment3.AttachmentBytes = []byte(`marble03 - test attachment`)
attachment3.ContentType = "application/octet-stream"
attachment3.Name = "data"
attachments3 := []Attachment{}
attachments3 = append(attachments3, *attachment3)
attachments3 := []*Attachment{}
attachments3 = append(attachments3, attachment3)

attachment4 := &Attachment{}
attachment4.AttachmentBytes = []byte(`marble04 - test attachment`)
attachment4.ContentType = "application/octet-stream"
attachment4.Name = "data"
attachments4 := []Attachment{}
attachments4 = append(attachments4, *attachment4)
attachments4 := []*Attachment{}
attachments4 = append(attachments4, attachment4)

attachment5 := &Attachment{}
attachment5.AttachmentBytes = []byte(`marble05 - test attachment`)
attachment5.ContentType = "application/octet-stream"
attachment5.Name = "data"
attachments5 := []Attachment{}
attachments5 = append(attachments5, *attachment5)
attachments5 := []*Attachment{}
attachments5 = append(attachments5, attachment5)

database := "testbatch"
err := cleanup(database)
Expand Down

0 comments on commit 7984725

Please sign in to comment.