Skip to content

Commit

Permalink
Fix/update attributes (#717)
Browse files Browse the repository at this point in the history
* Remove whopays attribute

* Remove dot logger

* Fix hashes of test

* Remove redundant payer id field

* Remove authticket association with readmarker

* Fix conflicts

* Remove proto file

* Update go modules

* Generate/Update grpc code

* Go mod tidy

* Remove payer in test insertion

* Make reader pay for read instead of owner

* Go mod tidy

Co-authored-by: peterlimg <54137706+peterlimg@users.noreply.github.com>
Co-authored-by: peterlimg <peterlimg@protonmail.com>
Co-authored-by: Lz <imlangzi@qq.com>
  • Loading branch information
4 people authored Jul 4, 2022
1 parent 6b072d0 commit 28ea8c0
Show file tree
Hide file tree
Showing 45 changed files with 568 additions and 1,966 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -16,3 +16,4 @@ dev.local/data
out/
**/tmp/
*.tar.gz
**/descriptor.proto
2 changes: 0 additions & 2 deletions code/go/0chain.net/blobbercore/allocation/allocationchange.go
Original file line number Diff line number Diff line change
Expand Up @@ -148,8 +148,6 @@ func (cc *AllocationChangeCollector) ComputeProperties() {
acp = new(RenameFileChange)
case constants.FileOperationCopy:
acp = new(CopyFileChange)
case constants.FileOperationUpdateAttrs:
acp = new(AttributesChange)
}

if acp == nil {
Expand Down
116 changes: 0 additions & 116 deletions code/go/0chain.net/blobbercore/allocation/attributesfilechange.go
Original file line number Diff line number Diff line change
@@ -1,117 +1 @@
package allocation

import (
"context"
"encoding/json"
"path/filepath"

"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats"
"github.com/0chain/blobber/code/go/0chain.net/core/common"

. "github.com/0chain/blobber/code/go/0chain.net/core/logging"
"go.uber.org/zap"
)

// The AttributesChange represents file attributes change.
type AttributesChange struct {
ConnectionID string `json:"connection_id"`
AllocationID string `json:"allocation_id"`
Path string `json:"path"`
Attributes *reference.Attributes `json:"attributes"` // new attributes
}

// ApplyChange processes the attributes changes.
func (ac *AttributesChange) ApplyChange(ctx context.Context, _ *AllocationChange, allocRoot string) (ref *reference.Ref, err error) {
var path, _ = filepath.Split(ac.Path)
path = filepath.Clean(path)

// root reference
ref, err = reference.GetReferencePath(ctx, ac.AllocationID, ac.Path)

if err != nil {
return nil, common.NewErrorf("process_attrs_update",
"getting root reference path: %v", err)
}

var (
tSubDirs = reference.GetSubDirsFromPath(path)
dirRef = ref
treelevel = 0
)
dirRef.HashToBeComputed = true
for treelevel < len(tSubDirs) {
var found bool
for _, child := range dirRef.Children {
if child.Type == reference.DIRECTORY && treelevel < len(tSubDirs) {
if child.Name == tSubDirs[treelevel] {
dirRef, found = child, true
dirRef.HashToBeComputed = true
break
}
}
}
if found {
treelevel++
} else {
return nil, common.NewError("process_attrs_update",
"invalid reference path from the blobber")
}
}

var idx = -1
for i, child := range dirRef.Children {
if child.Type == reference.FILE && child.Path == ac.Path {
idx = i
break
}
}

if idx < 0 {
Logger.Error("error in file attributes update", zap.Any("change", ac))
return nil, common.NewError("process_attrs_update",
"file to update not found in blobber")
}

var existingRef = dirRef.Children[idx]
existingRef.WriteMarker = allocRoot
if err = existingRef.SetAttributes(ac.Attributes); err != nil {
return nil, common.NewErrorf("process_attrs_update",
"setting new attributes: %v", err)
}

existingRef.HashToBeComputed = true

if _, err := ref.CalculateHash(ctx, true); err != nil {
return nil, common.NewErrorf("process_attrs_update",
"saving updated reference: %v", err)
}

stats.FileUpdated(ctx, existingRef.ID)
return
}

// Marshal to JSON-string.
func (ac *AttributesChange) Marshal() (val string, err error) {
var b []byte
if b, err = json.Marshal(ac); err != nil {
return
}
return string(b), nil
}

// Unmarshal from given JSON-string.
func (ac *AttributesChange) Unmarshal(val string) (err error) {
err = json.Unmarshal([]byte(val), ac)
return
}

// The DeleteTempFile returns OperationNotApplicable error.
func (ac *AttributesChange) DeleteTempFile() (err error) {
return nil
}

// The CommitToFileStore does nothing.
func (ac *AttributesChange) CommitToFileStore(_ context.Context) (err error) {
return
}
7 changes: 2 additions & 5 deletions code/go/0chain.net/blobbercore/allocation/copyfilechange.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,13 @@ package allocation
import (
"context"
"encoding/json"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
"path/filepath"

"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"

"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats"
"github.com/0chain/blobber/code/go/0chain.net/core/common"

"gorm.io/datatypes"
)

type CopyFileChange struct {
Expand Down Expand Up @@ -133,7 +132,6 @@ func (rf *CopyFileChange) processCopyRefs(ctx context.Context, affectedRef, dest
newRef.ParentPath = destRef.Path
newRef.Name = affectedRef.Name
newRef.LookupHash = reference.GetReferenceLookup(newRef.AllocationID, newRef.Path)
newRef.Attributes = datatypes.JSON(string(affectedRef.Attributes))
newRef.HashToBeComputed = true
destRef.AddChild(newRef)

Expand All @@ -160,7 +158,6 @@ func (rf *CopyFileChange) processCopyRefs(ctx context.Context, affectedRef, dest
newFile.ActualThumbnailHash = affectedRef.ActualThumbnailHash
newFile.ActualThumbnailSize = affectedRef.ActualThumbnailSize
newFile.EncryptedKey = affectedRef.EncryptedKey
newFile.Attributes = datatypes.JSON(string(affectedRef.Attributes))
newFile.ChunkSize = affectedRef.ChunkSize
newFile.HashToBeComputed = true
destRef.AddChild(newFile)
Expand Down
1 change: 0 additions & 1 deletion code/go/0chain.net/blobbercore/allocation/entity.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ type Allocation struct {
OwnerID string `gorm:"column:owner_id;size:64;not null"`
OwnerPublicKey string `gorm:"column:owner_public_key;size:512;not null"`
RepairerID string `gorm:"column:repairer_id;size:64;not null"`
PayerID string `gorm:"column:payer_id;size:64;not null"`
Expiration common.Timestamp `gorm:"column:expiration_date;not null"`
// AllocationRoot allcation_root of last write_marker
AllocationRoot string `gorm:"column:allocation_root;size:64;not null;default:''"`
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (

"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
)

Expand All @@ -29,7 +28,6 @@ type BaseFileChanger struct {
//client side:
MimeType string `json:"mimetype,omitempty"`
//client side:
Attributes reference.Attributes `json:"attributes,omitempty"`
//client side:
MerkleRoot string `json:"merkle_root,omitempty"`

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,10 +87,6 @@ func (nf *UpdateFileChanger) ApplyChange(ctx context.Context, change *Allocation
existingRef.EncryptedKey = nf.EncryptedKey
existingRef.ChunkSize = nf.ChunkSize

if err = existingRef.SetAttributes(&nf.Attributes); err != nil {
return nil, common.NewErrorf("process_update_file_change",
"setting file attributes: %v", err)
}
_, err = rootRef.CalculateHash(ctx, true)
stats.FileUpdated(ctx, existingRef.ID)
return rootRef, err
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@ package allocation
import (
"context"
"encoding/json"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
"path/filepath"
"strings"

"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"

"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/util"
Expand Down Expand Up @@ -96,10 +97,7 @@ func (nf *UploadFileChanger) ApplyChange(ctx context.Context, change *Allocation
newFile.EncryptedKey = nf.EncryptedKey
newFile.ChunkSize = nf.ChunkSize
newFile.HashToBeComputed = true
if err = newFile.SetAttributes(&nf.Attributes); err != nil {
return nil, common.NewErrorf("process_new_file_change",
"setting file attributes: %v", err)
}

dirRef.AddChild(newFile)
if _, err := rootRef.CalculateHash(ctx, true); err != nil {
return nil, err
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@ package allocation

import (
"context"
"testing"
"time"

"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
"github.com/0chain/gosdk/constants"
"github.com/stretchr/testify/assert"
"testing"
"time"

"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
"github.com/0chain/gosdk/core/zcncrypto"
Expand Down Expand Up @@ -100,7 +100,6 @@ func TestBlobberCore_FileChangerUpload(t *testing.T) {
Filename: "new",
Path: "/",
ActualSize: 2310,
Attributes: reference.Attributes{WhoPaysForReads: common.WhoPaysOwner},
AllocationID: tc.allocationID,
Hash: tc.hash,
Size: 2310,
Expand Down
10 changes: 2 additions & 8 deletions code/go/0chain.net/blobbercore/allocation/newfilechange.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@ package allocation
import (
"context"
"encoding/json"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
"path/filepath"
"strings"

"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"

"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats"
Expand All @@ -33,8 +34,6 @@ type NewFileChange struct {
//client side:
MimeType string `json:"mimetype,omitempty"`
//client side:
Attributes reference.Attributes `json:"attributes,omitempty"`
//client side:
MerkleRoot string `json:"merkle_root,omitempty"`

//server side: update them by ChangeProcessor
Expand Down Expand Up @@ -206,11 +205,6 @@ func (nf *NewFileChange) ApplyChange(ctx context.Context, change *AllocationChan
newFile.ChunkSize = nf.ChunkSize
newFile.HashToBeComputed = true

if err = newFile.SetAttributes(&nf.Attributes); err != nil {
return nil, common.NewErrorf("process_new_file_change",
"setting file attributes: %v", err)
}

dirRef.AddChild(newFile)

if _, err := rootRef.CalculateHash(ctx, true); err != nil {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,15 @@ package allocation
import (
"context"
"fmt"
"testing"
"time"

"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
"github.com/0chain/gosdk/constants"
"github.com/stretchr/testify/assert"
"testing"
"time"

"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
"github.com/0chain/blobber/code/go/0chain.net/core/logging"
"github.com/0chain/gosdk/core/zcncrypto"
Expand Down Expand Up @@ -117,7 +117,6 @@ func TestBlobberCore_NewFile(t *testing.T) {
Filename: "new",
Path: "/",
ActualSize: 2310,
Attributes: reference.Attributes{WhoPaysForReads: common.WhoPaysOwner},
AllocationID: tc.allocationID,
Hash: tc.hash,
Size: 2310,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -199,9 +199,9 @@ func TestBlobberCore_RenameFile(t *testing.T) {
"path": "/old_dir",
}},
)
query = `SELECT "id","allocation_id","type","name","path","parent_path","size","hash","path_hash","content_hash","merkle_root","actual_file_size","actual_file_hash","attributes","chunk_size","lookup_hash","thumbnail_hash","write_marker","level" FROM "reference_objects" WHERE ("reference_objects"."allocation_id" = $1 AND "reference_objects"."parent_path" = $2 OR ("reference_objects"."allocation_id" = $3 AND "reference_objects"."parent_path" = $4) OR (parent_path = $5 AND allocation_id = $6)) AND "reference_objects"."deleted_at" IS NULL ORDER BY path%!!(string=allocation id)!(string=)!(string=/)!(string=allocation id)!(string=/old_dir)(EXTRA string=allocation id)`
query = `SELECT "id","allocation_id","type","name","path","parent_path","size","hash","path_hash","content_hash","merkle_root","actual_file_size","actual_file_hash","chunk_size","lookup_hash","thumbnail_hash","write_marker","level" FROM "reference_objects" WHERE ("reference_objects"."allocation_id" = $1 AND "reference_objects"."parent_path" = $2 OR ("reference_objects"."allocation_id" = $3 AND "reference_objects"."parent_path" = $4) OR (parent_path = $5 AND allocation_id = $6)) AND "reference_objects"."deleted_at" IS NULL ORDER BY path%!!(string=allocation id)!(string=)!(string=/)!(string=allocation id)!(string=/old_dir)(EXTRA string=allocation id)`
mocket.Catcher.NewMock().OneTime().WithQuery(
`SELECT "id","allocation_id","type","name","path","parent_path","size","hash","path_hash","content_hash","merkle_root","actual_file_size","actual_file_hash","attributes","chunk_size","lookup_hash","thumbnail_hash","write_marker","level" FROM "reference_objects" WHERE`,
`SELECT "id","allocation_id","type","name","path","parent_path","size","hash","path_hash","content_hash","merkle_root","actual_file_size","actual_file_hash","chunk_size","lookup_hash","thumbnail_hash","write_marker","level" FROM "reference_objects" WHERE`,
).WithQuery(query).WithReply(
[]map[string]interface{}{{
"id": 1,
Expand Down Expand Up @@ -242,7 +242,7 @@ func TestBlobberCore_RenameFile(t *testing.T) {
"path": "old_file.pdf",
}},
)
query = `SELECT "id","allocation_id","type","name","path","parent_path","size","hash","path_hash","content_hash","merkle_root","actual_file_size","actual_file_hash","attributes","chunk_size","lookup_hash","thumbnail_hash","write_marker","level" FROM "reference_objects" WHERE ("reference_objects"."allocation_id" = $1 AND "reference_objects"."parent_path" = $2 OR ("reference_objects"."allocation_id" = $3 AND "reference_objects"."parent_path" = $4) OR (parent_path = $5 AND allocation_id = $6)) AND "reference_objects"."deleted_at" IS NULL ORDER BY path%!!(string=allocation id)!(string=)!(string=.)!(string=allocation id)!(string=old_file.pdf)(EXTRA string=allocation id)`
query = `SELECT "id","allocation_id","type","name","path","parent_path","size","hash","path_hash","content_hash","merkle_root","actual_file_size","actual_file_hash","chunk_size","lookup_hash","thumbnail_hash","write_marker","level" FROM "reference_objects" WHERE ("reference_objects"."allocation_id" = $1 AND "reference_objects"."parent_path" = $2 OR ("reference_objects"."allocation_id" = $3 AND "reference_objects"."parent_path" = $4) OR (parent_path = $5 AND allocation_id = $6)) AND "reference_objects"."deleted_at" IS NULL ORDER BY path%!!(string=allocation id)!(string=)!(string=.)!(string=allocation id)!(string=old_file.pdf)(EXTRA string=allocation id)`
mocket.Catcher.NewMock().OneTime().WithQuery(query).WithReply(
[]map[string]interface{}{{
"id": 1,
Expand All @@ -259,7 +259,7 @@ func TestBlobberCore_RenameFile(t *testing.T) {
"parent_path": "/",
}},
)
query = `SELECT "id","allocation_id","type","name","path","parent_path","size","hash","path_hash","content_hash","merkle_root","actual_file_size","actual_file_hash","attributes","chunk_size","lookup_hash","thumbnail_hash","write_marker","level" FROM "reference_objects" WHERE "id" = $1 AND "reference_objects"."deleted_at" IS NULL ORDER BY "reference_objects"."id" LIMIT 1%!(EXTRA int64=1)`
query = `SELECT "id","allocation_id","type","name","path","parent_path","size","hash","path_hash","content_hash","merkle_root","actual_file_size","actual_file_hash","chunk_size","lookup_hash","thumbnail_hash","write_marker","level" FROM "reference_objects" WHERE "id" = $1 AND "reference_objects"."deleted_at" IS NULL ORDER BY "reference_objects"."id" LIMIT 1%!(EXTRA int64=1)`
mocket.Catcher.NewMock().OneTime().WithQuery(query).
WithReply(
[]map[string]interface{}{{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ func TestBlobberCore_UpdateFile(t *testing.T) {
setupDbMock: func() {
mocket.Catcher.Reset()

query := `SELECT "id","allocation_id","type","name","path","parent_path","size","hash","path_hash","content_hash","merkle_root","actual_file_size","actual_file_hash","attributes","chunk_size","lookup_hash","thumbnail_hash","write_marker","level" FROM "reference_objects" WHERE`
query := `SELECT "id","allocation_id","type","name","path","parent_path","size","hash","path_hash","content_hash","merkle_root","actual_file_size","actual_file_hash","chunk_size","lookup_hash","thumbnail_hash","write_marker","level" FROM "reference_objects" WHERE`
mocket.Catcher.NewMock().WithQuery(query).WithReply(
[]map[string]interface{}{
{
Expand Down Expand Up @@ -118,7 +118,7 @@ func TestBlobberCore_UpdateFile(t *testing.T) {
setupDbMock: func() {
mocket.Catcher.Reset()

query := `SELECT "id","allocation_id","type","name","path","parent_path","size","hash","path_hash","content_hash","merkle_root","actual_file_size","actual_file_hash","attributes","chunk_size","lookup_hash","thumbnail_hash","write_marker","level" FROM "reference_objects" WHERE`
query := `SELECT "id","allocation_id","type","name","path","parent_path","size","hash","path_hash","content_hash","merkle_root","actual_file_size","actual_file_hash","chunk_size","lookup_hash","thumbnail_hash","write_marker","level" FROM "reference_objects" WHERE`
mocket.Catcher.NewMock().WithQuery(query).WithReply(
[]map[string]interface{}{
{
Expand Down Expand Up @@ -173,7 +173,7 @@ func TestBlobberCore_UpdateFile(t *testing.T) {
expectingError: false,
setupDbMock: func() {
mocket.Catcher.Reset()
query := `SELECT "id","allocation_id","type","name","path","parent_path","size","hash","path_hash","content_hash","merkle_root","actual_file_size","actual_file_hash","attributes","chunk_size","lookup_hash","thumbnail_hash","write_marker","level" FROM "reference_objects" WHERE`
query := `SELECT "id","allocation_id","type","name","path","parent_path","size","hash","path_hash","content_hash","merkle_root","actual_file_size","actual_file_hash","chunk_size","lookup_hash","thumbnail_hash","write_marker","level" FROM "reference_objects" WHERE`
mocket.Catcher.NewMock().WithQuery(query).WithReply(
[]map[string]interface{}{
{
Expand Down Expand Up @@ -238,7 +238,6 @@ func TestBlobberCore_UpdateFile(t *testing.T) {
ActualSize: 2310,
ActualThumbnailSize: 92,
ActualThumbnailHash: tc.thumbnailHash,
Attributes: reference.Attributes{WhoPaysForReads: common.WhoPaysOwner},
AllocationID: tc.allocationID,
Hash: tc.hash,
Size: 2310,
Expand Down
Loading

0 comments on commit 28ea8c0

Please sign in to comment.