Merge staging (#1365)

* wait for repair and increase numBlocks (#1338) * reorder wait group done (#1340) * hotfix / remove hard coded prefix handling of encrypted upload (#1344) * remove hard coded prefix handling of encrypted upload * fix for other parts of the file * increase batch size (#1349) * Sprint changes (#1355) * Update sprint 1.12 (#1341) * wait for repair and increase numBlocks (#1338) * reorder wait group done (#1340) --------- Co-authored-by: Ebrahim Gomaa <ebrahim.gomaa.hg@gmail.com> * fix trailing whitespace (#1343) * Merge staging changes (#1346) * wait for repair and increase numBlocks (#1338) * reorder wait group done (#1340) * hotfix / remove hard coded prefix handling of encrypted upload (#1344) * remove hard coded prefix handling of encrypted upload * fix for other parts of the file --------- Co-authored-by: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> Co-authored-by: Ebrahim Gomaa <ebrahim.gomaa.hg@gmail.com> * fix upload select (#1351) * fix workdir in mobile sdk (#1345) * fix mobile workdir * set multi op batch size * set option to download to disk in wasm (#1348) * fix panic in hash chan (#1352) * Fix merge conflict in sprint-1.12 (#1354) * wait for repair and increase numBlocks (#1338) * reorder wait group done (#1340) * hotfix / remove hard coded prefix handling of encrypted upload (#1344) * remove hard coded prefix handling of encrypted upload * fix for other parts of the file * increase batch size (#1349) --------- Co-authored-by: Ebrahim Gomaa <ebrahim.gomaa.hg@gmail.com> * repair in batches (#1347) * repair in batches * fix lint * fix unit test * fix batch size --------- Co-authored-by: Yury <yuderbasov@gmail.com> --------- Co-authored-by: Ebrahim Gomaa <ebrahim.gomaa.hg@gmail.com> Co-authored-by: peterlimg <54137706+peterlimg@users.noreply.github.com> Co-authored-by: Yury <yuderbasov@gmail.com> * fix repair after update (#1357) * fix err msg (#1361) * fix err msg * calc root hash once --------- Co-authored-by: Ebrahim Gomaa <ebrahim.gomaa.hg@gmail.com> Co-authored-by: peterlimg <54137706+peterlimg@users.noreply.github.com> Co-authored-by: Yury <yuderbasov@gmail.com>
0chain · Jan 12, 2024 · edf037a · edf037a
1 parent 4ed3f42
commit edf037a
Show file tree

Hide file tree

Showing 11 changed files with 20 additions and 22 deletions.
diff --git a/zboxcore/allocationchange/copyobject.go b/zboxcore/allocationchange/copyobject.go
@@ -76,7 +76,6 @@ func (ch *CopyFileChange) ProcessChange(rootRef *fileref.Ref, fileIDMeta map[str
 	ch.processChildren(affectedRef, fileIDMeta)
 	dirRef.AddChild(ch.ObjectTree)
 
-	rootRef.CalculateHash()
 	return
 }
 

diff --git a/zboxcore/allocationchange/createdir.go b/zboxcore/allocationchange/createdir.go
@@ -60,7 +60,6 @@ func (d *DirCreateChange) ProcessChange(rootRef *fileref.Ref, FileIDMeta map[str
 		}
 	}
 
-	rootRef.CalculateHash()
 	return
 }
 

diff --git a/zboxcore/allocationchange/deletefile.go b/zboxcore/allocationchange/deletefile.go
@@ -17,7 +17,6 @@ func (ch *DeleteFileChange) ProcessChange(rootRef *fileref.Ref, _ map[string]str
 
 	if ch.ObjectTree.GetPath() == "/" {
 		rootRef.Children = nil
-		rootRef.CalculateHash()
 		return
 	}
 
@@ -48,7 +47,6 @@ func (ch *DeleteFileChange) ProcessChange(rootRef *fileref.Ref, _ map[string]str
 	for i, child := range dirRef.Children {
 		if child.GetName() == ch.ObjectTree.GetName() {
 			dirRef.RemoveChild(i)
-			rootRef.CalculateHash()
 			return
 		}
 	}

diff --git a/zboxcore/allocationchange/moveobject.go b/zboxcore/allocationchange/moveobject.go
@@ -112,7 +112,6 @@ func (ch *MoveFileChange) ProcessChange(rootRef *fileref.Ref, fileIDMeta map[str
 
 	dirRef.AddChild(ch.ObjectTree)
 
-	rootRef.CalculateHash()
 	return
 }
 

diff --git a/zboxcore/allocationchange/newfile.go b/zboxcore/allocationchange/newfile.go
@@ -92,7 +92,6 @@ func (ch *NewFileChange) ProcessChange(rootRef *fileref.Ref, fileIDMeta map[stri
 	fileIDMeta[ch.File.GetPath()] = ch.File.FileID
 
 	dirRef.AddChild(ch.File)
-	rootRef.CalculateHash()
 	return
 }
 

diff --git a/zboxcore/allocationchange/renameobject.go b/zboxcore/allocationchange/renameobject.go
@@ -64,7 +64,6 @@ func (ch *RenameFileChange) ProcessChange(rootRef *fileref.Ref, _ map[string]str
 		return
 	}
 	ch.processChildren(affectedRef)
-	rootRef.CalculateHash()
 	return
 }
 

diff --git a/zboxcore/allocationchange/updatefile.go b/zboxcore/allocationchange/updatefile.go
@@ -86,7 +86,6 @@ func (ch *UpdateFileChange) ProcessChange(rootRef *fileref.Ref, _ map[string]str
 	ch.NewFile.HashToBeComputed = true
 	ch.NewFile.FileID = ch.OldFile.FileID
 	dirRef.Children[idx] = ch.NewFile
-	rootRef.CalculateHash()
 	return
 }
 

diff --git a/zboxcore/sdk/commitworker.go b/zboxcore/sdk/commitworker.go
@@ -181,6 +181,7 @@ func (commitreq *CommitRequest) processCommit() {
 		}
 		size += change.GetSize()
 	}
+	rootRef.CalculateHash()
 	err = commitreq.commitBlobber(rootRef, lR.LatestWM, size, fileIDMeta)
 	if err != nil {
 		commitreq.result = ErrorCommitResult(err.Error())

diff --git a/zboxcore/sdk/downloadworker.go b/zboxcore/sdk/downloadworker.go
@@ -285,7 +285,7 @@ func (req *DownloadRequest) fillShards(shards [][][]byte, result *downloadBlock)
 			data = result.BlockChunks[i]
 		}
 		if i >= len(shards) || len(shards[i]) <= result.idx {
-			l.Logger.Error("Invalid shard index", result.idx, len(shards), len(shards[i]))
+			l.Logger.Error("Invalid shard index", result.idx, len(shards))
 			return errors.New("invalid_shard_index", fmt.Sprintf("Invalid shard index %d shard len: %d shard block len: %d", result.idx, len(shards), i))
 		}
 		shards[i][result.idx] = data

diff --git a/zboxcore/sdk/listworker.go b/zboxcore/sdk/listworker.go
@@ -242,7 +242,10 @@ func (req *ListRequest) GetListFromBlobbers() (*ListResult, error) {
 	if req.forRepair {
 		for _, child := range childResultMap {
 			if child.consensus < child.fullconsensus {
-				result.Children = append(result.Children, child)
+				if _, ok := selected[child.LookupHash]; !ok {
+					result.Children = append(result.Children, child)
+					selected[child.LookupHash] = child
+				}
 			}
 		}
 	}

diff --git a/zboxcore/sdk/repairworker.go b/zboxcore/sdk/repairworker.go
@@ -128,9 +128,9 @@ func (r *RepairRequest) iterateDir(a *Allocation, dir *ListResult) []OperationRe
 		}
 	case fileref.FILE:
 		// this returns op object and mask
-		op := r.repairFile(a, dir)
-		if op != nil {
-			ops = append(ops, *op)
+		repairOps := r.repairFile(a, dir)
+		if repairOps != nil {
+			ops = append(ops, repairOps...)
 		}
 
 	default:
@@ -139,8 +139,8 @@ func (r *RepairRequest) iterateDir(a *Allocation, dir *ListResult) []OperationRe
 	return ops
 }
 
-func (r *RepairRequest) repairFile(a *Allocation, file *ListResult) *OperationRequest {
-	var op *OperationRequest
+func (r *RepairRequest) repairFile(a *Allocation, file *ListResult) []OperationRequest {
+	ops := make([]OperationRequest, 0)
 	if r.checkForCancel(a) {
 		return nil
 	}
@@ -162,16 +162,16 @@ func (r *RepairRequest) repairFile(a *Allocation, file *ListResult) *OperationRe
 
 			if deleteMask.CountOnes() > 0 {
 				l.Logger.Info("Deleting minority shards for the path :", zap.Any("path", file.Path))
-				// consensus := deleteMask.CountOnes()
-				// err := a.deleteFile(file.Path, 0, consensus, deleteMask)
-				op = &OperationRequest{
+				op := OperationRequest{
 					OperationType: constants.FileOperationDelete,
 					RemotePath:    file.Path,
 					Mask:          &deleteMask,
 				}
+				ops = append(ops, op)
 			}
 			wg.Add(1)
 			localPath := r.getLocalPath(file)
+			var op *OperationRequest
 			if !checkFileExists(localPath) {
 				if r.checkForCancel(a) {
 					return nil
@@ -190,27 +190,29 @@ func (r *RepairRequest) repairFile(a *Allocation, file *ListResult) *OperationRe
 				}
 				op = a.RepairFile(f, file.Path, statusCB, found, ref)
 			}
-
+			ops = append(ops, *op)
 			if r.checkForCancel(a) {
 				return nil
 			}
 		} else {
 			l.Logger.Info("Repair by delete", zap.Any("path", file.Path))
-			op = &OperationRequest{
+			op := OperationRequest{
 				OperationType: constants.FileOperationDelete,
 				RemotePath:    file.Path,
 				Mask:          &found,
 			}
+			ops = append(ops, op)
 		}
 	} else if deleteMask.CountOnes() > 0 {
 		l.Logger.Info("Deleting minority shards for the path :", zap.Any("path", file.Path))
-		op = &OperationRequest{
+		op := OperationRequest{
 			OperationType: constants.FileOperationDelete,
 			RemotePath:    file.Path,
 			Mask:          &deleteMask,
 		}
+		ops = append(ops, op)
 	}
-	return op
+	return ops
 }
 
 func (r *RepairRequest) repairOperation(a *Allocation, ops []OperationRequest) {
-Original file line number
+Diff line change
@@ Expand Up @@
     	ch.processChildren(affectedRef, fileIDMeta)
     	dirRef.AddChild(ch.ObjectTree)
-    	rootRef.CalculateHash()
     	return
     }
@@ Expand Down @@