Skip to content

Commit 2dbe1b0

Browse files
zzzhr1990zzzhr1990
authored
fix(halalcloud_open): halal-cloud upload issues (#1800)
fix halal-cloud upload issues
1 parent 1fc9c83 commit 2dbe1b0

File tree

5 files changed

+48
-39
lines changed

5 files changed

+48
-39
lines changed

‎drivers/halalcloud_open/driver_curd_impl.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package halalcloudopen
22

33
import (
44
"context"
5-
"strconv"
65

76
"github.com/OpenListTeam/OpenList/v4/internal/model"
87
sdkModel "github.com/halalcloud/golang-sdk-lite/halalcloud/model"
@@ -19,7 +18,7 @@ func (d *HalalCloudOpen) getFiles(ctx context.Context, dir model.Obj) ([]model.O
1918
result, err := d.sdkUserFileService.List(ctx, &sdkUserFile.FileListRequest{
2019
Parent: &sdkUserFile.File{Path: dir.GetPath()},
2120
ListInfo: &sdkModel.ScanListRequest{
22-
Limit: strconv.FormatInt(limit, 10),
21+
Limit: limit,
2322
Token: token,
2423
},
2524
})

‎drivers/halalcloud_open/halalcloud_upload.go‎

Lines changed: 36 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"github.com/OpenListTeam/OpenList/v4/internal/model"
1717
sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile"
1818
"github.com/ipfs/go-cid"
19+
log "github.com/sirupsen/logrus"
1920
)
2021

2122
func (d *HalalCloudOpen) put(ctx context.Context, dstDir model.Obj, fileStream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) {
@@ -51,52 +52,39 @@ func (d *HalalCloudOpen) put(ctx context.Context, dstDir model.Obj, fileStream m
5152
Version: 1,
5253
}
5354
blockSize := uploadTask.BlockSize
54-
useSingleUpload := true
5555
//
56-
if fileStream.GetSize() <= int64(blockSize) || d.uploadThread <= 1 {
57-
useSingleUpload = true
58-
}
5956
// Not sure whether FileStream supports concurrent read and write operations, so currently using single-threaded upload to ensure safety.
6057
// read file
61-
if useSingleUpload {
62-
bufferSize := int(blockSize)
63-
buffer := make([]byte, bufferSize)
64-
reader := driver.NewLimitedUploadStream(ctx, fileStream)
65-
teeReader := io.TeeReader(reader, driver.NewProgress(fileStream.GetSize(), up))
66-
// fileStream.Seek(0, os.SEEK_SET)
67-
for {
68-
n, err := teeReader.Read(buffer)
69-
if n > 0 {
70-
data := buffer[:n]
71-
uploadCid, err := postFileSlice(ctx, data, uploadTask.Task, uploadTask.UploadAddress, prefix, retryTimes)
58+
bufferSize := int(blockSize)
59+
buffer := make([]byte, bufferSize)
60+
offset := 0
61+
teeReader := io.TeeReader(fileStream, driver.NewProgress(fileStream.GetSize(), up))
62+
for {
63+
n, err := teeReader.Read(buffer[offset:]) // 这里 len(buf[offset:]) <= 4MB
64+
if n > 0 {
65+
offset += n
66+
if offset == int(blockSize) {
67+
uploadCid, err := postFileSlice(ctx, buffer, uploadTask.Task, uploadTask.UploadAddress, prefix, retryTimes)
7268
if err != nil {
7369
return nil, err
7470
}
7571
slicesList = append(slicesList, uploadCid.String())
76-
}
77-
if err == io.EOF || n == 0 {
78-
break
72+
offset = 0
7973
}
8074
}
81-
} else {
82-
// TODO: implement multipart upload, currently using single-threaded upload to ensure safety.
83-
bufferSize := int(blockSize)
84-
buffer := make([]byte, bufferSize)
85-
reader := driver.NewLimitedUploadStream(ctx, fileStream)
86-
teeReader := io.TeeReader(reader, driver.NewProgress(fileStream.GetSize(), up))
87-
for {
88-
n, err := teeReader.Read(buffer)
89-
if n > 0 {
90-
data := buffer[:n]
91-
uploadCid, err := postFileSlice(ctx, data, uploadTask.Task, uploadTask.UploadAddress, prefix, retryTimes)
92-
if err != nil {
93-
return nil, err
75+
76+
if err != nil {
77+
if err == io.EOF {
78+
if offset > 0 {
79+
uploadCid, err := postFileSlice(ctx, buffer[:offset], uploadTask.Task, uploadTask.UploadAddress, prefix, retryTimes)
80+
if err != nil {
81+
return nil, err
82+
}
83+
slicesList = append(slicesList, uploadCid.String())
9484
}
95-
slicesList = append(slicesList, uploadCid.String())
96-
}
97-
if err == io.EOF || n == 0 {
9885
break
9986
}
87+
return nil, err
10088
}
10189
}
10290
newFile, err := makeFile(ctx, slicesList, uploadTask.Task, uploadTask.UploadAddress, retryTimes)
@@ -118,6 +106,7 @@ func makeFile(ctx context.Context, fileSlice []string, taskID string, uploadAddr
118106
if ctx.Err() != nil {
119107
return nil, err
120108
}
109+
log.Errorf("make file slice failed, retrying... error: %s", err.Error())
121110
if strings.Contains(err.Error(), "not found") {
122111
return nil, err
123112
}
@@ -156,15 +145,23 @@ func doMakeFile(fileSlice []string, taskID string, uploadAddress string) (*sdkUs
156145
if httpResponse.StatusCode != http.StatusOK && httpResponse.StatusCode != http.StatusCreated {
157146
b, _ := io.ReadAll(httpResponse.Body)
158147
message := string(b)
148+
log.Errorf("make file failed, status code: %d, message: %s", httpResponse.StatusCode, message)
149+
159150
return nil, fmt.Errorf("mk file slice failed, status code: %d, message: %s", httpResponse.StatusCode, message)
160151
}
161152
b, _ := io.ReadAll(httpResponse.Body)
162-
var result *sdkUserFile.File
153+
var result *UploadedFile
163154
err = json.Unmarshal(b, &result)
164155
if err != nil {
156+
log.Errorf("make file failed from response, status code: %d, message: %s", httpResponse.StatusCode, string(b))
165157
return nil, err
166158
}
167-
return result, nil
159+
return &sdkUserFile.File{
160+
Identity: result.Identity,
161+
Path: result.Path,
162+
Size: result.Size,
163+
ContentIdentity: result.ContentIdentity,
164+
}, nil
168165
}
169166
func postFileSlice(ctx context.Context, fileSlice []byte, taskID string, uploadAddress string, preix cid.Prefix, retry int) (cid.Cid, error) {
170167
var lastError error = nil
@@ -214,9 +211,11 @@ func doPostFileSlice(fileSlice []byte, taskID string, uploadAddress string, prei
214211
}
215212
httpResponse, err := httpClient.Do(&httpRequest)
216213
if err != nil {
214+
log.Errorf("access %s failed, method: %s", accessUrl, http.MethodGet)
217215
return cid.Undef, err
218216
}
219217
if httpResponse.StatusCode != http.StatusOK {
218+
log.Errorf("access %s failed, method: %s, status code: %d", accessUrl, http.MethodGet, httpResponse.StatusCode)
220219
return cid.Undef, fmt.Errorf("upload file slice failed, status code: %d", httpResponse.StatusCode)
221220
}
222221
var result bool
@@ -250,6 +249,7 @@ func doPostFileSlice(fileSlice []byte, taskID string, uploadAddress string, prei
250249
if httpResponse.StatusCode != http.StatusOK && httpResponse.StatusCode != http.StatusCreated {
251250
b, _ := io.ReadAll(httpResponse.Body)
252251
message := string(b)
252+
log.Errorf("upload file slice failed, status code: %d, message: %s", httpResponse.StatusCode, message)
253253
return cid.Undef, fmt.Errorf("upload file slice failed, status code: %d, message: %s", httpResponse.StatusCode, message)
254254
}
255255
//

‎drivers/halalcloud_open/meta.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,3 +30,11 @@ func init() {
3030
return &HalalCloudOpen{}
3131
})
3232
}
33+
34+
type UploadedFile struct {
35+
Identity string `json:"identity"`
36+
UserIdentity string `json:"user_identity"`
37+
Path string `json:"path"`
38+
Size int64 `json:"size"`
39+
ContentIdentity string `json:"content_identity"`
40+
}

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ require (
4242
github.com/golang-jwt/jwt/v4 v4.5.2
4343
github.com/google/uuid v1.6.0
4444
github.com/gorilla/websocket v1.5.3
45-
github.com/halalcloud/golang-sdk-lite v0.0.0-20251006164234-3c629727c499
45+
github.com/halalcloud/golang-sdk-lite v0.0.0-20251105081800-78cbb6786c38
4646
github.com/hekmon/transmissionrpc/v3 v3.0.0
4747
github.com/henrybear327/go-proton-api v1.0.0
4848
github.com/ipfs/go-ipfs-api v0.7.0

‎go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -400,6 +400,8 @@ github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aN
400400
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
401401
github.com/halalcloud/golang-sdk-lite v0.0.0-20251006164234-3c629727c499 h1:4ovnBdiGDFi8putQGxhipuuhXItAgh4/YnzufPYkZkQ=
402402
github.com/halalcloud/golang-sdk-lite v0.0.0-20251006164234-3c629727c499/go.mod h1:8x1h4rm3s8xMcTyJrq848sQ6BJnKzl57mDY4CNshdPM=
403+
github.com/halalcloud/golang-sdk-lite v0.0.0-20251105081800-78cbb6786c38 h1:lsK2GVgI2Ox0NkRpQnN09GBOH7jtsjFK5tcIgxXlLr0=
404+
github.com/halalcloud/golang-sdk-lite v0.0.0-20251105081800-78cbb6786c38/go.mod h1:8x1h4rm3s8xMcTyJrq848sQ6BJnKzl57mDY4CNshdPM=
403405
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
404406
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
405407
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=

0 commit comments

Comments
 (0)