From eba4dfa0da2e5853e937e666f74f19600e77fd75 Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Thu, 21 Nov 2024 01:09:58 +0800 Subject: [PATCH] :zap: Improve data sync performance for booting https://github.com/siyuan-note/siyuan/issues/13216 --- kernel/go.mod | 4 +- kernel/go.sum | 8 ++-- kernel/model/repository.go | 79 ++++++++++++++++++++++++++------------ 3 files changed, 60 insertions(+), 31 deletions(-) diff --git a/kernel/go.mod b/kernel/go.mod index 88273a63574..1f3cda0a29a 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -56,7 +56,7 @@ require ( github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 github.com/sashabaranov/go-openai v1.29.1 github.com/shirou/gopsutil/v3 v3.24.5 - github.com/siyuan-note/dejavu v0.0.0-20241120031619-9de3833dc8e6 + github.com/siyuan-note/dejavu v0.0.0-20241120170449-12bce3fc2a58 github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4 github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97 github.com/siyuan-note/filelock v0.0.0-20240724034355-d1ed7bf21d04 @@ -91,7 +91,7 @@ require ( github.com/andybalholm/cascadia v1.3.2 // indirect github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect github.com/aws/aws-sdk-go v1.55.5 // indirect - github.com/bytedance/sonic v1.12.2 // indirect + github.com/bytedance/sonic v1.12.4 // indirect github.com/bytedance/sonic/loader v0.2.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudflare/circl v1.5.0 // indirect diff --git a/kernel/go.sum b/kernel/go.sum index e1031db9477..17f311cee00 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -53,8 +53,8 @@ github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII= github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= -github.com/bytedance/sonic v1.12.2 h1:oaMFuRTpMHYLpCntGca65YWt5ny+wAceDERTkT2L9lg= -github.com/bytedance/sonic v1.12.2/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= +github.com/bytedance/sonic v1.12.4 h1:9Csb3c9ZJhfUWeMtpCDCq6BUoH5ogfDFLUgQ/jG+R0k= +github.com/bytedance/sonic v1.12.4/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= @@ -340,8 +340,8 @@ github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+D github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d h1:lvCTyBbr36+tqMccdGMwuEU+hjux/zL6xSmf5S9ITaA= github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8= -github.com/siyuan-note/dejavu v0.0.0-20241120031619-9de3833dc8e6 h1:zrYI7XDXI+W4VgohqCNPWft3kJ1zKyR933TYWQN5TOg= -github.com/siyuan-note/dejavu v0.0.0-20241120031619-9de3833dc8e6/go.mod h1:sVINGoilQS1l5ZQJJBHzUwKmyHhx8qdQps7gKqZVbgU= +github.com/siyuan-note/dejavu v0.0.0-20241120170449-12bce3fc2a58 h1:P7dolAfpWlMW5LTqY/4uG+g2q2WG12/+EhbZcIVBFiE= +github.com/siyuan-note/dejavu v0.0.0-20241120170449-12bce3fc2a58/go.mod h1:sVINGoilQS1l5ZQJJBHzUwKmyHhx8qdQps7gKqZVbgU= github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4 h1:kJaw5L/evyW6LcB9IQT8PR4ppx8JVqOFP9Ix3rfwSrc= github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4/go.mod h1:UYcCCY+0wh+GmUoDOaO63j1sV5lgy7laLAk1XhEiUis= github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97 h1:lM5v8BfNtbOL5jYwhCdMYBcYtr06IYBKjjSLAPMKTM8= diff --git a/kernel/model/repository.go b/kernel/model/repository.go index f170eff8996..e00a2ace71b 100644 --- a/kernel/model/repository.go +++ b/kernel/model/repository.go @@ -1285,34 +1285,63 @@ func bootSyncRepo() (err error) { isBootSyncing.Store(true) start := time.Now() - _, _, err = indexRepoBeforeCloudSync(repo) - if err != nil { - autoSyncErrCount++ - planSyncAfter(fixSyncInterval) - msg := fmt.Sprintf(Conf.Language(80), formatRepoErrorMsg(err)) - Conf.Sync.Stat = msg - Conf.Save() - util.PushStatusBar(msg) - util.PushErrMsg(msg, 0) - BootSyncSucc = 1 - isBootSyncing.Store(false) - return - } + waitGroup := sync.WaitGroup{} + var errs []error + waitGroup.Add(1) + go func() { + defer waitGroup.Done() + + _, _, indexErr := indexRepoBeforeCloudSync(repo) + if indexErr != nil { + errs = append(errs, indexErr) + autoSyncErrCount++ + planSyncAfter(fixSyncInterval) + + msg := fmt.Sprintf(Conf.Language(80), formatRepoErrorMsg(indexErr)) + Conf.Sync.Stat = msg + Conf.Save() + util.PushStatusBar(msg) + util.PushErrMsg(msg, 0) + BootSyncSucc = 1 + isBootSyncing.Store(false) + return + } + }() - syncContext := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar} - fetchedFiles, err := repo.GetSyncCloudFiles(syncContext) - if errors.Is(err, dejavu.ErrRepoFatal) { - autoSyncErrCount++ - planSyncAfter(fixSyncInterval) + var fetchedFiles []*entity.File + waitGroup.Add(1) + go func() { + defer waitGroup.Done() + + syncContext := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar} + cloudLatest, getErr := repo.GetCloudLatest(syncContext) + if nil != getErr { + errs = append(errs, getErr) + if !errors.Is(getErr, cloud.ErrCloudObjectNotFound) { + logging.LogErrorf("download cloud latest failed: %s", getErr) + return + } + } + fetchedFiles, getErr = repo.GetSyncCloudFiles(cloudLatest, syncContext) + if errors.Is(getErr, dejavu.ErrRepoFatal) { + errs = append(errs, getErr) + autoSyncErrCount++ + planSyncAfter(fixSyncInterval) - msg := fmt.Sprintf(Conf.Language(80), formatRepoErrorMsg(err)) - Conf.Sync.Stat = msg - Conf.Save() - util.PushStatusBar(msg) - util.PushErrMsg(msg, 0) - BootSyncSucc = 1 - isBootSyncing.Store(false) + msg := fmt.Sprintf(Conf.Language(80), formatRepoErrorMsg(getErr)) + Conf.Sync.Stat = msg + Conf.Save() + util.PushStatusBar(msg) + util.PushErrMsg(msg, 0) + BootSyncSucc = 1 + isBootSyncing.Store(false) + return + } + }() + waitGroup.Wait() + if 0 < len(errs) { + err = errs[0] return }