0ct0pu5/siyuan
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

⚡ Improve data sync performance for booting https://github.com/siyuan-note/siyuan/issues/13216

Browse source
This commit is contained in:
Daniel 2024-11-21 01:09:58 +08:00
parent 0e357f7311
commit eba4dfa0da
No known key found for this signature in database
GPG key ID: 86211BA83DF03017
3 changed files with 60 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
kernel/go.mod
View file

@ -56,7 +56,7 @@ require (
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
github.com/sashabaranov/go-openai v1.29.1
github.com/shirou/gopsutil/v3 v3.24.5
github.com/siyuan-note/dejavu v0.0.0-20241120031619-9de3833dc8e6
github.com/siyuan-note/dejavu v0.0.0-20241120170449-12bce3fc2a58
github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4
github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97
github.com/siyuan-note/filelock v0.0.0-20240724034355-d1ed7bf21d04
@ -91,7 +91,7 @@ require (
github.com/andybalholm/cascadia v1.3.2 // indirect
github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect
github.com/aws/aws-sdk-go v1.55.5 // indirect
github.com/bytedance/sonic v1.12.2 // indirect
github.com/bytedance/sonic v1.12.4 // indirect
github.com/bytedance/sonic/loader v0.2.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cloudflare/circl v1.5.0 // indirect

8
kernel/go.sum
View file

@ -53,8 +53,8 @@ github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP
github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII=
github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/bytedance/sonic v1.12.2 h1:oaMFuRTpMHYLpCntGca65YWt5ny+wAceDERTkT2L9lg=
github.com/bytedance/sonic v1.12.2/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk=
github.com/bytedance/sonic v1.12.4 h1:9Csb3c9ZJhfUWeMtpCDCq6BUoH5ogfDFLUgQ/jG+R0k=
github.com/bytedance/sonic v1.12.4/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk=
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM=
github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
@ -340,8 +340,8 @@ github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+D
github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d h1:lvCTyBbr36+tqMccdGMwuEU+hjux/zL6xSmf5S9ITaA=
github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8=
github.com/siyuan-note/dejavu v0.0.0-20241120031619-9de3833dc8e6 h1:zrYI7XDXI+W4VgohqCNPWft3kJ1zKyR933TYWQN5TOg=
github.com/siyuan-note/dejavu v0.0.0-20241120031619-9de3833dc8e6/go.mod h1:sVINGoilQS1l5ZQJJBHzUwKmyHhx8qdQps7gKqZVbgU=
github.com/siyuan-note/dejavu v0.0.0-20241120170449-12bce3fc2a58 h1:P7dolAfpWlMW5LTqY/4uG+g2q2WG12/+EhbZcIVBFiE=
github.com/siyuan-note/dejavu v0.0.0-20241120170449-12bce3fc2a58/go.mod h1:sVINGoilQS1l5ZQJJBHzUwKmyHhx8qdQps7gKqZVbgU=
github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4 h1:kJaw5L/evyW6LcB9IQT8PR4ppx8JVqOFP9Ix3rfwSrc=
github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4/go.mod h1:UYcCCY+0wh+GmUoDOaO63j1sV5lgy7laLAk1XhEiUis=
github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97 h1:lM5v8BfNtbOL5jYwhCdMYBcYtr06IYBKjjSLAPMKTM8=

79
kernel/model/repository.go
View file

@ -1285,34 +1285,63 @@ func bootSyncRepo() (err error) {
isBootSyncing.Store(true)
start := time.Now()
_, _, err = indexRepoBeforeCloudSync(repo)
if err != nil {
autoSyncErrCount++
planSyncAfter(fixSyncInterval)
msg := fmt.Sprintf(Conf.Language(80), formatRepoErrorMsg(err))
Conf.Sync.Stat = msg
Conf.Save()
util.PushStatusBar(msg)
util.PushErrMsg(msg, 0)
BootSyncSucc = 1
isBootSyncing.Store(false)
return
}
waitGroup := sync.WaitGroup{}
var errs []error
waitGroup.Add(1)
go func() {
defer waitGroup.Done()
syncContext := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar}
fetchedFiles, err := repo.GetSyncCloudFiles(syncContext)
if errors.Is(err, dejavu.ErrRepoFatal) {
autoSyncErrCount++
planSyncAfter(fixSyncInterval)
_, _, indexErr := indexRepoBeforeCloudSync(repo)
if indexErr != nil {
errs = append(errs, indexErr)
autoSyncErrCount++
planSyncAfter(fixSyncInterval)
msg := fmt.Sprintf(Conf.Language(80), formatRepoErrorMsg(err))
Conf.Sync.Stat = msg
Conf.Save()
util.PushStatusBar(msg)
util.PushErrMsg(msg, 0)
BootSyncSucc = 1
isBootSyncing.Store(false)
msg := fmt.Sprintf(Conf.Language(80), formatRepoErrorMsg(indexErr))
Conf.Sync.Stat = msg
Conf.Save()
util.PushStatusBar(msg)
util.PushErrMsg(msg, 0)
BootSyncSucc = 1
isBootSyncing.Store(false)
return
}
}()
var fetchedFiles []*entity.File
waitGroup.Add(1)
go func() {
defer waitGroup.Done()
syncContext := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar}
cloudLatest, getErr := repo.GetCloudLatest(syncContext)
if nil != getErr {
errs = append(errs, getErr)
if !errors.Is(getErr, cloud.ErrCloudObjectNotFound) {
logging.LogErrorf("download cloud latest failed: %s", getErr)
return
}
}
fetchedFiles, getErr = repo.GetSyncCloudFiles(cloudLatest, syncContext)
if errors.Is(getErr, dejavu.ErrRepoFatal) {
errs = append(errs, getErr)
autoSyncErrCount++
planSyncAfter(fixSyncInterval)
msg := fmt.Sprintf(Conf.Language(80), formatRepoErrorMsg(getErr))
Conf.Sync.Stat = msg
Conf.Save()
util.PushStatusBar(msg)
util.PushErrMsg(msg, 0)
BootSyncSucc = 1
isBootSyncing.Store(false)
return
}
}()
waitGroup.Wait()
if 0 < len(errs) {
err = errs[0]
return
}