0ct0pu5/siyuan
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

🎨 云端同步数据在启动后执行 https://github.com/siyuan-note/siyuan/issues/6290

Browse source
This commit is contained in:
Liang Ding 2022-10-22 11:55:12 +08:00
parent b19613ea94
commit 24d41f22ad
No known key found for this signature in database
GPG key ID: 136F30F901A2231D
8 changed files with 160 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
kernel/api/filetree.go
View file

@ -662,6 +662,9 @@ func getDoc(c *gin.Context) {
return
}
// 判断是否正在同步中 https://github.com/siyuan-note/siyuan/issues/6290
isSyncing := model.IsSyncingFile(rootID)
ret.Data = map[string]interface{}{
"id": id,
"mode": mode,
@ -675,6 +678,7 @@ func getDoc(c *gin.Context) {
"eof": eof,
"box": boxID,
"path": docPath,
"isSyncing": isSyncing,
}
}

2
kernel/api/sync.go
View file

@ -45,7 +45,7 @@ func performSync(c *gin.Context) {
func performBootSync(c *gin.Context) {
ret := gulu.Ret.NewResult()
defer c.JSON(http.StatusOK, ret)
model.SyncData(true, false, true)
model.BootSyncData()
ret.Code = model.BootSyncSucc
}

2
kernel/go.mod
View file

@ -37,7 +37,7 @@ require (
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/qiniu/go-sdk/v7 v7.13.0
github.com/radovskyb/watcher v1.0.7
github.com/siyuan-note/dejavu v0.0.0-20221021094047-e0b46553c42c
github.com/siyuan-note/dejavu v0.0.0-20221022034239-a08f6ea40952
github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75
github.com/siyuan-note/eventbus v0.0.0-20220916025349-3ac6e75522da
github.com/siyuan-note/filelock v0.0.0-20221007163134-7e64809023ef

2
kernel/go.sum
View file

@ -357,6 +357,8 @@ github.com/siyuan-note/dejavu v0.0.0-20221021093518-60b24cf55189 h1:o7q3PVZtTTuV
github.com/siyuan-note/dejavu v0.0.0-20221021093518-60b24cf55189/go.mod h1:i245FL1nmWaLlor+79tGDWZrAPeDvFOkHPJbZ844PyM=
github.com/siyuan-note/dejavu v0.0.0-20221021094047-e0b46553c42c h1:A7zaN6Jfk3Z//qh2aOUoHXzxR6D1qp1epovtDtQWqwY=
github.com/siyuan-note/dejavu v0.0.0-20221021094047-e0b46553c42c/go.mod h1:i245FL1nmWaLlor+79tGDWZrAPeDvFOkHPJbZ844PyM=
github.com/siyuan-note/dejavu v0.0.0-20221022034239-a08f6ea40952 h1:OQuOyyvrGLJ+Gj5z3/tUK6tGu+nRbAYelPTCeVrtY38=
github.com/siyuan-note/dejavu v0.0.0-20221022034239-a08f6ea40952/go.mod h1:i245FL1nmWaLlor+79tGDWZrAPeDvFOkHPJbZ844PyM=
github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 h1:Bi7/7f29LW+Fm0cHc0J1NO1cZqyJwljSWVmfOqVZgaE=
github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw=
github.com/siyuan-note/eventbus v0.0.0-20220916025349-3ac6e75522da h1:/jNhl7LC+9BhkWvNxuJDdsNfA/2wvfuj9mqWx4CbV90=

2
kernel/main.go
View file

@ -37,7 +37,7 @@ func main() {
sql.InitHistoryDatabase(false)
sql.SetCaseSensitive(model.Conf.Search.CaseSensitive)
model.SyncData(true, false, false)
model.BootSyncData()
model.InitBoxes()
go model.AutoGenerateDocHistory()

2
kernel/mobile/kernel.go
View file

@ -51,7 +51,7 @@ func StartKernel(container, appDir, workspaceDir, nativeLibDir, privateDataDir,
sql.InitHistoryDatabase(false)
sql.SetCaseSensitive(model.Conf.Search.CaseSensitive)
model.SyncData(true, false, false)
model.BootSyncData()
model.InitBoxes()
go model.AutoGenerateDocHistory()

114
kernel/model/repository.go
View file

@ -25,8 +25,10 @@ import (
"fmt"
"math"
"os"
"path"
"path/filepath"
"strings"
"sync"
"time"
"github.com/88250/gulu"
@ -468,7 +470,14 @@ func IndexRepo(memo string) (err error) {
return
}
func syncRepo(boot, exit, byHand bool) (err error) {
var syncingFiles = sync.Map{}
func IsSyncingFile(rootID string) (ret bool) {
_, ret = syncingFiles.Load(rootID)
return
}
func bootSyncRepo() (err error) {
if 1 > len(Conf.Repo.Key) {
syncDownloadErrCount++
planSyncAfter(fixSyncInterval)
@ -506,11 +515,101 @@ func syncRepo(boot, exit, byHand bool) (err error) {
}
syncContext := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar}
_, mergeResult, trafficStat, err := repo.Sync(cloudInfo, syncContext)
fetchedFiles, err := repo.GetSyncCloudFiles(cloudInfo, syncContext)
if errors.Is(err, dejavu.ErrRepoFatalErr) {
// 重置仓库并再次尝试同步
if _, resetErr := resetRepository(repo); nil == resetErr {
_, mergeResult, trafficStat, err = repo.Sync(cloudInfo, syncContext)
fetchedFiles, err = repo.GetSyncCloudFiles(cloudInfo, syncContext)
}
}
syncingFiles = sync.Map{}
for _, fetchedFile := range fetchedFiles {
name := path.Base(fetchedFile.Path)
if !(strings.HasSuffix(name, ".sy")) {
continue
}
id := name[:len(name)-3]
syncingFiles.Store(id, true)
}
elapsed := time.Since(start)
logging.LogInfof("boot get sync cloud files elapsed [%.2fs]", elapsed.Seconds())
if nil != err {
syncDownloadErrCount++
planSyncAfter(fixSyncInterval)
logging.LogErrorf("sync data repo failed: %s", err)
msg := fmt.Sprintf(Conf.Language(80), formatErrorMsg(err))
if errors.Is(err, dejavu.ErrCloudStorageSizeExceeded) {
msg = fmt.Sprintf(Conf.Language(43), humanize.Bytes(uint64(Conf.User.UserSiYuanRepoSize)))
if 2 == Conf.User.UserSiYuanSubscriptionPlan {
msg = fmt.Sprintf(Conf.Language(68), humanize.Bytes(uint64(Conf.User.UserSiYuanRepoSize)))
}
}
Conf.Sync.Stat = msg
util.PushStatusBar(msg)
util.PushErrMsg(msg, 0)
BootSyncSucc = 1
return
}
go func() {
time.Sleep(7 * time.Second) // 等待一段时间后前端完成界面初始化后再同步
syncErr := syncRepo(false, false)
if nil != err {
logging.LogErrorf("boot background sync repo failed: %s", syncErr)
return
}
syncingFiles = sync.Map{}
}()
return
}
func syncRepo(exit, byHand bool) (err error) {
if 1 > len(Conf.Repo.Key) {
syncDownloadErrCount++
planSyncAfter(fixSyncInterval)
msg := Conf.Language(26)
util.PushStatusBar(msg)
util.PushErrMsg(msg, 0)
err = errors.New(msg)
return
}
repo, err := newRepository()
if nil != err {
syncDownloadErrCount++
planSyncAfter(fixSyncInterval)
msg := fmt.Sprintf("sync repo failed: %s", err)
logging.LogErrorf(msg)
util.PushStatusBar(msg)
util.PushErrMsg(msg, 0)
return
}
start := time.Now()
err = indexRepoBeforeCloudSync(repo)
if nil != err {
syncDownloadErrCount++
planSyncAfter(fixSyncInterval)
return
}
cloudInfo, err := buildCloudInfo()
if nil != err {
return
}
syncContext := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar}
mergeResult, trafficStat, err := repo.Sync(cloudInfo, syncContext)
if errors.Is(err, dejavu.ErrRepoFatalErr) {
// 重置仓库并再次尝试同步
if _, resetErr := resetRepository(repo); nil == resetErr {
mergeResult, trafficStat, err = repo.Sync(cloudInfo, syncContext)
}
}
elapsed := time.Since(start)
@ -529,9 +628,6 @@ func syncRepo(boot, exit, byHand bool) (err error) {
Conf.Sync.Stat = msg
util.PushStatusBar(msg)
util.PushErrMsg(msg, 0)
if boot {
BootSyncSucc = 1
}
if exit {
ExitSyncSucc = 1
}
@ -610,10 +706,6 @@ func syncRepo(boot, exit, byHand bool) (err error) {
removes = append(removes, file.Path)
}
if boot && gulu.File.IsExist(util.BlockTreePath) {
treenode.InitBlockTree(false)
}
cache.ClearDocsIAL() // 同步后文档树文档图标没有更新 https://github.com/siyuan-note/siyuan/issues/4939
if needFullReindex(upsertTrees) { // 改进同步后全量重建索引判断 https://github.com/siyuan-note/siyuan/issues/5764
@ -622,7 +714,7 @@ func syncRepo(boot, exit, byHand bool) (err error) {
}
incReindex(upserts, removes)
if !boot && !exit {
if !exit {
util.ReloadUI()
}

48
kernel/model/sync.go
View file

@ -55,6 +55,52 @@ func AutoSync() {
}
}
func BootSyncData() {
defer logging.Recover()
if util.IsMutexLocked(&syncLock) {
logging.LogWarnf("sync is in progress")
planSyncAfter(30 * time.Second)
return
}
syncLock.Lock()
defer syncLock.Unlock()
util.IncBootProgress(3, "Syncing data from the cloud...")
BootSyncSucc = 0
if !IsSubscriber() || !Conf.Sync.Enabled || "" == Conf.Sync.CloudName || !IsValidCloudDirName(Conf.Sync.CloudName) {
return
}
logging.LogInfof("sync before boot")
if 7 < syncDownloadErrCount {
logging.LogErrorf("sync download error too many times, cancel auto sync, try to sync by hand")
util.PushErrMsg(Conf.Language(125), 1000*60*60)
planSyncAfter(64 * time.Minute)
return
}
now := util.CurrentTimeMillis()
Conf.Sync.Synced = now
util.BroadcastByType("main", "syncing", 0, Conf.Language(81), nil)
err := bootSyncRepo()
synced := util.Millisecond2Time(Conf.Sync.Synced).Format("2006-01-02 15:04:05") + "\n\n"
if nil == err {
synced += Conf.Sync.Stat
} else {
synced += fmt.Sprintf(Conf.Language(80), formatErrorMsg(err))
}
msg := fmt.Sprintf(Conf.Language(82), synced)
Conf.Sync.Stat = msg
Conf.Save()
util.BroadcastByType("main", "syncing", 1, msg, nil)
return
}
func SyncData(boot, exit, byHand bool) {
defer logging.Recover()
@ -112,7 +158,7 @@ func SyncData(boot, exit, byHand bool) {
Conf.Sync.Synced = now
util.BroadcastByType("main", "syncing", 0, Conf.Language(81), nil)
err := syncRepo(boot, exit, byHand)
err := syncRepo(exit, byHand)
synced := util.Millisecond2Time(Conf.Sync.Synced).Format("2006-01-02 15:04:05") + "\n\n"
if nil == err {
synced += Conf.Sync.Stat