Merge remote-tracking branch 'origin/dev' into dev

This commit is contained in:
Vanessa 2022-06-25 20:40:24 +08:00
commit 51d3d68981
5 changed files with 209 additions and 51 deletions

View file

@ -40,8 +40,9 @@ require (
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/qiniu/go-sdk/v7 v7.13.0
github.com/radovskyb/watcher v1.0.7
github.com/siyuan-note/dejavu v0.0.0-20220620134708-bcbbf7f6478e
github.com/siyuan-note/dejavu v0.0.0-20220625122139-d442191b9acc
github.com/siyuan-note/encryption v0.0.0-20220612074546-f1dd94fe8676
github.com/siyuan-note/eventbus v0.0.0-20220624162334-ca7c06dc771f
github.com/siyuan-note/filelock v0.0.0-20220616063212-74cfba0754ee
github.com/siyuan-note/httpclient v0.0.0-20220622170705-004622a8138d
github.com/vmihailenco/msgpack/v5 v5.3.5
@ -57,6 +58,7 @@ require (
github.com/Masterminds/semver/v3 v3.1.1 // indirect
github.com/alecthomas/chroma v0.10.0 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/dlclark/regexp2 v1.4.0 // indirect
github.com/dsnet/compress v0.0.1 // indirect
@ -97,11 +99,11 @@ require (
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
golang.org/x/mod v0.5.1 // indirect
golang.org/x/net v0.0.0-20220621193019-9d032be2e588 // indirect
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c // indirect
golang.org/x/sys v0.0.0-20220624220833-87e55d714810 // indirect
golang.org/x/tools v0.1.8 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/protobuf v1.28.0 // indirect

View file

@ -83,6 +83,8 @@ github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoU
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM=
github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@ -419,10 +421,14 @@ github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJ
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/siyuan-note/dejavu v0.0.0-20220620134708-bcbbf7f6478e h1:AOxGL9q3HYivlj5dP6y2zG+JA0ZN4H1oiTVjcbW8dOQ=
github.com/siyuan-note/dejavu v0.0.0-20220620134708-bcbbf7f6478e/go.mod h1:nCckhUVoaofwwlBOglCui+Mr7Hhyd059vMXzs7jdne0=
github.com/siyuan-note/dejavu v0.0.0-20220625094848-7e1437e9c8c2 h1:G0gSYBcywwHgHQesFo/tGbKc2tEGsWBmncEuGpGH1Xg=
github.com/siyuan-note/dejavu v0.0.0-20220625094848-7e1437e9c8c2/go.mod h1:i7dnOgHM41EA7xIX2CYSxYe0WhksGZidQQsLvWryK7w=
github.com/siyuan-note/dejavu v0.0.0-20220625122139-d442191b9acc h1:wbgMZUgzWFOyeTJ9vtyW+N8v+bCzcBRR2LbaurN76so=
github.com/siyuan-note/dejavu v0.0.0-20220625122139-d442191b9acc/go.mod h1:i7dnOgHM41EA7xIX2CYSxYe0WhksGZidQQsLvWryK7w=
github.com/siyuan-note/encryption v0.0.0-20220612074546-f1dd94fe8676 h1:QB9TjJQFhXhZ6dAtPpY02DlzHAQm1C+WqZq6OadG8mI=
github.com/siyuan-note/encryption v0.0.0-20220612074546-f1dd94fe8676/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw=
github.com/siyuan-note/eventbus v0.0.0-20220624162334-ca7c06dc771f h1:JMobMNZ7AqaKKyEK+WeWFhix/2TDQXgPZDajU00IybU=
github.com/siyuan-note/eventbus v0.0.0-20220624162334-ca7c06dc771f/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI=
github.com/siyuan-note/filelock v0.0.0-20220616063212-74cfba0754ee h1:8nBBJKHbXuMF2dWKUKsrnnFblgeNBfPjLgVm0aDas/s=
github.com/siyuan-note/filelock v0.0.0-20220616063212-74cfba0754ee/go.mod h1:c4vwvWRrnfa75OXiO21K/76BFRJ4cFITKNoVs5lFdwc=
github.com/siyuan-note/httpclient v0.0.0-20220622170705-004622a8138d h1:zMkahXrTDPhBUg9W7HkGlYP6WY5AkQoJqj7Cmk++qoU=
@ -493,8 +499,9 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM=
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -579,8 +586,8 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220111093109-d55c255bac03/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220621193019-9d032be2e588 h1:9ubFuySsnAJYGyJrZ3koiEv8FyqofCBdz3G9Mbf2YFc=
golang.org/x/net v0.0.0-20220621193019-9d032be2e588/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e h1:TsQ7F31D3bUCLeqPT0u+yjp1guoArKaNKmCr22PYgTQ=
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -657,8 +664,8 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c h1:aFV+BgZ4svzjfabn8ERpuB4JI4N6/rdy1iusx77G3oU=
golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220624220833-87e55d714810 h1:rHZQSjJdAI4Xf5Qzeh2bBc5YJIkPFVM6oDtMFYmgws0=
golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.0.0-20180302201248-b7ef84aaf62a/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

View file

@ -26,27 +26,159 @@ import (
"github.com/88250/gulu"
"github.com/siyuan-note/dejavu"
"github.com/siyuan-note/dejavu/entity"
"github.com/siyuan-note/encryption"
"github.com/siyuan-note/eventbus"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/util"
)
func UploadSnapshot(id string) (err error) {
if 1 > len(Conf.Repo.Key) {
err = errors.New(Conf.Language(26))
return
}
func init() {
eventbus.Subscribe(dejavu.EvtIndexWalkData, func(context map[string]interface{}, path string) {
msg := "Indexing repo [walk data " + path + "]"
util.SetBootDetails(msg)
repo, err := dejavu.NewRepo(util.DataDir, util.RepoDir, Conf.Repo.Key)
if nil != err {
util.LogErrorf("init repo failed: %s", err)
return
}
switch context[CtxPushMsg].(int) {
case CtxPushMsgToProgress:
util.PushEndlessProgress(msg)
case CtxPushMsgToStatusBar:
util.PushStatusBar(msg)
case CtxPushMsgToStatusBarAndProgress:
util.PushStatusBar(msg)
util.PushEndlessProgress(msg)
}
})
eventbus.Subscribe(dejavu.EvtIndexGetLatestFile, func(context map[string]interface{}, path string) {
msg := "Indexing repo [get latest file " + path + "]"
util.SetBootDetails(msg)
_ = repo
return
switch context[CtxPushMsg].(int) {
case CtxPushMsgToProgress:
util.PushEndlessProgress(msg)
case CtxPushMsgToStatusBar:
util.PushStatusBar(msg)
case CtxPushMsgToStatusBarAndProgress:
util.PushStatusBar(msg)
util.PushEndlessProgress(msg)
}
})
eventbus.Subscribe(dejavu.EvtIndexUpsertFile, func(context map[string]interface{}, path string) {
msg := "Indexing repo [upsert file " + path + "]"
util.SetBootDetails(msg)
switch context[CtxPushMsg].(int) {
case CtxPushMsgToProgress:
util.PushEndlessProgress(msg)
case CtxPushMsgToStatusBar:
util.PushStatusBar(msg)
case CtxPushMsgToStatusBarAndProgress:
util.PushStatusBar(msg)
util.PushEndlessProgress(msg)
}
})
eventbus.Subscribe(dejavu.EvtCheckoutWalkData, func(context map[string]interface{}, path string) {
msg := "Checkout repo [walk data " + path + "]"
util.SetBootDetails(msg)
switch context[CtxPushMsg].(int) {
case CtxPushMsgToProgress:
util.PushEndlessProgress(msg)
case CtxPushMsgToStatusBar:
util.PushStatusBar(msg)
case CtxPushMsgToStatusBarAndProgress:
util.PushStatusBar(msg)
util.PushEndlessProgress(msg)
}
})
eventbus.Subscribe(dejavu.EvtCheckoutUpsertFile, func(context map[string]interface{}, path string) {
msg := "Checkout repo [upsert file " + path + "]"
util.SetBootDetails(msg)
switch context[CtxPushMsg].(int) {
case CtxPushMsgToProgress:
util.PushEndlessProgress(msg)
case CtxPushMsgToStatusBar:
util.PushStatusBar(msg)
case CtxPushMsgToStatusBarAndProgress:
util.PushStatusBar(msg)
util.PushEndlessProgress(msg)
}
})
eventbus.Subscribe(dejavu.EvtCheckoutRemoveFile, func(context map[string]interface{}, path string) {
msg := "Checkout repo [remove file " + path + "]"
util.SetBootDetails(msg)
switch context[CtxPushMsg].(int) {
case CtxPushMsgToProgress:
util.PushEndlessProgress(msg)
case CtxPushMsgToStatusBar:
util.PushStatusBar(msg)
case CtxPushMsgToStatusBarAndProgress:
util.PushStatusBar(msg)
util.PushEndlessProgress(msg)
}
})
eventbus.Subscribe(dejavu.EvtSyncBeforeDownloadCloudIndexes, func(context map[string]interface{}, latestSync string) {
msg := "Downloading repo indexes..."
util.SetBootDetails(msg)
switch context[CtxPushMsg].(int) {
case CtxPushMsgToProgress:
util.PushEndlessProgress(msg)
case CtxPushMsgToStatusBar:
util.PushStatusBar(msg)
case CtxPushMsgToStatusBarAndProgress:
util.PushStatusBar(msg)
util.PushEndlessProgress(msg)
}
})
eventbus.Subscribe(dejavu.EvtSyncBeforeDownloadCloudFile, func(context map[string]interface{}, id string) {
msg := "Downloading repo object [" + id + "]"
util.SetBootDetails(msg)
switch context[CtxPushMsg].(int) {
case CtxPushMsgToProgress:
util.PushEndlessProgress(msg)
case CtxPushMsgToStatusBar:
util.PushStatusBar(msg)
case CtxPushMsgToStatusBarAndProgress:
util.PushStatusBar(msg)
util.PushEndlessProgress(msg)
}
})
eventbus.Subscribe(dejavu.EvtSyncBeforeDownloadCloudChunk, func(context map[string]interface{}, id string) {
msg := "Downloading repo object [" + id + "]"
util.SetBootDetails(msg)
switch context[CtxPushMsg].(int) {
case CtxPushMsgToProgress:
util.PushEndlessProgress(msg)
case CtxPushMsgToStatusBar:
util.PushStatusBar(msg)
case CtxPushMsgToStatusBarAndProgress:
util.PushStatusBar(msg)
util.PushEndlessProgress(msg)
}
})
eventbus.Subscribe(dejavu.EvtSyncBeforeUploadObject, func(context map[string]interface{}, id string) {
msg := "Uploading repo object [" + id + "]"
util.SetBootDetails(msg)
switch context[CtxPushMsg].(int) {
case CtxPushMsgToProgress:
util.PushEndlessProgress(msg)
case CtxPushMsgToStatusBar:
util.PushStatusBar(msg)
case CtxPushMsgToStatusBarAndProgress:
util.PushStatusBar(msg)
util.PushEndlessProgress(msg)
}
})
}
func GetRepoIndexLogs(page int) (logs []*dejavu.Log, pageCount, totalCount int, err error) {
@ -161,18 +293,6 @@ func InitRepoKey() (err error) {
return
}
var checkoutCallbacks = map[string]dejavu.Callback{
"walkData": func(context, arg interface{}, err error) {
context.(func(msg string))(arg.(string))
},
"upsertFile": func(context, arg interface{}, err error) {
context.(func(msg string))(arg.(*entity.File).Path)
},
"removeFile": func(context, arg interface{}, err error) {
context.(func(msg string))(arg.(string))
},
}
func CheckoutRepo(id string) (err error) {
if 1 > len(Conf.Repo.Key) {
err = errors.New(Conf.Language(26))
@ -199,7 +319,9 @@ func CheckoutRepo(id string) (err error) {
Conf.Sync.Enabled = false
Conf.Save()
err = repo.Checkout(id, util.PushEndlessProgress, checkoutCallbacks)
err = repo.Checkout(id, map[string]interface{}{
CtxPushMsg: CtxPushMsgToStatusBarAndProgress,
})
if nil != err {
util.PushClearProgress()
return
@ -215,18 +337,6 @@ func CheckoutRepo(id string) (err error) {
return
}
var indexCallbacks = map[string]dejavu.Callback{
"walkData": func(context, arg interface{}, err error) {
context.(func(msg string))(arg.(string))
},
"getLatestFile": func(context, arg interface{}, err error) {
context.(func(msg string))(arg.(*entity.File).Path)
},
"upsertFile": func(context, arg interface{}, err error) {
context.(func(msg string))(arg.(*entity.File).Path)
},
}
func IndexRepo(memo string) (err error) {
if 1 > len(Conf.Repo.Key) {
err = errors.New(Conf.Language(26))
@ -251,11 +361,21 @@ func IndexRepo(memo string) (err error) {
WaitForWritingFiles()
sql.WaitForWritingDatabase()
filelock.ReleaseAllFileLocks()
_, err = repo.Index(memo, util.PushEndlessProgress, indexCallbacks)
_, err = repo.Index(memo, map[string]interface{}{
CtxPushMsg: CtxPushMsgToStatusBarAndProgress,
})
util.PushClearProgress()
return
}
const (
CtxPushMsg = "pushMsg"
CtxPushMsgToProgress = iota
CtxPushMsgToStatusBar
CtxPushMsgToStatusBarAndProgress
)
func indexRepoBeforeCloudSync() {
if 1 > len(Conf.Repo.Key) {
return
@ -269,7 +389,9 @@ func indexRepoBeforeCloudSync() {
start := time.Now()
latest, err := repo.Latest()
index, err := repo.Index("[Auto] Cloud sync", nil, nil)
index, err := repo.Index("[Auto] Cloud sync", map[string]interface{}{
CtxPushMsg: CtxPushMsgToStatusBar,
})
if nil != err {
util.LogErrorf("index repo before cloud sync failed: %s", err)
return
@ -288,3 +410,21 @@ func indexRepoBeforeCloudSync() {
util.LogWarnf("index repo before cloud sync elapsed [%dms]", elapsed.Milliseconds())
}
}
func syncRepo() (err error) {
if 1 > len(Conf.Repo.Key) {
return
}
repo, err := dejavu.NewRepo(util.DataDir, util.RepoDir, Conf.Repo.Key)
if nil != err {
util.LogErrorf("init repo failed: %s", err)
return
}
start := time.Now()
err = repo.Sync(Conf.Sync.CloudName, Conf.User.UserId, Conf.User.UserToken, Conf.System.NetworkProxy.String(), util.AliyunServer)
elapsed := time.Since(start)
util.LogInfof("sync repo elapsed [%.2fs]", elapsed.Seconds())
return
}

View file

@ -154,6 +154,11 @@ func SyncData(boot, exit, byHand bool) {
// 创建数据快照 https://github.com/siyuan-note/siyuan/issues/5161
indexRepoBeforeCloudSync()
// 同步数据仓库
if syncRepoErr := syncRepo(); nil != syncRepoErr {
util.LogErrorf("sync repo failed: %s", syncRepoErr)
}
// 获取工作空间数据配置(数据版本)
dataConf, err := getWorkspaceDataConf()
if nil != err {

View file

@ -144,6 +144,10 @@ func PushErrMsg(msg string, timeout int) (msgId string) {
return
}
func PushStatusBar(msg string) {
BroadcastByType("main", "statusbar", 0, msg, nil)
}
const (
PushProgressCodeProgressed = 0 // 有进度
PushProgressCodeEndless = 1 // 无进度