0ct0pu5/siyuan
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

🎨 Improve data indexing performance when importing .sy.zip and markdown https://github.com/siyuan-note/siyuan/issues/10986

Browse source
This commit is contained in:
Daniel 2024-04-11 21:54:34 +08:00
parent 2bcf26166d
commit f5ceb847f4
No known key found for this signature in database
GPG key ID: 86211BA83DF03017
15 changed files with 45 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
kernel/model/assets.go
View file

@ -235,7 +235,7 @@ func NetImg2LocalAssets(rootID, originalURL string) (err error) {
})
if 0 < files {
util.PushUpdateMsg(msgId, Conf.Language(113), 7000)
if err = writeJSONQueue(tree); nil != err {
if err = writeTreeUpsertQueue(tree); nil != err {
return
}
util.PushUpdateMsg(msgId, fmt.Sprintf(Conf.Language(120), files), 5000)
@ -422,7 +422,7 @@ func NetAssets2LocalAssets(rootID string) (err error) {
if 0 < files {
util.PushUpdateMsg(msgId, Conf.Language(113), 7000)
if err = writeJSONQueue(tree); nil != err {
if err = writeTreeUpsertQueue(tree); nil != err {
return
}
util.PushUpdateMsg(msgId, fmt.Sprintf(Conf.Language(120), files), 5000)

2
kernel/model/attribute_view.go
View file

@ -1609,7 +1609,7 @@ func (tx *Transaction) doRemoveAttrViewView(operation *Operation) (ret *TxErr) {
}
for _, tree := range trees {
if err = indexWriteJSONQueue(tree); nil != err {
if err = indexWriteTreeUpsertQueue(tree); nil != err {
return
}
}

6
kernel/model/block.go
View file

@ -205,7 +205,7 @@ func TransferBlockRef(fromID, toID string, refIDs []string) (err error) {
}
}
if err = indexWriteJSONQueue(tree); nil != err {
if err = indexWriteTreeUpsertQueue(tree); nil != err {
return
}
}
@ -317,11 +317,11 @@ func SwapBlockRef(refID, defID string, includeChildren bool) (err error) {
}
refPivot.Unlink()
if err = indexWriteJSONQueue(refTree); nil != err {
if err = indexWriteTreeUpsertQueue(refTree); nil != err {
return
}
if !sameTree {
if err = indexWriteJSONQueue(defTree); nil != err {
if err = indexWriteTreeUpsertQueue(defTree); nil != err {
return
}
}

8
kernel/model/blockial.go
View file

@ -87,7 +87,7 @@ func SetBlockReminder(id string, timed string) (err error) {
node.SetIALAttr(attrName, timed)
util.PushMsg(fmt.Sprintf(Conf.Language(101), time.UnixMilli(timedMills).Format("2006-01-02 15:04")), 5000)
}
if err = indexWriteJSONQueue(tree); nil != err {
if err = indexWriteTreeUpsertQueue(tree); nil != err {
return
}
IncSync()
@ -143,7 +143,7 @@ func BatchSetBlockAttrs(blockAttrs []map[string]interface{}) (err error) {
}
for _, tree := range trees {
if err = indexWriteJSONQueue(tree); nil != err {
if err = indexWriteTreeUpsertQueue(tree); nil != err {
return
}
}
@ -180,7 +180,7 @@ func setNodeAttrs(node *ast.Node, tree *parse.Tree, nameValues map[string]string
return
}
if err = indexWriteJSONQueue(tree); nil != err {
if err = indexWriteTreeUpsertQueue(tree); nil != err {
return
}
@ -279,7 +279,7 @@ func ResetBlockAttrs(id string, nameValues map[string]string) (err error) {
updateRefTextRenameDoc(tree)
}
if err = indexWriteJSONQueue(tree); nil != err {
if err = indexWriteTreeUpsertQueue(tree); nil != err {
return
}
IncSync()

4
kernel/model/bookmark.go
View file

@ -64,7 +64,7 @@ func RemoveBookmark(bookmark string) (err error) {
}
util.PushEndlessProgress(fmt.Sprintf(Conf.Language(111), util.EscapeHTML(tree.Root.IALAttr("title"))))
if err = writeJSONQueue(tree); nil != err {
if err = writeTreeUpsertQueue(tree); nil != err {
util.ClearPushProgress(100)
return
}
@ -122,7 +122,7 @@ func RenameBookmark(oldBookmark, newBookmark string) (err error) {
}
util.PushEndlessProgress(fmt.Sprintf(Conf.Language(111), util.EscapeHTML(tree.Root.IALAttr("title"))))
if err = writeJSONQueue(tree); nil != err {
if err = writeTreeUpsertQueue(tree); nil != err {
util.ClearPushProgress(100)
return
}

2
kernel/model/export.go
View file

@ -295,7 +295,7 @@ func Export2Liandi(id string) (err error) {
articleId = result.Data.(string)
tree, _ = LoadTreeByBlockID(id) // 这里必须重新加载，因为前面导出时已经修改了树结构
tree.Root.SetIALAttr(liandiArticleIdAttrName, articleId)
if err = writeJSONQueue(tree); nil != err {
if err = writeTreeUpsertQueue(tree); nil != err {
return
}
}

25
kernel/model/file.go
View file

@ -1065,7 +1065,7 @@ func loadNodesByMode(node *ast.Node, inputIndex, mode, size int, isDoc, isHeadin
return
}
func writeJSONQueue(tree *parse.Tree) (err error) {
func writeTreeUpsertQueue(tree *parse.Tree) (err error) {
if err = filesys.WriteTree(tree); nil != err {
return
}
@ -1073,9 +1073,22 @@ func writeJSONQueue(tree *parse.Tree) (err error) {
return
}
func indexWriteJSONQueue(tree *parse.Tree) (err error) {
func writeTreeIndexQueue(tree *parse.Tree) (err error) {
if err = filesys.WriteTree(tree); nil != err {
return
}
sql.IndexTreeQueue(tree)
return
}
func indexWriteTreeIndexQueue(tree *parse.Tree) (err error) {
treenode.IndexBlockTree(tree)
return writeJSONQueue(tree)
return writeTreeIndexQueue(tree)
}
func indexWriteTreeUpsertQueue(tree *parse.Tree) (err error) {
treenode.IndexBlockTree(tree)
return writeTreeUpsertQueue(tree)
}
func renameWriteJSONQueue(tree *parse.Tree) (err error) {
@ -1181,7 +1194,7 @@ func CreateDailyNote(boxID string) (p string, existed bool, err error) {
date := time.Now().Format("20060102")
if tree.Root.IALAttr("custom-dailynote-"+date) == "" {
tree.Root.SetIALAttr("custom-dailynote-"+date, date)
if err = indexWriteJSONQueue(tree); nil != err {
if err = indexWriteTreeUpsertQueue(tree); nil != err {
return
}
}
@ -1232,7 +1245,7 @@ func CreateDailyNote(boxID string) (p string, existed bool, err error) {
}
tree.Root.SetIALAttr("updated", util.CurrentTimeSecondsStr())
if err = indexWriteJSONQueue(tree); nil != err {
if err = indexWriteTreeUpsertQueue(tree); nil != err {
return
}
}
@ -1249,7 +1262,7 @@ func CreateDailyNote(boxID string) (p string, existed bool, err error) {
p = tree.Path
date := time.Now().Format("20060102")
tree.Root.SetIALAttr("custom-dailynote-"+date, date)
if err = indexWriteJSONQueue(tree); nil != err {
if err = indexWriteTreeUpsertQueue(tree); nil != err {
return
}

2
kernel/model/format.go
View file

@ -68,7 +68,7 @@ func AutoSpace(rootID string) (err error) {
newTree.Path = tree.Path
newTree.HPath = tree.HPath
newTree.Box = tree.Box
err = writeJSONQueue(newTree)
err = writeTreeUpsertQueue(newTree)
if nil != err {
return
}

6
kernel/model/heading.go
View file

@ -256,7 +256,7 @@ func Doc2Heading(srcID, targetID string, after bool) (srcTreeBox, srcTreePath st
targetTree.Root.SetIALAttr("updated", util.CurrentTimeSecondsStr())
treenode.RemoveBlockTreesByRootID(srcTree.ID)
treenode.RemoveBlockTreesByRootID(targetTree.ID)
err = indexWriteJSONQueue(targetTree)
err = indexWriteTreeUpsertQueue(targetTree)
IncSync()
RefreshBacklink(srcTree.ID)
RefreshBacklink(targetTree.ID)
@ -355,7 +355,7 @@ func Heading2Doc(srcHeadingID, targetBoxID, targetPath string) (srcRootBlockID,
srcTree.Root.AppendChild(treenode.NewParagraph())
}
treenode.RemoveBlockTreesByRootID(srcTree.ID)
if err = indexWriteJSONQueue(srcTree); nil != err {
if err = indexWriteTreeUpsertQueue(srcTree); nil != err {
return "", "", err
}
@ -363,7 +363,7 @@ func Heading2Doc(srcHeadingID, targetBoxID, targetPath string) (srcRootBlockID,
newTree.Root.SetIALAttr("updated", util.CurrentTimeSecondsStr())
newTree.Root.Spec = "1"
box.addMinSort(path.Dir(newTargetPath), newTree.ID)
if err = indexWriteJSONQueue(newTree); nil != err {
if err = indexWriteTreeUpsertQueue(newTree); nil != err {
return "", "", err
}
IncSync()

3
kernel/model/import.go
View file

@ -894,8 +894,7 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) {
buildBlockRefInText()
for i, tree := range importTrees {
treenode.IndexBlockTree(tree)
sql.IndexTreeQueue(tree)
indexWriteTreeIndexQueue(tree)
if 0 == i%4 {
util.PushEndlessProgress(fmt.Sprintf(Conf.Language(66), fmt.Sprintf("%d/%d ", i, len(importTrees))+tree.HPath))
}

2
kernel/model/index_fix.go
View file

@ -481,7 +481,7 @@ func reindexTree0(tree *parse.Tree, i, size int) {
if "" == updated {
updated = util.TimeFromID(tree.Root.ID)
tree.Root.SetIALAttr("updated", updated)
indexWriteJSONQueue(tree)
indexWriteTreeUpsertQueue(tree)
} else {
treenode.IndexBlockTree(tree)
sql.IndexTreeQueue(tree)

4
kernel/model/listitem.go
View file

@ -97,7 +97,7 @@ func ListItem2Doc(srcListItemID, targetBoxID, targetPath string) (srcRootBlockID
srcTree.Root.AppendChild(treenode.NewParagraph())
}
treenode.RemoveBlockTreesByRootID(srcTree.ID)
if err = indexWriteJSONQueue(srcTree); nil != err {
if err = indexWriteTreeUpsertQueue(srcTree); nil != err {
return "", "", err
}
@ -105,7 +105,7 @@ func ListItem2Doc(srcListItemID, targetBoxID, targetPath string) (srcRootBlockID
newTree.Root.SetIALAttr("updated", util.CurrentTimeSecondsStr())
newTree.Root.Spec = "1"
box.addMinSort(path.Dir(newTargetPath), newTree.ID)
if err = indexWriteJSONQueue(newTree); nil != err {
if err = indexWriteTreeUpsertQueue(newTree); nil != err {
return "", "", err
}
IncSync()

2
kernel/model/search.go
View file

@ -726,7 +726,7 @@ func FindReplace(keyword, replacement string, replaceTypes map[string]bool, ids
return ast.WalkContinue
})
if err = writeJSONQueue(tree); nil != err {
if err = writeTreeUpsertQueue(tree); nil != err {
return
}
}

4
kernel/model/tag.go
View file

@ -93,7 +93,7 @@ func RemoveTag(label string) (err error) {
n.Unlink()
}
util.PushEndlessProgress(fmt.Sprintf(Conf.Language(111), util.EscapeHTML(tree.Root.IALAttr("title"))))
if err = writeJSONQueue(tree); nil != err {
if err = writeTreeUpsertQueue(tree); nil != err {
util.ClearPushProgress(100)
return
}
@ -176,7 +176,7 @@ func RenameTag(oldLabel, newLabel string) (err error) {
}
}
util.PushEndlessProgress(fmt.Sprintf(Conf.Language(111), util.EscapeHTML(tree.Root.IALAttr("title"))))
if err = writeJSONQueue(tree); nil != err {
if err = writeTreeUpsertQueue(tree); nil != err {
util.ClearPushProgress(100)
return
}

4
kernel/model/transaction.go
View file

@ -1309,7 +1309,7 @@ func (tx *Transaction) begin() (err error) {
func (tx *Transaction) commit() (err error) {
for _, tree := range tx.trees {
if err = writeJSONQueue(tree); nil != err {
if err = writeTreeUpsertQueue(tree); nil != err {
return
}
@ -1474,7 +1474,7 @@ func refreshDynamicRefTexts(updatedDefNodes map[string]*ast.Node, updatedTrees m
// 3. 保存变更
for _, tree := range changedRefTree {
indexWriteJSONQueue(tree)
indexWriteTreeUpsertQueue(tree)
}
}