Merge remote-tracking branch 'origin/dev' into dev

kernel/api/setting.go

@@ -221,7 +221,6 @@ func setSearch(c *gin.Context) {
 	if s.CaseSensitive != oldCaseSensitive {
 		model.FullReindex()
 	}
-	sql.ClearVirtualRefKeywords()
 	ret.Data = s
 }
 

kernel/job/cron.go

@@ -39,6 +39,7 @@ func StartCron() {
 	go every(util.SQLFlushInterval, sql.FlushTxJob)
 	go every(10*time.Minute, model.FixIndexJob)
 	go every(10*time.Minute, model.IndexEmbedBlockJob)
+	go every(10*time.Minute, model.CacheVirtualBlockRefJob)
 	go every(12*time.Second, model.OCRAssetsJob)
 	go every(12*time.Second, model.FlushAssetsTextsJob)
 	go every(30*time.Second, model.HookDesktopUIProcJob)

kernel/model/backlink.go

@@ -167,7 +167,7 @@ func buildBacklink(refID string, refTree *parse.Tree, keywords []string, luteEng
 					return ast.WalkContinue
 				}
 
-				markReplaceSpan(n, &unlinks, keywords, searchMarkDataType, luteEngine)
+				markReplaceSpan(n, &unlinks, keywords, search.MarkDataType, luteEngine)
 				return ast.WalkContinue
 			})
 
@@ -690,7 +690,7 @@ func searchBackmention(mentionKeywords []string, keyword string, excludeBacklink
 			continue
 		}
 
-		newText := markReplaceSpanWithSplit(text, mentionKeywords, getMarkSpanStart(searchMarkDataType), getMarkSpanEnd())
+		newText := markReplaceSpanWithSplit(text, mentionKeywords, search.GetMarkSpanStart(search.MarkDataType), search.GetMarkSpanEnd())
 		if text != newText {
 			tmp = append(tmp, b)
 		}

kernel/model/file.go

@@ -440,19 +440,6 @@ func StatTree(id string) (ret *util.BlockStatResult) {
 	}
 }
 
-const (
-	searchMarkDataType      = "search-mark"
-	virtualBlockRefDataType = "virtual-block-ref"
-)
-
-func getMarkSpanStart(dataType string) string {
-	return fmt.Sprintf("<span data-type=\"%s\">", dataType)
-}
-
-func getMarkSpanEnd() string {
-	return "</span>"
-}
-
 func GetDoc(startID, endID, id string, index int, keyword string, mode int, size int, isBacklink bool) (blockCount, childBlockCount int, dom, parentID, parent2ID, rootID, typ string, eof bool, boxID, docPath string, isBacklinkExpand bool, err error) {
 	//os.MkdirAll("pprof", 0755)
 	//cpuProfile, _ := os.Create("pprof/GetDoc")
@@ -618,7 +605,7 @@ func GetDoc(startID, endID, id string, index int, keyword string, mode int, size
 	}
 
 	refCount := sql.QueryRootChildrenRefCount(rootID)
-	virtualBlockRefKeywords := getVirtualRefKeywords(tree.Root.IALAttr("title"))
+	virtualBlockRefKeywords := getBlockVirtualRefKeywords(tree.Root)
 
 	subTree := &parse.Tree{ID: rootID, Root: &ast.Node{Type: ast.NodeDocument}, Marks: tree.Marks}
 	keyword = strings.Join(strings.Split(keyword, " "), search.TermSep)
@@ -659,7 +646,7 @@ func GetDoc(startID, endID, id string, index int, keyword string, mode int, size
 					}
 				}
 				if hitBlock {
-					if markReplaceSpan(n, &unlinks, keywords, searchMarkDataType, luteEngine) {
+					if markReplaceSpan(n, &unlinks, keywords, search.MarkDataType, luteEngine) {
 						return ast.WalkContinue
 					}
 				}

kernel/model/history.go

@@ -180,7 +180,7 @@ func GetDocHistoryContent(historyPath, keyword string) (id, rootID, content stri
 			n.RemoveIALAttr("fold")
 
 			if 0 < len(keywords) {
-				if markReplaceSpan(n, &unlinks, keywords, searchMarkDataType, luteEngine) {
+				if markReplaceSpan(n, &unlinks, keywords, search.MarkDataType, luteEngine) {
 					return ast.WalkContinue
 				}
 			}

kernel/model/search.go

@@ -915,9 +915,9 @@ func stringQuery(query string) string {
 func markReplaceSpan(n *ast.Node, unlinks *[]*ast.Node, keywords []string, markSpanDataType string, luteEngine *lute.Lute) bool {
 	text := n.Content()
 	if ast.NodeText == n.Type {
-		text = search.EncloseHighlighting(text, keywords, getMarkSpanStart(markSpanDataType), getMarkSpanEnd(), Conf.Search.CaseSensitive)
+		text = search.EncloseHighlighting(text, keywords, search.GetMarkSpanStart(markSpanDataType), search.GetMarkSpanEnd(), Conf.Search.CaseSensitive)
 		n.Tokens = gulu.Str.ToBytes(text)
-		if bytes.Contains(n.Tokens, []byte(searchMarkDataType)) {
+		if bytes.Contains(n.Tokens, []byte(search.MarkDataType)) {
 			linkTree := parse.Inline("", n.Tokens, luteEngine.ParseOptions)
 			var children []*ast.Node
 			for c := linkTree.Root.FirstChild.FirstChild; nil != c; c = c.Next {
@@ -935,10 +935,10 @@ func markReplaceSpan(n *ast.Node, unlinks *[]*ast.Node, keywords []string, markS
 			return false
 		}
 
-		startTag := getMarkSpanStart(markSpanDataType)
-		text = search.EncloseHighlighting(text, keywords, startTag, getMarkSpanEnd(), Conf.Search.CaseSensitive)
-		if strings.Contains(text, searchMarkDataType) {
-			dataType := getMarkSpanStart(n.TextMarkType + " " + searchMarkDataType)
+		startTag := search.GetMarkSpanStart(markSpanDataType)
+		text = search.EncloseHighlighting(text, keywords, startTag, search.GetMarkSpanEnd(), Conf.Search.CaseSensitive)
+		if strings.Contains(text, search.MarkDataType) {
+			dataType := search.GetMarkSpanStart(n.TextMarkType + " " + search.MarkDataType)
 			text = strings.ReplaceAll(text, startTag, dataType)
 			tokens := gulu.Str.ToBytes(text)
 			linkTree := parse.Inline("", tokens, luteEngine.ParseOptions)

kernel/model/virutalref.go

@@ -17,6 +17,7 @@
 package model
 
 import (
+	"bytes"
 	"regexp"
 	"sort"
 	"strings"
@@ -25,12 +26,83 @@ import (
 	"github.com/88250/lute"
 	"github.com/88250/lute/ast"
 	"github.com/88250/lute/parse"
+	"github.com/dgraph-io/ristretto"
+	"github.com/siyuan-note/siyuan/kernel/search"
 	"github.com/siyuan-note/siyuan/kernel/sql"
 	"github.com/siyuan-note/siyuan/kernel/treenode"
 )
 
+// virtualBlockRefCache 用于保存块关联的虚拟引用关键字。
+// 改进打开虚拟引用后加载文档的性能 https://github.com/siyuan-note/siyuan/issues/7378
+var virtualBlockRefCache, _ = ristretto.NewCache(&ristretto.Config{
+	NumCounters: 102400,
+	MaxCost:     10240,
+	BufferItems: 64,
+})
+
+func getBlockVirtualRefKeywords(root *ast.Node) (ret []string) {
+	val, ok := virtualBlockRefCache.Get(root.ID)
+	if !ok {
+		buf := bytes.Buffer{}
+		ast.Walk(root, func(n *ast.Node, entering bool) ast.WalkStatus {
+			if !entering || !n.IsBlock() {
+				return ast.WalkContinue
+			}
+
+			content := treenode.NodeStaticContent(n, nil)
+			buf.WriteString(content)
+			return ast.WalkContinue
+		})
+		content := buf.String()
+		ret = putBlockVirtualRefKeywords(content, root.ID, root.IALAttr("title"))
+		return
+	}
+	ret = val.([]string)
+	return
+}
+
+func putBlockVirtualRefKeywords(blockContent, blockID, docTitle string) (ret []string) {
+	keywords := getVirtualRefKeywords(docTitle)
+	if 1 > len(keywords) {
+		return
+	}
+
+	contentTmp := blockContent
+	if !Conf.Search.CaseSensitive {
+		contentTmp = strings.ToLower(blockContent)
+	}
+	for _, keyword := range keywords {
+		keywordTmp := keyword
+		if !Conf.Search.CaseSensitive {
+			keywordTmp = strings.ToLower(keyword)
+		}
+
+		if strings.Contains(contentTmp, keywordTmp) {
+			ret = append(ret, keyword)
+		}
+	}
+
+	if 1 > len(ret) {
+		return
+	}
+
+	ret = gulu.Str.RemoveDuplicatedElem(ret)
+	virtualBlockRefCache.Set(blockID, ret, 1)
+	return
+}
+
+func CacheVirtualBlockRefJob() {
+	virtualBlockRefCache.Del("virtual_ref")
+	if !Conf.Editor.VirtualBlockRef {
+		return
+	}
+
+	keywords := sql.QueryVirtualRefKeywords(Conf.Search.VirtualRefName, Conf.Search.VirtualRefAlias, Conf.Search.VirtualRefAnchor, Conf.Search.VirtualRefDoc)
+	virtualBlockRefCache.Set("virtual_ref", keywords, 1)
+}
+
 func processVirtualRef(n *ast.Node, unlinks *[]*ast.Node, virtualBlockRefKeywords []string, refCount map[string]int, luteEngine *lute.Lute) bool {
-	if !Conf.Editor.VirtualBlockRef || 1 > len(virtualBlockRefKeywords) {
+	if !Conf.Editor.VirtualBlockRef {
 		return false
 	}
 
@@ -43,8 +115,18 @@ func processVirtualRef(n *ast.Node, unlinks *[]*ast.Node, virtualBlockRefKeyword
 		return false
 	}
 
+	if 1 > len(virtualBlockRefKeywords) {
+		return false
+	}
+
 	content := string(n.Tokens)
-	newContent := markReplaceSpanWithSplit(content, virtualBlockRefKeywords, getMarkSpanStart(virtualBlockRefDataType), getMarkSpanEnd())
+	tmp := gulu.Str.RemoveInvisible(content)
+	tmp = strings.TrimSpace(tmp)
+	if "" == tmp {
+		return false
+	}
+
+	newContent := markReplaceSpanWithSplit(content, virtualBlockRefKeywords, search.GetMarkSpanStart(search.VirtualBlockRefDataType), search.GetMarkSpanEnd())
 	if content != newContent {
 		// 虚拟引用排除命中自身块命名和别名的情况 https://github.com/siyuan-note/siyuan/issues/3185
 		var blockKeys []string
@@ -55,7 +137,7 @@ func processVirtualRef(n *ast.Node, unlinks *[]*ast.Node, virtualBlockRefKeyword
 			blockKeys = append(blockKeys, alias)
 		}
 		if 0 < len(blockKeys) {
-			keys := gulu.Str.SubstringsBetween(newContent, getMarkSpanStart(virtualBlockRefDataType), getMarkSpanEnd())
+			keys := gulu.Str.SubstringsBetween(newContent, search.GetMarkSpanStart(search.VirtualBlockRefDataType), search.GetMarkSpanEnd())
 			for _, k := range keys {
 				if gulu.Str.Contains(k, blockKeys) {
 					return true
@@ -83,7 +165,10 @@ func getVirtualRefKeywords(docName string) (ret []string) {
 		return
 	}
 
-	ret = sql.QueryVirtualRefKeywords(Conf.Search.VirtualRefName, Conf.Search.VirtualRefAlias, Conf.Search.VirtualRefAnchor, Conf.Search.VirtualRefDoc)
+	if val, ok := virtualBlockRefCache.Get("virtual_ref"); ok {
+		ret = val.([]string)
+	}
+
 	if "" != strings.TrimSpace(Conf.Editor.VirtualBlockRefInclude) {
 		include := strings.ReplaceAll(Conf.Editor.VirtualBlockRefInclude, "\\,", "__comma@sep__")
 		includes := strings.Split(include, ",")
@@ -129,6 +214,7 @@ func getVirtualRefKeywords(docName string) (ret []string) {
 	ret = gulu.Str.ExcludeElem(ret, []string{docName})
 	ret = prepareMarkKeywords(ret)
 
+	// 在 设置 - 搜索 中分别增加虚拟引用和反链提及 `关键字数量限制` https://github.com/siyuan-note/siyuan/issues/6603
 	if Conf.Search.VirtualRefKeywordsLimit < len(ret) {
 		ret = ret[:Conf.Search.VirtualRefKeywordsLimit]
 	}

kernel/search/mark.go

@@ -17,6 +17,7 @@
 package search
 
 import (
+	"fmt"
 	"regexp"
 	"strings"
 	"unicode/utf8"
@@ -99,3 +100,16 @@ func EncloseHighlighting(text string, keywords []string, openMark, closeMark str
 	}
 	return
 }
+
+const (
+	MarkDataType            = "search-mark"
+	VirtualBlockRefDataType = "virtual-block-ref"
+)
+
+func GetMarkSpanStart(dataType string) string {
+	return fmt.Sprintf("<span data-type=\"%s\">", dataType)
+}
+
+func GetMarkSpanEnd() string {
+	return "</span>"
+}

kernel/sql/block_ref_query.go

@@ -29,11 +29,6 @@ import (
 )
 
 func QueryVirtualRefKeywords(name, alias, anchor, doc bool) (ret []string) {
-	ret, ok := getVirtualRefKeywordsCache()
-	if ok {
-		return ret
-	}
-
 	if name {
 		ret = append(ret, queryNames()...)
 	}
@@ -50,7 +45,6 @@ func QueryVirtualRefKeywords(name, alias, anchor, doc bool) (ret []string) {
 	sort.SliceStable(ret, func(i, j int) bool {
 		return len(ret[i]) >= len(ret[j])
 	})
-	setVirtualRefKeywords(ret)
 	return
 }
 

kernel/sql/cache.go

@@ -28,28 +28,29 @@ import (
 	"github.com/siyuan-note/logging"
 )
 
-var memCache, _ = ristretto.NewCache(&ristretto.Config{
+var cacheDisabled = true
+
+func enableCache() {
+	cacheDisabled = false
+}
+
+func disableCache() {
+	cacheDisabled = true
+}
+
+var blockCache, _ = ristretto.NewCache(&ristretto.Config{
 	NumCounters: 10240,
 	MaxCost:     1024,
 	BufferItems: 64,
 })
-var disabled = true
 
-func enableCache() {
-	disabled = false
-}
-
-func disableCache() {
-	disabled = true
-}
-
-func ClearBlockCache() {
-	memCache.Clear()
+func ClearCache() {
+	blockCache.Clear()
 	debug.FreeOSMemory()
 }
 
 func putBlockCache(block *Block) {
-	if disabled {
+	if cacheDisabled {
 		return
 	}
 
@@ -58,15 +59,15 @@ func putBlockCache(block *Block) {
 		logging.LogErrorf("clone block failed: %v", err)
 		return
 	}
-	memCache.Set(cloned.ID, cloned, 1)
+	blockCache.Set(cloned.ID, cloned, 1)
 }
 
 func getBlockCache(id string) (ret *Block) {
-	if disabled {
+	if cacheDisabled {
 		return
 	}
 
-	b, _ := memCache.Get(id)
+	b, _ := blockCache.Get(id)
 	if nil != b {
 		ret = b.(*Block)
 	}
@@ -74,41 +75,10 @@ func getBlockCache(id string) (ret *Block) {
 }
 
 func removeBlockCache(id string) {
-	memCache.Del(id)
+	blockCache.Del(id)
 	removeRefCacheByDefID(id)
 }
 
-var virtualRefKeywordsCacheTime = time.Now()
-
-func getVirtualRefKeywordsCache() ([]string, bool) {
-	if disabled {
-		return nil, false
-	}
-
-	// 虚拟引用关键字缓存调整为 10 分钟 https://github.com/siyuan-note/siyuan/issues/6602
-	if 10 < time.Now().Sub(virtualRefKeywordsCacheTime).Minutes() {
-		ClearVirtualRefKeywords()
-		return nil, false
-	}
-
-	if val, ok := memCache.Get("virtual_ref"); ok {
-		return val.([]string), true
-	}
-	return nil, false
-}
-
-func setVirtualRefKeywords(keywords []string) {
-	if disabled {
-		return
-	}
-
-	memCache.Set("virtual_ref", keywords, 1)
-}
-
-func ClearVirtualRefKeywords() {
-	memCache.Del("virtual_ref")
-}
-
 var defIDRefsCache = gcache.New(30*time.Minute, 5*time.Minute) // [defBlockID]map[refBlockID]*Ref
 
 func GetRefsCacheByDefID(defID string) (ret []*Ref) {

kernel/sql/database.go

@@ -67,7 +67,7 @@ func InitDatabase(forceRebuild bool) (err error) {
 	initDatabaseLock.Lock()
 	defer initDatabaseLock.Unlock()
 
-	ClearBlockCache()
+	ClearCache()
 	disableCache()
 	defer enableCache()
 
@@ -888,7 +888,7 @@ func deleteBlocksByBoxTx(tx *sql.Tx, box string) (err error) {
 	if err = execStmtTx(tx, stmt, box); nil != err {
 		return
 	}
-	ClearBlockCache()
+	ClearCache()
 	return
 }
 
@@ -1012,7 +1012,7 @@ func deleteByRootID(tx *sql.Tx, rootID string, context map[string]interface{}) (
 	if err = execStmtTx(tx, stmt, rootID); nil != err {
 		return
 	}
-	ClearBlockCache()
+	ClearCache()
 	eventbus.Publish(eventbus.EvtSQLDeleteBlocks, context, rootID)
 	return
 }
@@ -1048,7 +1048,7 @@ func batchDeleteByRootIDs(tx *sql.Tx, rootIDs []string, context map[string]inter
 	if err = execStmtTx(tx, stmt); nil != err {
 		return
 	}
-	ClearBlockCache()
+	ClearCache()
 	eventbus.Publish(eventbus.EvtSQLDeleteBlocks, context, fmt.Sprintf("%d", len(rootIDs)))
 	return
 }
@@ -1082,7 +1082,7 @@ func batchDeleteByPathPrefix(tx *sql.Tx, boxID, pathPrefix string) (err error) {
 	if err = execStmtTx(tx, stmt, boxID, pathPrefix+"%"); nil != err {
 		return
 	}
-	ClearBlockCache()
+	ClearCache()
 	return
 }
 
@@ -1099,7 +1099,7 @@ func batchUpdateHPath(tx *sql.Tx, boxID, rootID, oldHPath, newHPath string) (err
 	if err = execStmtTx(tx, stmt, newHPath, boxID, rootID, oldHPath); nil != err {
 		return
 	}
-	ClearBlockCache()
+	ClearCache()
 	return
 }