siyuan/kernel/model/backlink.go

// SiYuan - Build Your Eternal Digital Garden
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

package model

import (
	"bytes"
	"fmt"
	"path"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"github.com/88250/gulu"
	"github.com/88250/lute/ast"
	"github.com/88250/lute/parse"
	"github.com/emirpasic/gods/sets/hashset"
	"github.com/siyuan-note/siyuan/kernel/search"
	"github.com/siyuan-note/siyuan/kernel/sql"
	"github.com/siyuan-note/siyuan/kernel/treenode"
	"github.com/siyuan-note/siyuan/kernel/util"
)

func RefreshBacklink(id string) {
	WaitForWritingFiles()

	tx, err := sql.BeginTx()
	if nil != err {
		return
	}
	defer sql.CommitTx(tx)

	refs := sql.QueryRefsByDefID(id, false)
	trees := map[string]*parse.Tree{}
	for _, ref := range refs {
		tree := trees[ref.RootID]
		if nil == tree {
			tree, err = loadTreeByBlockID(ref.RootID)
			if nil != err {
				util.LogErrorf("refresh tree refs failed: %s", err)
				continue
			}
			trees[ref.RootID] = tree
			sql.UpsertRefs(tx, tree)
		}
	}
}

func CreateBacklink(defID, refID, refText string, isDynamic bool) (refRootID string, err error) {
	refTree, err := loadTreeByBlockID(refID)
	if nil != err {
		return "", err
	}
	refNode := treenode.GetNodeInTree(refTree, refID)
	if nil == refNode {
		return
	}
	refRootID = refTree.Root.ID

	defBlockTree := treenode.GetBlockTree(defID)
	if nil == defBlockTree {
		return
	}
	defRoot := sql.GetBlock(defBlockTree.RootID)
	if nil == defRoot {
		return
	}

	refTextLower := strings.ToLower(refText)
	defBlock := sql.QueryBlockByNameOrAlias(defRoot.ID, refText)
	if nil == defBlock {
		if strings.ToLower(defRoot.Content) == refTextLower {
			// 如果命名别名没有命中，但文档名和提及关键字匹配，则使用文档作为定义块
			defBlock = defRoot
		}
		if nil == defBlock {
			// 使用锚文本进行搜索，取第一个匹配的定义块
			if defIDs := sql.QueryBlockDefIDsByRefText(refTextLower, nil); 0 < len(defIDs) {
				if defBlock = sql.GetBlock(defIDs[0]); nil != defBlock {
					goto OK
				}
			}
		}
		if nil == defBlock {
			defBlock = sql.GetBlock(defBlockTree.ID)
		}
		if nil == defBlock {
			return
		}
		if strings.ToLower(defBlock.Content) != refTextLower {
			return
		}
	}

OK:
	luteEngine := NewLute()
	found := false
	var toRemove []*ast.Node
	ast.Walk(refNode, func(n *ast.Node, entering bool) ast.WalkStatus {
		if !entering {
			return ast.WalkContinue
		}

		if ast.NodeText != n.Type {
			return ast.WalkContinue
		}

		text := gulu.Str.FromBytes(n.Tokens)
		re := regexp.MustCompile("(?i)" + refText)
		if strings.Contains(strings.ToLower(text), refTextLower) {
			if isDynamic {
				text = re.ReplaceAllString(text, "(("+defBlock.ID+" '"+refText+"'))")
			} else {
				text = re.ReplaceAllString(text, "(("+defBlock.ID+" \""+refText+"\"))")
			}
			found = true
			subTree := parse.Inline("", []byte(text), luteEngine.ParseOptions)
			var toInsert []*ast.Node
			for newNode := subTree.Root.FirstChild.FirstChild; nil != newNode; newNode = newNode.Next {
				toInsert = append(toInsert, newNode)
			}
			for _, insert := range toInsert {
				n.InsertBefore(insert)
			}
			toRemove = append(toRemove, n)
		}
		return ast.WalkContinue
	})

	for _, n := range toRemove {
		n.Unlink()
	}

	if found {
		refTree.Root.SetIALAttr("updated", util.CurrentTimeSecondsStr())
		if err = indexWriteJSONQueue(refTree); nil != err {
			return "", err
		}
		IncWorkspaceDataVer()
	}
	sql.WaitForWritingDatabase()
	return
}

func BuildTreeBacklink(id, keyword, mentionKeyword string, beforeLen int) (boxID string, linkPaths, mentionPaths []*Path, linkRefsCount, mentionsCount int) {
	linkPaths = []*Path{}
	mentionPaths = []*Path{}

	sqlBlock := sql.GetBlock(id)
	if nil == sqlBlock {
		return
	}
	rootID := sqlBlock.RootID
	boxID = sqlBlock.Box

	var links []*Block
	refs := sql.QueryRefsByDefID(id, true)

	// 为了减少查询，组装好 IDs 后一次查出
	defSQLBlockIDs, refSQLBlockIDs := map[string]bool{}, map[string]bool{}
	var queryBlockIDs []string
	for _, ref := range refs {
		defSQLBlockIDs[ref.DefBlockID] = true
		refSQLBlockIDs[ref.BlockID] = true
		queryBlockIDs = append(queryBlockIDs, ref.DefBlockID)
		queryBlockIDs = append(queryBlockIDs, ref.BlockID)
	}
	querySQLBlocks := sql.GetBlocks(queryBlockIDs)
	defSQLBlocksCache := map[string]*sql.Block{}
	for _, defSQLBlock := range querySQLBlocks {
		if nil != defSQLBlock && defSQLBlockIDs[defSQLBlock.ID] {
			defSQLBlocksCache[defSQLBlock.ID] = defSQLBlock
		}
	}
	refSQLBlocksCache := map[string]*sql.Block{}
	for _, refSQLBlock := range querySQLBlocks {
		if nil != refSQLBlock && refSQLBlockIDs[refSQLBlock.ID] {
			refSQLBlocksCache[refSQLBlock.ID] = refSQLBlock
		}
	}

	excludeBacklinkIDs := hashset.New()
	for _, ref := range refs {
		defSQLBlock := defSQLBlocksCache[(ref.DefBlockID)]
		if nil == defSQLBlock {
			continue
		}

		refSQLBlock := refSQLBlocksCache[ref.BlockID]
		if nil == refSQLBlock {
			continue
		}
		refBlock := fromSQLBlock(refSQLBlock, "", beforeLen)
		if rootID == refBlock.RootID { // 排除当前文档内引用提及
			excludeBacklinkIDs.Add(refBlock.RootID, refBlock.ID)
		}
		defBlock := fromSQLBlock(defSQLBlock, "", beforeLen)
		if defBlock.RootID == rootID { // 当前文档的定义块
			links = append(links, defBlock)
			if ref.DefBlockID == defBlock.ID {
				defBlock.Refs = append(defBlock.Refs, refBlock)
			}
		}
	}

	for _, link := range links {
		for _, ref := range link.Refs {
			excludeBacklinkIDs.Add(ref.RootID, ref.ID)
		}
		linkRefsCount += len(link.Refs)
	}

	var linkRefs []*Block
	processedParagraphs := hashset.New()
	var paragraphParentIDs []string
	for _, link := range links {
		for _, ref := range link.Refs {
			if "NodeParagraph" == ref.Type {
				paragraphParentIDs = append(paragraphParentIDs, ref.ParentID)
			}
		}
	}
	paragraphParents := sql.GetBlocks(paragraphParentIDs)
	for _, p := range paragraphParents {
		if "i" == p.Type {
			linkRefs = append(linkRefs, fromSQLBlock(p, keyword, beforeLen))
			processedParagraphs.Add(p.ID)
		}
	}
	for _, link := range links {
		for _, ref := range link.Refs {
			if "NodeParagraph" == ref.Type {
				if processedParagraphs.Contains(ref.ParentID) {
					continue
				}
			}

			ref.DefID = link.ID
			ref.DefPath = link.Path

			content := ref.Content
			if "" != keyword {
				_, content = search.MarkText(content, keyword, beforeLen, Conf.Search.CaseSensitive)
				ref.Content = content
			}
			linkRefs = append(linkRefs, ref)
		}
	}
	linkPaths = toSubTree(linkRefs, keyword)

	mentions := buildTreeBackmention(sqlBlock, linkRefs, mentionKeyword, excludeBacklinkIDs, beforeLen)
	mentionsCount = len(mentions)
	mentionPaths = toFlatTree(mentions, 0, "backlink")
	return
}

func buildTreeBackmention(defSQLBlock *sql.Block, refBlocks []*Block, keyword string, excludeBacklinkIDs *hashset.Set, beforeLen int) (ret []*Block) {
	ret = []*Block{}

	var names, aliases []string
	var fName, rootID string
	if "d" == defSQLBlock.Type {
		if Conf.Search.BacklinkMentionName {
			names = sql.QueryBlockNamesByRootID(defSQLBlock.ID)
		}
		if Conf.Search.BacklinkMentionAlias {
			aliases = sql.QueryBlockAliases(defSQLBlock.ID)
		}
		if Conf.Search.BacklinkMentionDoc {
			fName = path.Base(defSQLBlock.HPath)
		}
		rootID = defSQLBlock.ID
	} else {
		if Conf.Search.BacklinkMentionName {
			if "" != defSQLBlock.Name {
				names = append(names, defSQLBlock.Name)
			}
		}
		if Conf.Search.BacklinkMentionAlias {
			if "" != defSQLBlock.Alias {
				aliases = strings.Split(defSQLBlock.Alias, ",")
			}
		}
		root := treenode.GetBlockTree(defSQLBlock.RootID)
		rootID = root.ID
	}

	set := hashset.New()
	for _, name := range names {
		set.Add(name)
	}
	for _, alias := range aliases {
		set.Add(alias)
	}
	if "" != fName {
		set.Add(fName)
	}

	if Conf.Search.BacklinkMentionAnchor {
		for _, refBlock := range refBlocks {
			refs := sql.QueryRefsByDefIDRefID(refBlock.DefID, refBlock.ID)
			for _, ref := range refs {
				set.Add(ref.Content)
			}
		}
	}

	var mentionKeywords []string
	for _, v := range set.Values() {
		mentionKeywords = append(mentionKeywords, v.(string))
	}
	ret = searchBackmention(mentionKeywords, keyword, excludeBacklinkIDs, rootID, beforeLen)
	return
}

func searchBackmention(mentionKeywords []string, keyword string, excludeBacklinkIDs *hashset.Set, rootID string, beforeLen int) (ret []*Block) {
	ret = []*Block{}

	if 1 > len(mentionKeywords) {
		return
	}
	sort.SliceStable(mentionKeywords, func(i, j int) bool {
		return len(mentionKeywords[i]) < len(mentionKeywords[j])
	})

	table := "blocks_fts" // 大小写敏感
	if !Conf.Search.CaseSensitive {
		table = "blocks_fts_case_insensitive"
	}

	buf := bytes.Buffer{}
	buf.WriteString("SELECT * FROM " + table + " WHERE " + table + " MATCH '{content}:(")
	for i, mentionKeyword := range mentionKeywords {
		if 511 < i { // 提及搜索最大限制 https://github.com/siyuan-note/siyuan/issues/3715
			util.PushMsg(fmt.Sprintf(Conf.Language(38), len(mentionKeywords)), 5000)
			mentionKeyword = strings.ReplaceAll(mentionKeyword, "\"", "\"\"")
			buf.WriteString("\"" + mentionKeyword + "\"")
			break
		}

		mentionKeyword = strings.ReplaceAll(mentionKeyword, "\"", "\"\"")
		buf.WriteString("\"" + mentionKeyword + "\"")
		if i < len(mentionKeywords)-1 {
			buf.WriteString(" OR ")
		}
	}
	buf.WriteString(")'")
	if "" != keyword {
		buf.WriteString(" AND MATCH '{content}:'")
		buf.WriteString("\"" + keyword + "\"")
		keyword = strings.ReplaceAll(keyword, "\"", "\"\"")
	}
	buf.WriteString(" AND root_id != '" + rootID + "'") // 不在定义块所在文档中搜索
	buf.WriteString(" AND type IN ('d', 'h', 'p', 't')")
	buf.WriteString(" ORDER BY id DESC LIMIT " + strconv.Itoa(Conf.Search.Limit))
	query := buf.String()

	sqlBlocks := sql.SelectBlocksRawStmt(query, Conf.Search.Limit)
	blocks := fromSQLBlocks(&sqlBlocks, strings.Join(mentionKeywords, search.TermSep), beforeLen)

	// 排除链接文本 https://github.com/siyuan-note/siyuan/issues/1542
	luteEngine := NewLute()
	var tmp []*Block
	for _, b := range blocks {
		tree := parse.Parse("", gulu.Str.ToBytes(b.Markdown), luteEngine.ParseOptions)
		if nil == tree {
			continue
		}

		textBuf := &bytes.Buffer{}
		ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
			if !entering || n.IsBlock() {
				return ast.WalkContinue
			}
			if ast.NodeText == n.Type || ast.NodeLinkText == n.Type {
				textBuf.Write(n.Tokens)
			}
			return ast.WalkContinue
		})

		text := textBuf.String()
		text = strings.ToLower(text)
		var contain bool
		for _, mentionKeyword := range mentionKeywords {
			if strings.Contains(text, strings.ToLower(mentionKeyword)) {
				contain = true
				break
			}
		}
		if contain {
			tmp = append(tmp, b)
		}
	}
	blocks = tmp

	mentionBlockMap := map[string]*Block{}
	for _, block := range blocks {
		mentionBlockMap[block.ID] = block

		refText := getContainStr(block.Content, mentionKeywords)
		block.RefText = refText
	}

	for _, mentionBlock := range mentionBlockMap {
		if !excludeBacklinkIDs.Contains(mentionBlock.ID) {
			ret = append(ret, mentionBlock)
		}
	}

	sort.SliceStable(ret, func(i, j int) bool {
		return ret[i].ID > ret[j].ID
	})
	return
}

func getContainStr(str string, strs []string) string {
	str = strings.ToLower(str)
	for _, s := range strs {
		if strings.Contains(str, strings.ToLower(s)) {
			return s
		}
	}
	return ""
}

// buildFullLinks 构建正向和反向链接列表。
// forwardlinks：正向链接关系 refs
// backlinks：反向链接关系 defs
func buildFullLinks(condition string) (forwardlinks, backlinks []*Block) {
	forwardlinks, backlinks = []*Block{}, []*Block{}
	defs := buildDefsAndRefs(condition)
	backlinks = append(backlinks, defs...)
	for _, def := range defs {
		for _, ref := range def.Refs {
			forwardlinks = append(forwardlinks, ref)
		}
	}
	return
}

func buildDefsAndRefs(condition string) (defBlocks []*Block) {
	defBlockMap := map[string]*Block{}
	refBlockMap := map[string]*Block{}
	defRefs := sql.DefRefs(condition)

	// 将 sql block 转为 block
	for _, row := range defRefs {
		for def, ref := range row {
			if nil == ref {
				continue
			}

			refBlock := refBlockMap[ref.ID]
			if nil == refBlock {
				refBlock = fromSQLBlock(ref, "", 0)
				refBlockMap[ref.ID] = refBlock
			}

			// ref 块自己也需要作为定义块，否则图上没有节点
			if defBlock := defBlockMap[ref.ID]; nil == defBlock {
				defBlockMap[ref.ID] = refBlock
			}

			if defBlock := defBlockMap[def.ID]; nil == defBlock {
				defBlock = fromSQLBlock(def, "", 0)
				defBlockMap[def.ID] = defBlock
			}
		}
	}

	// 组装 block.Defs 和 block.Refs 字段
	for _, row := range defRefs {
		for def, ref := range row {
			if nil == ref {
				defBlock := fromSQLBlock(def, "", 0)
				defBlockMap[def.ID] = defBlock
				continue
			}

			refBlock := refBlockMap[ref.ID]
			defBlock := defBlockMap[def.ID]
			if refBlock.ID == defBlock.ID { // 自引用
				continue
			}

			refBlock.Defs = append(refBlock.Defs, defBlock)
			defBlock.Refs = append(defBlock.Refs, refBlock)
		}
	}

	for _, def := range defBlockMap {
		defBlocks = append(defBlocks, def)
	}
	return
}