Merge remote-tracking branch 'origin/dev' into dev

kernel/api/router.go

@@ -158,6 +158,7 @@ func ServeAPI(ginServer *gin.Engine) {
 	ginServer.Handle("POST", "/api/search/findReplace", model.CheckAuth, findReplace)
 	ginServer.Handle("POST", "/api/search/fullTextSearchAssetContent", model.CheckAuth, fullTextSearchAssetContent)
 	ginServer.Handle("POST", "/api/search/getAssetContent", model.CheckAuth, getAssetContent)
+	ginServer.Handle("POST", "/api/search/listInvalidBlockRefs", model.CheckAuth, listInvalidBlockRefs)
 
 	ginServer.Handle("POST", "/api/block/getBlockInfo", model.CheckAuth, getBlockInfo)
 	ginServer.Handle("POST", "/api/block/getBlockDOM", model.CheckAuth, getBlockDOM)

kernel/api/search.go

@@ -26,6 +26,40 @@ import (
 	"github.com/siyuan-note/siyuan/kernel/util"
 )
 
+func listInvalidBlockRefs(c *gin.Context) {
+	ret := gulu.Ret.NewResult()
+	defer c.JSON(http.StatusOK, ret)
+
+	arg, ok := util.JsonArg(c, ret)
+	if !ok {
+		return
+	}
+
+	page := 1
+	if nil != arg["page"] {
+		page = int(arg["page"].(float64))
+	}
+	if 0 >= page {
+		page = 1
+	}
+
+	pageSize := 32
+	if nil != arg["pageSize"] {
+		pageSize = int(arg["pageSize"].(float64))
+	}
+	if 0 >= pageSize {
+		pageSize = 32
+	}
+
+	blocks, matchedBlockCount, matchedRootCount, pageCount := model.ListInvalidBlockRefs(page, pageSize)
+	ret.Data = map[string]interface{}{
+		"blocks":            blocks,
+		"matchedBlockCount": matchedBlockCount,
+		"matchedRootCount":  matchedRootCount,
+		"pageCount":         pageCount,
+	}
+}
+
 func getAssetContent(c *gin.Context) {
 	ret := gulu.Ret.NewResult()
 	defer c.JSON(http.StatusOK, ret)

kernel/model/search.go

@@ -50,6 +50,135 @@ import (
 	"github.com/xrash/smetrics"
 )
 
+func ListInvalidBlockRefs(page, pageSize int) (ret []*Block, matchedBlockCount, matchedRootCount, pageCount int) {
+	refBlockMap := map[string][]string{}
+	blockMap := map[string]bool{}
+	var invalidBlockIDs []string
+	notebooks, err := ListNotebooks()
+	if nil != err {
+		return
+	}
+	luteEngine := util.NewLute()
+	for _, notebook := range notebooks {
+		pages := pagedPaths(filepath.Join(util.DataDir, notebook.ID), 32)
+		for _, paths := range pages {
+			var trees []*parse.Tree
+			for _, localPath := range paths {
+				tree, loadTreeErr := loadTree(localPath, luteEngine)
+				if nil != loadTreeErr {
+					continue
+				}
+				trees = append(trees, tree)
+			}
+			for _, tree := range trees {
+				ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
+					if entering {
+						if n.IsBlock() {
+							blockMap[n.ID] = true
+							return ast.WalkContinue
+						}
+
+						if ast.NodeTextMark == n.Type {
+							if n.IsTextMarkType("a") {
+								if strings.HasPrefix(n.TextMarkAHref, "siyuan://blocks/") {
+									defID := strings.TrimPrefix(n.TextMarkAHref, "siyuan://blocks/")
+									if strings.Contains(defID, "?") {
+										defID = strings.Split(defID, "?")[0]
+									}
+									refID := treenode.ParentBlock(n).ID
+									if defIDs := refBlockMap[refID]; 1 > len(defIDs) {
+										refBlockMap[refID] = []string{defID}
+									} else {
+										refBlockMap[refID] = append(defIDs, defID)
+									}
+								}
+							} else if n.IsTextMarkType("block-ref") {
+								defID := n.TextMarkBlockRefID
+								refID := treenode.ParentBlock(n).ID
+								if defIDs := refBlockMap[refID]; 1 > len(defIDs) {
+									refBlockMap[refID] = []string{defID}
+								} else {
+									refBlockMap[refID] = append(defIDs, defID)
+								}
+							}
+						}
+					}
+					return ast.WalkContinue
+				})
+			}
+		}
+	}
+
+	invalidDefIDs := map[string]bool{}
+	for _, refDefIDs := range refBlockMap {
+		for _, defID := range refDefIDs {
+			invalidDefIDs[defID] = true
+		}
+	}
+
+	var toRemoves []string
+	for defID, _ := range invalidDefIDs {
+		if _, ok := blockMap[defID]; ok {
+			toRemoves = append(toRemoves, defID)
+		}
+	}
+	for _, toRemove := range toRemoves {
+		delete(invalidDefIDs, toRemove)
+	}
+
+	toRemoves = nil
+	for refID, defIDs := range refBlockMap {
+		var tmp []string
+		for _, defID := range defIDs {
+			if _, ok := invalidDefIDs[defID]; !ok {
+				tmp = append(tmp, defID)
+			}
+		}
+
+		for _, toRemove := range tmp {
+			defIDs = gulu.Str.RemoveElem(defIDs, toRemove)
+		}
+
+		if 1 > len(defIDs) {
+			toRemoves = append(toRemoves, refID)
+		}
+	}
+	for _, toRemove := range toRemoves {
+		delete(refBlockMap, toRemove)
+	}
+
+	for refID, _ := range refBlockMap {
+		invalidBlockIDs = append(invalidBlockIDs, refID)
+	}
+	invalidBlockIDs = gulu.Str.RemoveDuplicatedElem(invalidBlockIDs)
+
+	sort.Strings(invalidBlockIDs)
+
+	start := (page - 1) * pageSize
+	end := page * pageSize
+	if end > len(invalidBlockIDs) {
+		end = len(invalidBlockIDs)
+	}
+	invalidBlockIDs = invalidBlockIDs[start:end]
+
+	sqlBlocks := sql.GetBlocks(invalidBlockIDs)
+	ret = fromSQLBlocks(&sqlBlocks, "", 36)
+	if 1 > len(ret) {
+		ret = []*Block{}
+	}
+	matchedBlockCount = len(ret)
+	rootCount := map[string]bool{}
+	for _, block := range ret {
+		if nil == block {
+			continue
+		}
+		rootCount[block.RootID] = true
+	}
+	matchedRootCount = len(rootCount)
+	pageCount = (matchedBlockCount + pageSize - 1) / pageSize
+	return
+}
+
 type EmbedBlock struct {
 	Block      *Block       `json:"block"`
 	BlockPaths []*BlockPath `json:"blockPaths"`