:recycle: Improve database loading performance https://github.com/siyuan-note/siyuan/issues/12818

kernel/filesys/tree.go

@@ -25,11 +25,13 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/88250/lute"
 	"github.com/88250/lute/parse"
 	"github.com/88250/lute/render"
 	jsoniter "github.com/json-iterator/go"
+	"github.com/panjf2000/ants/v2"
 	"github.com/siyuan-note/filelock"
 	"github.com/siyuan-note/logging"
 	"github.com/siyuan-note/siyuan/kernel/cache"
@@ -53,7 +55,7 @@ func LoadTrees(ids []string) (ret map[string]*parse.Tree) {
 		blockIDs[bt.RootID] = append(blockIDs[bt.RootID], bt.ID)
 	}
 
-	trees, errs := BatchLoadTrees(boxIDs, paths, luteEngine)
+	trees, errs := batchLoadTrees(boxIDs, paths, luteEngine)
 	for i := range trees {
 		tree := trees[i]
 		err := errs[i]
@@ -82,31 +84,37 @@ func LoadTree(boxID, p string, luteEngine *lute.Lute) (ret *parse.Tree, err erro
 	return
 }
 
-func BatchLoadTrees(boxIDs, paths []string, luteEngine *lute.Lute) (ret []*parse.Tree, errs []error) {
-	var wg sync.WaitGroup
+func batchLoadTrees(boxIDs, paths []string, luteEngine *lute.Lute) (ret []*parse.Tree, errs []error) {
+	waitGroup := sync.WaitGroup{}
 	lock := sync.Mutex{}
 	loaded := map[string]bool{}
+
+	start := time.Now()
+	p, _ := ants.NewPoolWithFunc(8, func(arg interface{}) {
+		defer waitGroup.Done()
+
+		i := arg.(int)
+		boxID := boxIDs[i]
+		path := paths[i]
+		tree, err := LoadTree(boxID, path, luteEngine)
+		lock.Lock()
+		ret = append(ret, tree)
+		errs = append(errs, err)
+		lock.Unlock()
+	})
 	for i := range paths {
 		if loaded[boxIDs[i]+paths[i]] {
 			continue
 		}
 
 		loaded[boxIDs[i]+paths[i]] = true
-		wg.Add(1)
-
-		go func(i int) {
-			defer wg.Done()
-
-			boxID := boxIDs[i]
-			path := paths[i]
-			tree, err := LoadTree(boxID, path, luteEngine)
-			lock.Lock()
-			ret = append(ret, tree)
-			errs = append(errs, err)
-			lock.Unlock()
-		}(i)
+
+		waitGroup.Add(1)
+		p.Invoke(i)
 	}
-	wg.Wait()
+	waitGroup.Wait()
+	p.Release()
+	logging.LogInfof("batch load trees [%d] cost [%s]", len(paths), time.Since(start))
 	return
 }
 

kernel/sql/av.go

@@ -176,6 +176,16 @@ func RenderAttributeViewTable(attrView *av.AttributeView, view *av.View, query s
 		ret.Rows = append(ret.Rows, &tableRow)
 	}
 
+	// 批量获取块属性以提升性能
+	var ialIDs []string
+	for _, row := range ret.Rows {
+		block := row.GetBlockValue()
+		if nil != block && !block.IsDetached {
+			ialIDs = append(ialIDs, row.ID)
+		}
+	}
+	ials := BatchGetBlockAttrs(ialIDs)
+
 	// 渲染自动生成的列值，比如关联列、汇总列、创建时间列和更新时间列
 	for _, row := range ret.Rows {
 		for _, cell := range row.Cells {
@@ -266,11 +276,11 @@ func RenderAttributeViewTable(attrView *av.AttributeView, view *av.View, query s
 				keyValues = append(keyValues, &av.KeyValues{Key: createdKey, Values: []*av.Value{{ID: cell.Value.ID, KeyID: createdKey.ID, BlockID: row.ID, Type: av.KeyTypeCreated, Created: cell.Value.Created}}})
 				rows[row.ID] = keyValues
 			case av.KeyTypeUpdated: // 渲染更新时间
-				ial := map[string]string{}
-				block := row.GetBlockValue()
-				if nil != block && !block.IsDetached {
-					ial = GetBlockAttrs(row.ID)
+				ial := ials[row.ID]
+				if nil == ial {
+					ial = map[string]string{}
 				}
+				block := row.GetBlockValue()
 				updatedStr := ial["updated"]
 				if "" == updatedStr && nil != block {
 					cell.Value.Updated = av.NewFormattedValueUpdated(block.Block.Updated, 0, av.UpdatedFormatNone)
@@ -302,10 +312,9 @@ func RenderAttributeViewTable(attrView *av.AttributeView, view *av.View, query s
 			switch cell.ValueType {
 			case av.KeyTypeTemplate: // 渲染模板列
 				keyValues := rows[row.ID]
-				ial := map[string]string{}
-				block := row.GetBlockValue()
-				if nil != block && !block.IsDetached {
-					ial = GetBlockAttrs(row.ID)
+				ial := ials[row.ID]
+				if nil == ial {
+					ial = map[string]string{}
 				}
 				content, renderErr := RenderTemplateCol(ial, keyValues, cell.Value.Template.Content)
 				cell.Value.Template.Content = content