0ct0pu5/siyuan
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

♻️ Improve database loading performance https://github.com/siyuan-note/siyuan/issues/12818

Browse source
This commit is contained in:
Daniel 2024-10-17 23:31:54 +08:00
parent da64370496
commit c42064ec0b
No known key found for this signature in database
GPG key ID: 86211BA83DF03017
9 changed files with 77 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
kernel/model/attribute_view.go
View file

@ -493,7 +493,7 @@ func GetBlockAttributeViewKeys(blockID string) (ret []*BlockAttributeViewKeys) {
waitForSyncingStorages()
ret = []*BlockAttributeViewKeys{}
attrs := GetBlockAttrsWithoutWaitWriting(blockID)
attrs := sql.GetBlockAttrsWithoutWaitWriting(blockID)
avs := attrs[av.NodeAttrNameAvs]
if "" == avs {
return
@ -631,7 +631,7 @@ func GetBlockAttributeViewKeys(blockID string) (ret []*BlockAttributeViewKeys) {
kv.Values[0].Created = av.NewFormattedValueCreated(time.Now().UnixMilli(), 0, av.CreatedFormatNone)
}
case av.KeyTypeUpdated:
ial := GetBlockAttrsWithoutWaitWriting(blockID)
ial := sql.GetBlockAttrsWithoutWaitWriting(blockID)
updatedStr := ial["updated"]
updated, parseErr := time.ParseInLocation("20060102150405", updatedStr, time.Local)
if nil == parseErr {
@ -655,7 +655,7 @@ func GetBlockAttributeViewKeys(blockID string) (ret []*BlockAttributeViewKeys) {
ial := map[string]string{}
block := av.GetKeyBlockValue(keyValues)
if nil != block && !block.IsDetached {
ial = GetBlockAttrsWithoutWaitWriting(block.BlockID)
ial = sql.GetBlockAttrsWithoutWaitWriting(block.BlockID)
}
if nil == kv.Values[0].Template {
@ -965,7 +965,7 @@ func renderAttributeView(attrView *av.AttributeView, viewID, query string, page,
}
view.Table.Sorts = tmpSorts
viewable = sql.RenderAttributeViewTable(attrView, view, query, GetBlockAttrsWithoutWaitWriting)
viewable = sql.RenderAttributeViewTable(attrView, view, query)
}
viewable.FilterRows(attrView)
@ -2008,7 +2008,7 @@ func addAttributeViewBlock(now int64, avID, blockID, previousBlockID, addingBloc
// 如果存在过滤条件，则将过滤条件应用到新添加的块上
view, _ := getAttrViewViewByBlockID(attrView, blockID)
if nil != view && 0 < len(view.Table.Filters) && !ignoreFillFilter {
viewable := sql.RenderAttributeViewTable(attrView, view, "", GetBlockAttrsWithoutWaitWriting)
viewable := sql.RenderAttributeViewTable(attrView, view, "")
viewable.FilterRows(attrView)
viewable.SortRows(attrView)

46
kernel/model/blockial.go
View file

@ -24,11 +24,9 @@ import (
"github.com/88250/gulu"
"github.com/88250/lute/ast"
"github.com/88250/lute/html"
"github.com/88250/lute/lex"
"github.com/88250/lute/parse"
"github.com/araddon/dateparse"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/sql"
@ -67,7 +65,7 @@ func SetBlockReminder(id string, timed string) (err error) {
if ast.NodeDocument != node.Type && node.IsContainerBlock() {
node = treenode.FirstLeafBlock(node)
}
content := sql.NodeStaticContent(node, nil, false, false, false, GetBlockAttrsWithoutWaitWriting)
content := sql.NodeStaticContent(node, nil, false, false, false)
content = gulu.Str.SubStr(content, 128)
err = SetCloudBlockReminder(id, content, timedMills)
if err != nil {
@ -297,7 +295,7 @@ func BatchGetBlockAttrs(ids []string) (ret map[string]map[string]string) {
continue
}
ret[id] = getBlockAttrs0(id, tree)
ret[id] = sql.GetBlockAttrs0(id, tree)
cache.PutBlockIAL(id, ret[id])
}
return
@ -312,45 +310,7 @@ func GetBlockAttrs(id string) (ret map[string]string) {
WaitForWritingFiles()
ret = getBlockAttrs(id)
ret = sql.GetBlockAttrs(id)
cache.PutBlockIAL(id, ret)
return
}
func GetBlockAttrsWithoutWaitWriting(id string) (ret map[string]string) {
ret = map[string]string{}
if cached := cache.GetBlockIAL(id); nil != cached {
ret = cached
return
}
ret = getBlockAttrs(id)
cache.PutBlockIAL(id, ret)
return
}
func getBlockAttrs(id string) (ret map[string]string) {
ret = map[string]string{}
tree, err := LoadTreeByBlockID(id)
if err != nil {
return
}
ret = getBlockAttrs0(id, tree)
return
}
func getBlockAttrs0(id string, tree *parse.Tree) (ret map[string]string) {
ret = map[string]string{}
node := treenode.GetNodeInTree(tree, id)
if nil == node {
logging.LogWarnf("block [%s] not found", id)
return
}
for _, kv := range node.KramdownIAL {
ret[kv[0]] = html.UnescapeAttrVal(kv[1])
}
return
}

6
kernel/model/export.go
View file

@ -77,7 +77,7 @@ func ExportAv2CSV(avID, blockID string) (zipPath string, err error) {
}
name := util.FilterFileName(getAttrViewName(attrView))
table := sql.RenderAttributeViewTable(attrView, view, "", GetBlockAttrsWithoutWaitWriting)
table := sql.RenderAttributeViewTable(attrView, view, "")
// 遵循视图过滤和排序规则 Use filtering and sorting of current view settings when exporting database blocks https://github.com/siyuan-note/siyuan/issues/10474
table.FilterRows(attrView)
@ -2256,7 +2256,7 @@ func exportTree(tree *parse.Tree, wysiwyg, keepFold, avHiddenCol bool,
return ast.WalkContinue
}
table := sql.RenderAttributeViewTable(attrView, view, "", GetBlockAttrsWithoutWaitWriting)
table := sql.RenderAttributeViewTable(attrView, view, "")
// 遵循视图过滤和排序规则 Use filtering and sorting of current view settings when exporting database blocks https://github.com/siyuan-note/siyuan/issues/10474
table.FilterRows(attrView)
@ -3061,7 +3061,7 @@ func getDestViewVal(attrView *av.AttributeView, keyID, blockID string) *av.Table
return nil
}
destTable := sql.RenderAttributeViewTable(destAv, destView, "", GetBlockAttrsWithoutWaitWriting)
destTable := sql.RenderAttributeViewTable(destAv, destView, "")
if nil == destTable {
return nil
}

4
kernel/model/render.go
View file

@ -97,7 +97,7 @@ func renderBlockText(node *ast.Node, excludeTypes []string) (ret string) {
return
}
ret = sql.NodeStaticContent(node, excludeTypes, false, false, false, GetBlockAttrsWithoutWaitWriting)
ret = sql.NodeStaticContent(node, excludeTypes, false, false, false)
ret = strings.TrimSpace(ret)
ret = strings.ReplaceAll(ret, "\n", "")
ret = util.EscapeHTML(ret)
@ -160,7 +160,7 @@ func renderBlockContentByNodes(nodes []*ast.Node) string {
buf := bytes.Buffer{}
for _, n := range subNodes {
buf.WriteString(sql.NodeStaticContent(n, nil, false, false, false, GetBlockAttrsWithoutWaitWriting))
buf.WriteString(sql.NodeStaticContent(n, nil, false, false, false))
}
return buf.String()
}

2
kernel/model/template.go
View file

@ -314,7 +314,7 @@ func RenderTemplate(p, id string, preview bool) (tree *parse.Tree, dom string, e
return ast.WalkContinue
}
table := sql.RenderAttributeViewTable(attrView, view, "", GetBlockAttrsWithoutWaitWriting)
table := sql.RenderAttributeViewTable(attrView, view, "")
var aligns []int
for range table.Columns {

2
kernel/model/virutalref.go
View file

@ -52,7 +52,7 @@ func getBlockVirtualRefKeywords(root *ast.Node) (ret []string) {
return ast.WalkContinue
}
content := sql.NodeStaticContent(n, nil, false, false, false, GetBlockAttrsWithoutWaitWriting)
content := sql.NodeStaticContent(n, nil, false, false, false)
buf.WriteString(content)
return ast.WalkContinue
})

19
kernel/sql/av.go
View file

@ -27,23 +27,11 @@ import (
"github.com/88250/lute/ast"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/av"
"github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
)
func RenderAttributeViewTable(attrView *av.AttributeView, view *av.View, query string,
GetBlockAttrsWithoutWaitWriting func(id string) (ret map[string]string)) (ret *av.Table) {
if nil == GetBlockAttrsWithoutWaitWriting {
GetBlockAttrsWithoutWaitWriting = func(id string) (ret map[string]string) {
ret = cache.GetBlockIAL(id)
if nil == ret {
ret = map[string]string{}
}
return
}
}
func RenderAttributeViewTable(attrView *av.AttributeView, view *av.View, query string) (ret *av.Table) {
ret = &av.Table{
ID: view.ID,
Icon: view.Icon,
@ -568,8 +556,7 @@ func FillAttributeViewTableCellNilValue(tableCell *av.TableCell, rowID, colID st
}
}
func getAttributeViewContent(avID string,
GetBlockAttrsWithoutWaitWriting func(id string) (ret map[string]string)) (content string) {
func getAttributeViewContent(avID string) (content string) {
if "" == avID {
return
}
@ -605,7 +592,7 @@ func getAttributeViewContent(avID string,
return
}
table := RenderAttributeViewTable(attrView, view, "", GetBlockAttrsWithoutWaitWriting)
table := RenderAttributeViewTable(attrView, view, "")
for _, col := range table.Columns {
buf.WriteString(col.Name)
buf.WriteByte(' ')

61
kernel/sql/block.go
View file

@ -19,6 +19,8 @@ package sql
import (
"bytes"
"database/sql"
"github.com/88250/lute/parse"
"github.com/siyuan-note/logging"
"strings"
"github.com/88250/gulu"
@ -104,7 +106,6 @@ func indexNode(tx *sql.Tx, id string) (err error) {
return
}
luteEngine := util.NewLute()
tree, _ := filesys.LoadTree(bt.BoxID, bt.Path, luteEngine)
if nil == tree {
return
@ -115,7 +116,7 @@ func indexNode(tx *sql.Tx, id string) (err error) {
return
}
content := NodeStaticContent(node, nil, true, indexAssetPath, true, nil)
content := NodeStaticContent(node, nil, true, indexAssetPath, true)
stmt := "UPDATE blocks SET content = ? WHERE id = ?"
if err = execStmtTx(tx, stmt, content, id); err != nil {
tx.Rollback()
@ -136,15 +137,14 @@ func indexNode(tx *sql.Tx, id string) (err error) {
return
}
func NodeStaticContent(node *ast.Node, excludeTypes []string, includeTextMarkATitleURL, includeAssetPath, fullAttrView bool,
GetBlockAttrsWithoutWaitWriting func(id string) (ret map[string]string)) string {
func NodeStaticContent(node *ast.Node, excludeTypes []string, includeTextMarkATitleURL, includeAssetPath, fullAttrView bool) string {
if nil == node {
return ""
}
if ast.NodeAttributeView == node.Type {
if fullAttrView {
return getAttributeViewContent(node.AttributeViewID, GetBlockAttrsWithoutWaitWriting)
return getAttributeViewContent(node.AttributeViewID)
}
ret, _ := av.GetAttributeViewName(node.AttributeViewID)
@ -274,3 +274,54 @@ func nodeStaticContent(node *ast.Node, excludeTypes []string, includeTextMarkATi
// Improve search and replace for spaces https://github.com/siyuan-note/siyuan/issues/10231
return buf.String()
}
func GetBlockAttrsWithoutWaitWriting(id string) (ret map[string]string) {
ret = map[string]string{}
if cached := cache.GetBlockIAL(id); nil != cached {
ret = cached
return
}
ret = GetBlockAttrs(id)
cache.PutBlockIAL(id, ret)
return
}
func GetBlockAttrs(id string) (ret map[string]string) {
ret = map[string]string{}
tree := loadTreeByBlockID(id)
if nil == tree {
return
}
ret = GetBlockAttrs0(id, tree)
return
}
func GetBlockAttrs0(id string, tree *parse.Tree) (ret map[string]string) {
ret = map[string]string{}
node := treenode.GetNodeInTree(tree, id)
if nil == node {
logging.LogWarnf("block [%s] not found", id)
return
}
for _, kv := range node.KramdownIAL {
ret[kv[0]] = html.UnescapeAttrVal(kv[1])
}
return
}
func loadTreeByBlockID(id string) (ret *parse.Tree) {
bt := treenode.GetBlockTree(id)
if nil == bt {
return
}
ret, err := filesys.LoadTree(bt.BoxID, bt.Path, luteEngine)
if nil != err {
return
}
return
}

6
kernel/sql/database.go
View file

@ -828,13 +828,13 @@ func buildBlockFromNode(n *ast.Node, tree *parse.Tree) (block *Block, attributes
if !treenode.IsNodeOCRed(n) {
util.PushNodeOCRQueue(n)
}
content = NodeStaticContent(n, nil, true, indexAssetPath, true, nil)
content = NodeStaticContent(n, nil, true, indexAssetPath, true)
fc := treenode.FirstLeafBlock(n)
if !treenode.IsNodeOCRed(fc) {
util.PushNodeOCRQueue(fc)
}
fcontent = NodeStaticContent(fc, nil, true, false, true, nil)
fcontent = NodeStaticContent(fc, nil, true, false, true)
parentID = n.Parent.ID
if h := heading(n); nil != h { // 如果在标题块下方，则将标题块作为父节点
@ -846,7 +846,7 @@ func buildBlockFromNode(n *ast.Node, tree *parse.Tree) (block *Block, attributes
if !treenode.IsNodeOCRed(n) {
util.PushNodeOCRQueue(n)
}
content = NodeStaticContent(n, nil, true, indexAssetPath, true, nil)
content = NodeStaticContent(n, nil, true, indexAssetPath, true)
parentID = n.Parent.ID
if h := heading(n); nil != h {