0ct0pu5/siyuan
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/dev' into dev

Browse source
This commit is contained in:
Vanessa 2024-04-10 16:19:43 +08:00
commit 86e3829d80
11 changed files with 236 additions and 87 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
kernel/av/av.go
View file

@ -431,7 +431,7 @@ func SaveAttributeView(av *AttributeView) (err error) {
}
if 0 == v.UpdatedAt {
v.UpdatedAt = v.CreatedAt + 1000
v.UpdatedAt = v.CreatedAt
}
}
}

75
kernel/av/mirror.go
View file

@ -6,6 +6,7 @@ import (
"sync"
"github.com/88250/gulu"
"github.com/88250/lute/ast"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/util"
@ -16,6 +17,30 @@ var (
AttributeViewBlocksLock = sync.Mutex{}
)
func GetBlockRels() (ret map[string][]string) {
AttributeViewBlocksLock.Lock()
defer AttributeViewBlocksLock.Unlock()
ret = map[string][]string{}
blocks := filepath.Join(util.DataDir, "storage", "av", "blocks.msgpack")
if !filelock.IsExist(blocks) {
return
}
data, err := filelock.ReadFile(blocks)
if nil != err {
logging.LogErrorf("read attribute view blocks failed: %s", err)
return
}
if err = msgpack.Unmarshal(data, &ret); nil != err {
logging.LogErrorf("unmarshal attribute view blocks failed: %s", err)
return
}
return
}
func IsMirror(avID string) bool {
AttributeViewBlocksLock.Lock()
defer AttributeViewBlocksLock.Unlock()
@ -86,6 +111,56 @@ func RemoveBlockRel(avID, blockID string) {
}
}
func BatchUpsertBlockRel(nodes []*ast.Node) {
AttributeViewBlocksLock.Lock()
defer AttributeViewBlocksLock.Unlock()
avBlocks := map[string][]string{}
blocks := filepath.Join(util.DataDir, "storage", "av", "blocks.msgpack")
if !filelock.IsExist(blocks) {
if err := os.MkdirAll(filepath.Dir(blocks), 0755); nil != err {
logging.LogErrorf("create attribute view dir failed: %s", err)
return
}
} else {
data, err := filelock.ReadFile(blocks)
if nil != err {
logging.LogErrorf("read attribute view blocks failed: %s", err)
return
}
if err = msgpack.Unmarshal(data, &avBlocks); nil != err {
logging.LogErrorf("unmarshal attribute view blocks failed: %s", err)
return
}
}
for _, n := range nodes {
if ast.NodeAttributeView != n.Type {
continue
}
if "" == n.AttributeViewID || "" == n.ID {
continue
}
blockIDs := avBlocks[n.AttributeViewID]
blockIDs = append(blockIDs, n.ID)
blockIDs = gulu.Str.RemoveDuplicatedElem(blockIDs)
avBlocks[n.AttributeViewID] = blockIDs
}
data, err := msgpack.Marshal(avBlocks)
if nil != err {
logging.LogErrorf("marshal attribute view blocks failed: %s", err)
return
}
if err = filelock.WriteFile(blocks, data); nil != err {
logging.LogErrorf("write attribute view blocks failed: %s", err)
return
}
}
func UpsertBlockRel(avID, blockID string) {
AttributeViewBlocksLock.Lock()
defer AttributeViewBlocksLock.Unlock()

8
kernel/av/sort.go
View file

@ -74,9 +74,9 @@ func (value *Value) Compare(other *Value, attrView *AttributeView) int {
return 0
} else {
if !other.Number.IsNotEmpty {
return 0
return 1
}
return 1
return 0
}
}
case KeyTypeDate:
@ -94,9 +94,9 @@ func (value *Value) Compare(other *Value, attrView *AttributeView) int {
return 0
} else {
if !other.Date.IsNotEmpty {
return 0
return 1
}
return 1
return 0
}
}
case KeyTypeCreated:

79
kernel/model/attribute_view.go
View file

@ -36,6 +36,7 @@ import (
"github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
"github.com/xrash/smetrics"
)
func SetDatabaseBlockView(blockID, viewID string) (err error) {
@ -194,16 +195,20 @@ func SearchAttributeView(keyword string) (ret []*SearchAttributeViewResult) {
ret = []*SearchAttributeViewResult{}
keyword = strings.TrimSpace(keyword)
avs := map[string]string{}
type result struct {
AvID string
AvName string
AvUpdated int64
Score float64
}
var avs []*result
avDir := filepath.Join(util.DataDir, "storage", "av")
const limit = 16
entries, err := os.ReadDir(avDir)
if nil != err {
logging.LogErrorf("read directory [%s] failed: %s", avDir, err)
return
}
count := 0
avBlockRels := av.GetBlockRels()
for _, entry := range entries {
if entry.IsDir() {
continue
@ -214,25 +219,55 @@ func SearchAttributeView(keyword string) (ret []*SearchAttributeViewResult) {
continue
}
name, _ := av.GetAttributeViewNameByPath(filepath.Join(avDir, entry.Name()))
if "" == name {
if nil == avBlockRels[id] {
continue
}
if strings.Contains(strings.ToLower(name), strings.ToLower(keyword)) {
avs[id] = name
count++
if limit <= count {
break
name, _ := av.GetAttributeViewNameByPath(filepath.Join(avDir, entry.Name()))
info, _ := entry.Info()
if "" != keyword {
if strings.Contains(strings.ToLower(name), strings.ToLower(keyword)) {
score := smetrics.JaroWinkler(name, keyword, 0.7, 4)
a := &result{AvID: id, AvName: name, Score: score}
if nil != info && !info.ModTime().IsZero() {
a.AvUpdated = info.ModTime().UnixMilli()
}
avs = append(avs, a)
}
} else {
a := &result{AvID: id, AvName: name}
if nil != info && !info.ModTime().IsZero() {
a.AvUpdated = info.ModTime().UnixMilli()
}
avs = append(avs, a)
}
}
var avIDs []string
for avID := range avs {
avIDs = append(avIDs, avID)
if "" == keyword {
sort.Slice(avs, func(i, j int) bool { return avs[i].AvUpdated > avs[j].AvUpdated })
} else {
sort.SliceStable(avs, func(i, j int) bool {
if avs[i].Score == avs[j].Score {
return avs[i].AvUpdated > avs[j].AvUpdated
}
return avs[i].Score > avs[j].Score
})
}
blockIDs := treenode.BatchGetMirrorAttrViewBlockIDs(avIDs)
if 12 <= len(avs) {
avs = avs[:12]
}
var avIDs []string
for _, a := range avs {
avIDs = append(avIDs, a.AvID)
}
avBlocks := treenode.BatchGetMirrorAttrViewBlocks(avIDs)
var blockIDs []string
for _, avBlock := range avBlocks {
blockIDs = append(blockIDs, avBlock.BlockIDs...)
}
blockIDs = gulu.Str.RemoveDuplicatedElem(blockIDs)
trees := map[string]*parse.Tree{}
for _, blockID := range blockIDs {
bt := treenode.GetBlockTree(blockID)
@ -261,8 +296,14 @@ func SearchAttributeView(keyword string) (ret []*SearchAttributeViewResult) {
}
avID := node.AttributeViewID
name := avs[avID]
if "" == name {
var existAv *result
for _, av := range avs {
if av.AvID == avID {
existAv = av
break
}
}
if nil == existAv {
continue
}
@ -287,7 +328,7 @@ func SearchAttributeView(keyword string) (ret []*SearchAttributeViewResult) {
if !exist {
ret = append(ret, &SearchAttributeViewResult{
AvID: avID,
AvName: name,
AvName: existAv.AvName,
BlockID: blockID,
HPath: hPath,
})
@ -733,7 +774,7 @@ func renderAttributeView(attrView *av.AttributeView, viewID, query string, page,
}
if 0 == v.UpdatedAt {
v.UpdatedAt = v.CreatedAt + 1000
v.UpdatedAt = v.CreatedAt
}
}
}

5
kernel/model/import.go
View file

@ -281,6 +281,10 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
}
return ast.WalkContinue
})
// 关联数据库和块
avNodes := tree.Root.ChildrenByType(ast.NodeAttributeView)
av.BatchUpsertBlockRel(avNodes)
}
}
@ -895,6 +899,7 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) {
util.PushEndlessProgress(fmt.Sprintf(Conf.Language(66), fmt.Sprintf("%d/%d ", i, len(importTrees))+tree.HPath))
}
}
util.PushClearProgress()
importTrees = []*parse.Tree{}
searchLinks = map[string]string{}

9
kernel/model/index.go
View file

@ -36,6 +36,7 @@ import (
"github.com/siyuan-note/eventbus"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/av"
"github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/sql"
@ -143,6 +144,7 @@ func index(boxID string) {
poolSize = 4
}
waitGroup := &sync.WaitGroup{}
var avNodes []*ast.Node
p, _ := ants.NewPoolWithFunc(poolSize, func(arg interface{}) {
defer waitGroup.Done()
@ -168,6 +170,10 @@ func index(boxID string) {
}
}
lock.Lock()
avNodes = append(avNodes, tree.Root.ChildrenByType(ast.NodeAttributeView)...)
lock.Unlock()
cache.PutDocIAL(file.path, docIAL)
treenode.IndexBlockTree(tree)
sql.IndexTreeQueue(tree)
@ -191,6 +197,9 @@ func index(boxID string) {
waitGroup.Wait()
p.Release()
// 关联数据库和块
av.BatchUpsertBlockRel(avNodes)
box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间
end := time.Now()
elapsed := end.Sub(start).Seconds()

17
kernel/model/search.go
View file

@ -384,17 +384,22 @@ func SearchRefBlock(id, rootID, keyword string, beforeLen int, isSquareBrackets
}
}
if b.ID != id && !hitFirstChildID && b.ID != rootID {
if "NodeAttributeView" == b.Type {
// 数据库块可以添加到自身数据库块中，当前文档也可以添加到自身数据库块中
tmp = append(tmp, b)
} else {
// 排除自身块、父块和根块
if b.ID != id && !hitFirstChildID && b.ID != rootID {
tmp = append(tmp, b)
}
}
}
ret = tmp
if "" != keyword {
if block := treenode.GetBlockTree(id); nil != block {
p := path.Join(block.HPath, keyword)
newDoc = nil == treenode.GetBlockTreeRootByHPath(block.BoxID, p)
}
if block := treenode.GetBlockTree(id); nil != block {
p := path.Join(block.HPath, keyword)
newDoc = nil == treenode.GetBlockTreeRootByHPath(block.BoxID, p)
}
// 在 hPath 中加入笔记本名 Show notebooks in hpath of block ref search list results https://github.com/siyuan-note/siyuan/issues/9378

11
kernel/model/virutalref.go
View file

@ -144,7 +144,7 @@ func AddVirtualBlockRefExclude(keyword []string) {
}
func processVirtualRef(n *ast.Node, unlinks *[]*ast.Node, virtualBlockRefKeywords []string, refCount map[string]int, luteEngine *lute.Lute) bool {
if !Conf.Editor.VirtualBlockRef {
if !Conf.Editor.VirtualBlockRef || 1 > len(virtualBlockRefKeywords) {
return false
}
@ -153,12 +153,15 @@ func processVirtualRef(n *ast.Node, unlinks *[]*ast.Node, virtualBlockRefKeyword
}
parentBlock := treenode.ParentBlock(n)
if nil == parentBlock || 0 < refCount[parentBlock.ID] {
if nil == parentBlock {
return false
}
if 1 > len(virtualBlockRefKeywords) {
return false
if 0 < refCount[parentBlock.ID] {
// 如果块被引用过，则将其自身的文本排除在虚拟引用关键字之外
// Referenced blocks support rendering virtual references https://github.com/siyuan-note/siyuan/issues/10960
parentText := getNodeRefText(parentBlock)
virtualBlockRefKeywords = gulu.Str.RemoveElem(virtualBlockRefKeywords, parentText)
}
content := string(n.Tokens)

34
kernel/sql/database.go
View file

@ -130,6 +130,11 @@ func initDBTables() {
logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create table [blocks] failed: %s", err)
}
_, err = db.Exec("CREATE INDEX idx_blocks_id ON blocks(id)")
if nil != err {
logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create index [idx_blocks_id] failed: %s", err)
}
_, err = db.Exec("CREATE INDEX idx_blocks_root_id ON blocks(root_id)")
if nil != err {
logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create index [idx_blocks_root_id] failed: %s", err)
@ -139,7 +144,7 @@ func initDBTables() {
if nil != err {
logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "drop table [blocks_fts] failed: %s", err)
}
_, err = db.Exec("CREATE VIRTUAL TABLE blocks_fts USING fts5(id UNINDEXED, parent_id UNINDEXED, root_id UNINDEXED, hash UNINDEXED, box UNINDEXED, path UNINDEXED, hpath, name, alias, memo, tag, content, fcontent, markdown UNINDEXED, length UNINDEXED, type UNINDEXED, subtype UNINDEXED, ial, sort UNINDEXED, created UNINDEXED, updated UNINDEXED, tokenize=\"siyuan\")")
_, err = db.Exec("CREATE VIRTUAL TABLE blocks_fts USING fts5(id, parent_id UNINDEXED, root_id UNINDEXED, hash UNINDEXED, box UNINDEXED, path UNINDEXED, hpath, name, alias, memo, tag, content, fcontent, markdown UNINDEXED, length UNINDEXED, type UNINDEXED, subtype UNINDEXED, ial, sort UNINDEXED, created UNINDEXED, updated UNINDEXED, tokenize=\"siyuan\")")
if nil != err {
logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create table [blocks_fts] failed: %s", err)
}
@ -148,7 +153,7 @@ func initDBTables() {
if nil != err {
logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "drop table [blocks_fts_case_insensitive] failed: %s", err)
}
_, err = db.Exec("CREATE VIRTUAL TABLE blocks_fts_case_insensitive USING fts5(id UNINDEXED, parent_id UNINDEXED, root_id UNINDEXED, hash UNINDEXED, box UNINDEXED, path UNINDEXED, hpath, name, alias, memo, tag, content, fcontent, markdown UNINDEXED, length UNINDEXED, type UNINDEXED, subtype UNINDEXED, ial, sort UNINDEXED, created UNINDEXED, updated UNINDEXED, tokenize=\"siyuan case_insensitive\")")
_, err = db.Exec("CREATE VIRTUAL TABLE blocks_fts_case_insensitive USING fts5(id, parent_id UNINDEXED, root_id UNINDEXED, hash UNINDEXED, box UNINDEXED, path UNINDEXED, hpath, name, alias, memo, tag, content, fcontent, markdown UNINDEXED, length UNINDEXED, type UNINDEXED, subtype UNINDEXED, ial, sort UNINDEXED, created UNINDEXED, updated UNINDEXED, tokenize=\"siyuan case_insensitive\")")
if nil != err {
logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create table [blocks_fts_case_insensitive] failed: %s", err)
}
@ -954,30 +959,25 @@ func deleteByBoxTx(tx *sql.Tx, box string) (err error) {
}
func deleteBlocksByIDs(tx *sql.Tx, ids []string) (err error) {
in := bytes.Buffer{}
in.Grow(4096)
in.WriteString("(")
for i, id := range ids {
in.WriteString("'")
in.WriteString(id)
in.WriteString("'")
if i < len(ids)-1 {
in.WriteString(",")
}
var ftsIDs []string
for _, id := range ids {
removeBlockCache(id)
ftsIDs = append(ftsIDs, "\""+id+"\"")
}
in.WriteString(")")
stmt := "DELETE FROM blocks WHERE id IN " + in.String()
stmt := "DELETE FROM blocks WHERE id IN (" + strings.Join(ftsIDs, ",") + ")"
if err = execStmtTx(tx, stmt); nil != err {
return
}
stmt = "DELETE FROM blocks_fts WHERE id IN " + in.String()
ftsIDsMatch := strings.Join(ftsIDs, " OR ")
stmt = "DELETE FROM blocks_fts WHERE ROWID IN (SELECT ROWID FROM blocks_fts WHERE blocks_fts MATCH 'id:(" + ftsIDsMatch + ")')"
if err = execStmtTx(tx, stmt); nil != err {
return
}
if !caseSensitive {
stmt = "DELETE FROM blocks_fts_case_insensitive WHERE id IN " + in.String()
stmt = "DELETE FROM blocks_fts_case_insensitive WHERE ROWID IN (SELECT ROWID FROM blocks_fts_case_insensitive WHERE blocks_fts_case_insensitive MATCH 'id:(" + ftsIDsMatch + ")')"
if err = execStmtTx(tx, stmt); nil != err {
return
}

81
kernel/treenode/av.go
View file

@ -26,41 +26,6 @@ import (
"github.com/vmihailenco/msgpack/v5"
)
func BatchGetMirrorAttrViewBlockIDs(avIDs []string) (ret map[string]string) {
av.AttributeViewBlocksLock.Lock()
defer av.AttributeViewBlocksLock.Unlock()
ret = map[string]string{}
blocks := filepath.Join(util.DataDir, "storage", "av", "blocks.msgpack")
if !filelock.IsExist(blocks) {
return
}
data, err := filelock.ReadFile(blocks)
if nil != err {
logging.LogErrorf("read attribute view blocks failed: %s", err)
return
}
avBlocks := map[string][]string{}
if err = msgpack.Unmarshal(data, &avBlocks); nil != err {
logging.LogErrorf("unmarshal attribute view blocks failed: %s", err)
return
}
for _, avID := range avIDs {
blockIDs := avBlocks[avID]
for _, blockID := range blockIDs {
if nil != GetBlockTree(blockID) {
ret[avID] = blockID
break
}
}
}
return
}
func GetMirrorAttrViewBlockIDs(avID string) (ret []string) {
av.AttributeViewBlocksLock.Lock()
defer av.AttributeViewBlocksLock.Unlock()
@ -91,3 +56,49 @@ func GetMirrorAttrViewBlockIDs(avID string) (ret []string) {
}
return
}
type AvBlock struct {
AvID string
BlockIDs []string
}
func BatchGetMirrorAttrViewBlocks(avIDs []string) (ret []*AvBlock) {
av.AttributeViewBlocksLock.Lock()
defer av.AttributeViewBlocksLock.Unlock()
ret = []*AvBlock{}
blocks := filepath.Join(util.DataDir, "storage", "av", "blocks.msgpack")
if !filelock.IsExist(blocks) {
return
}
data, err := filelock.ReadFile(blocks)
if nil != err {
logging.LogErrorf("read attribute view blocks failed: %s", err)
return
}
avBlocks := map[string][]string{}
if err = msgpack.Unmarshal(data, &avBlocks); nil != err {
logging.LogErrorf("unmarshal attribute view blocks failed: %s", err)
return
}
for _, avID := range avIDs {
var blockIDs []string
for _, blockID := range avBlocks[avID] {
if nil == GetBlockTree(blockID) {
continue
}
blockIDs = append(blockIDs, blockID)
}
avBlock := &AvBlock{
AvID: avID,
BlockIDs: blockIDs,
}
ret = append(ret, avBlock)
}
return
}

2
kernel/treenode/node.go
View file

@ -954,7 +954,7 @@ func GetAttributeViewDefaultValue(valueID, keyID, blockID string, typ av.KeyType
ret.CreatedAt = time.Now().UnixMilli()
}
if 0 == ret.UpdatedAt {
ret.UpdatedAt = ret.CreatedAt + 1000
ret.UpdatedAt = ret.CreatedAt
}
switch typ {