0ct0pu5/siyuan
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

♻️ Refactor the blocktree storage https://github.com/siyuan-note/siyuan/issues/11773

Browse source
This commit is contained in:
Daniel 2024-06-20 22:53:27 +08:00
parent 32413fa11c
commit 7a3d4a05ad
No known key found for this signature in database
GPG key ID: 86211BA83DF03017
22 changed files with 453 additions and 497 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
app/appearance/langs/en_US.json
View file

@ -1377,7 +1377,7 @@
"88": "Finished parsing [%d] data files, remaining to be processed [%d]",
"89": "[%d/%d] Created [%d] of data indexes of block-level elements [%s]",
"90": "[%d/%d] Created [%d] of search indexes of block-level elements [%s]",
"91": "Reading block tree data...",
"91": "TODO",
"92": "Parsing document tree [%s]",
"93": "[%d/%d] Cleaned up the index related to document [%s]",
"94": "Upload failed: %s",

2
app/appearance/langs/es_ES.json
View file

@ -1377,7 +1377,7 @@
"88": "Se ha terminado de analizar [%d] archivos de datos, quedan por procesar [%d]",
"89": "[%d/%d] Creado [%d] de índices de datos de elementos a nivel de bloque [%s]",
"90": "[%d/%d] Creado [%d] de índices de búsqueda de elementos a nivel de bloque [%s]",
"91": "Leyendo datos del árbol de bloques...",
"91": "TODO",
"92": "Analizando el árbol del documento [%s]",
"93": "[%d/%d] ha limpiado el índice relacionado con el documento [%s]",
"94": "Carga fallida: %s",

2
app/appearance/langs/fr_FR.json
View file

@ -1377,7 +1377,7 @@
"88": "Fin de l'analyse des fichiers de données [%d], restant à traiter [%d]",
"89": "[%d/%d] Créé [%d] d'index de données d'éléments de niveau bloc [%s]",
"90": "[%d/%d] Création de [%d] index de recherche d'éléments de niveau bloc [%s]",
"91": "Lecture des données de l'arborescence des blocs...",
"91": "TODO",
"92": "Analyse de l'arborescence du document [%s]",
"93": "[%d/%d] a nettoyé l'index lié au document [%s]",
"94": "Échec du téléchargement : %s",

2
app/appearance/langs/ja_JP.json
View file

@ -1377,7 +1377,7 @@
"88": "[%d] 個のデータファイルの解析が完了し、処理待ちのデータファイルが [%d] 個残っています",
"89": "[%d/%d] ブロックレベル要素 [%s] のデータインデックスを [%d] 個作成しました",
"90": "[%d/%d] ブロックレベル要素 [%s] の検索インデックスを [%d] 個作成しました",
"91": "ブロックツリーデータを読み込んでいます...",
"91": "TODO",
"92": "ドキュメントツリーを解析しています [%s]",
"93": "[%d/%d] ドキュメント [%s] に関連するインデックスをクリーンアップしました",
"94": "アップロードに失敗しました: %s",

2
app/appearance/langs/zh_CHT.json
View file

@ -1377,7 +1377,7 @@
"88": "已完成解析 [%d] 個資料文件，剩餘待處理 [%d]",
"89": "[%d/%d] 已經創建 [%d] 個塊級元素的資料索引 [%s]",
"90": "[%d/%d] 已經創建 [%d] 個塊級元素的搜索索引 [%s]",
"91": "正在讀取塊樹資料...",
"91": "TODO",
"92": "正在解析文檔樹 [%s]",
"93": "[%d/%d] 已經清理文檔 [%s] 相關的索引",
"94": "上傳失敗：%s",

2
app/appearance/langs/zh_CN.json
View file

@ -1377,7 +1377,7 @@
"88": "已完成解析 [%d] 个数据文件，剩余待处理 [%d]",
"89": "[%d/%d] 已经创建 [%d] 个块级元素的数据索引 [%s]",
"90": "[%d/%d] 已经创建 [%d] 个块级元素的搜索索引 [%s]",
"91": "正在读取块树数据...",
"91": "TODO",
"92": "正在解析文档树 [%s]",
"93": "[%d/%d] 已经清理文档 [%s] 相关的索引",
"94": "上传失败：%s",

4
kernel/filesys/tree.go
View file

@ -88,7 +88,7 @@ func LoadTreeByData(data []byte, boxID, p string, luteEngine *lute.Lute) (ret *p
logging.LogErrorf("rebuild parent tree [%s] failed: %s", parentAbsPath, writeErr)
} else {
logging.LogInfof("rebuilt parent tree [%s]", parentAbsPath)
treenode.IndexBlockTree(parentTree)
treenode.UpsertBlockTree(parentTree)
}
} else {
logging.LogWarnf("read parent tree data [%s] failed: %s", parentAbsPath, readErr)
@ -137,7 +137,7 @@ func prepareWriteTree(tree *parse.Tree) (data []byte, filePath string, err error
newP := treenode.NewParagraph()
tree.Root.AppendChild(newP)
tree.Root.SetIALAttr("updated", util.TimeFromID(newP.ID))
treenode.IndexBlockTree(tree)
treenode.UpsertBlockTree(tree)
}
filePath = filepath.Join(util.DataDir, tree.Box, tree.Path)

2
kernel/job/cron.go
View file

@ -23,14 +23,12 @@ import (
"github.com/siyuan-note/siyuan/kernel/model"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
)
func StartCron() {
go every(100*time.Millisecond, task.ExecTaskJob)
go every(5*time.Second, task.StatusJob)
go every(5*time.Second, treenode.SaveBlockTreeJob)
go every(5*time.Second, model.SyncDataJob)
go every(2*time.Hour, model.StatJob)
go every(2*time.Hour, model.RefreshCheckJob)

2
kernel/model/assets.go
View file

@ -825,7 +825,7 @@ func RenameAsset(oldPath, newName string) (err error) {
continue
}
treenode.IndexBlockTree(tree)
treenode.UpsertBlockTree(tree)
sql.UpsertTreeQueue(tree)
util.PushEndlessProgress(fmt.Sprintf(Conf.Language(111), util.EscapeHTML(tree.Root.IALAttr("title"))))

1
kernel/model/box.go
View file

@ -531,7 +531,6 @@ func fullReindex() {
for _, openedBox := range openedBoxes {
index(openedBox.ID)
}
treenode.SaveBlockTree(true)
LoadFlashcards()
debug.FreeOSMemory()
}

27
kernel/model/conf.go
View file

@ -627,7 +627,6 @@ func Close(force, setCurrentWorkspace bool, execInstallPkg int) (exitCode int) {
Conf.Close()
sql.CloseDatabase()
treenode.SaveBlockTree(false)
util.SaveAssetsTexts()
clearWorkspaceTemp()
clearCorruptedNotebooks()
@ -818,24 +817,7 @@ func (conf *AppConf) language(num int) (ret string) {
}
func InitBoxes() {
initialized := false
if 1 > treenode.CountBlocks() {
if gulu.File.IsExist(util.BlockTreePath) {
util.IncBootProgress(20, Conf.Language(91))
go func() {
for i := 0; i < 40; i++ {
util.RandomSleep(50, 100)
util.IncBootProgress(1, Conf.Language(91))
}
}()
treenode.InitBlockTree(false)
initialized = true
}
} else { // 大于 1 的话说明在同步阶段已经加载过了
initialized = true
}
initialized := 0 < treenode.CountBlocks()
for _, box := range Conf.GetOpenedBoxes() {
box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间
@ -844,10 +826,6 @@ func InitBoxes() {
}
}
if !initialized {
treenode.SaveBlockTree(true)
}
var dbSize string
if dbFile, err := os.Stat(util.DBPath); nil == err {
dbSize = humanize.BytesCustomCeil(uint64(dbFile.Size()), 2)
@ -982,7 +960,8 @@ func clearWorkspaceTemp() {
os.RemoveAll(filepath.Join(util.TempDir, "import"))
os.RemoveAll(filepath.Join(util.TempDir, "repo"))
os.RemoveAll(filepath.Join(util.TempDir, "os"))
os.RemoveAll(filepath.Join(util.TempDir, "blocktree.msgpack")) // v2.7.2 前旧版的块数数据
os.RemoveAll(filepath.Join(util.TempDir, "blocktree.msgpack")) // v2.7.2 前旧版的块树数据
os.RemoveAll(filepath.Join(util.TempDir, "blocktree")) // v3.1.0 前旧版的块树数据
// 退出时自动删除超过 7 天的安装包 https://github.com/siyuan-note/siyuan/issues/6128
install := filepath.Join(util.TempDir, "install")

4
kernel/model/file.go
View file

@ -1086,7 +1086,7 @@ func indexWriteTreeIndexQueue(tree *parse.Tree) (err error) {
}
func indexWriteTreeUpsertQueue(tree *parse.Tree) (err error) {
treenode.IndexBlockTree(tree)
treenode.UpsertBlockTree(tree)
return writeTreeUpsertQueue(tree)
}
@ -1095,7 +1095,7 @@ func renameWriteJSONQueue(tree *parse.Tree) (err error) {
return
}
sql.RenameTreeQueue(tree)
treenode.IndexBlockTree(tree)
treenode.UpsertBlockTree(tree)
return
}

2
kernel/model/index_fix.go
View file

@ -478,7 +478,7 @@ func reindexTree0(tree *parse.Tree, i, size int) {
tree.Root.SetIALAttr("updated", updated)
indexWriteTreeUpsertQueue(tree)
} else {
treenode.IndexBlockTree(tree)
treenode.UpsertBlockTree(tree)
sql.IndexTreeQueue(tree)
}

2
kernel/model/mount.go
View file

@ -30,7 +30,6 @@ import (
"github.com/88250/lute/ast"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
)
@ -247,7 +246,6 @@ func Mount(boxID string) (alreadyMount bool, err error) {
box.Index()
// 缓存根一级的文档树展开
ListDocTree(box.ID, "/", util.SortModeUnassigned, false, false, Conf.FileTree.MaxListCount)
treenode.SaveBlockTree(false)
util.ClearPushProgress(100)
if reMountGuide {

2
kernel/model/sync.go
View file

@ -352,7 +352,7 @@ func upsertIndexes(upsertFilePaths []string) (upsertRootIDs []string) {
if nil != err0 {
continue
}
treenode.IndexBlockTree(tree)
treenode.UpsertBlockTree(tree)
sql.UpsertTreeQueue(tree)
bts := treenode.GetBlockTreesByRootID(tree.ID)

2
kernel/model/transaction.go
View file

@ -1378,7 +1378,7 @@ func (tx *Transaction) loadTree(id string) (ret *parse.Tree, err error) {
func (tx *Transaction) writeTree(tree *parse.Tree) (err error) {
tx.trees[tree.ID] = tree
treenode.IndexBlockTree(tree)
treenode.UpsertBlockTree(tree)
return
}

2
kernel/model/tree.go
View file

@ -274,7 +274,7 @@ func searchTreeInFilesystem(rootID string) {
return
}
treenode.IndexBlockTree(tree)
treenode.UpsertBlockTree(tree)
sql.IndexTreeQueue(tree)
logging.LogInfof("reindexed tree by filesystem [rootID=%s]", rootID)
}

9
kernel/sql/database.go
View file

@ -82,6 +82,7 @@ func InitDatabase(forceRebuild bool) (err error) {
}
initDBConnection()
treenode.InitBlockTree(forceRebuild)
if !forceRebuild {
// 检查数据库结构版本，如果版本不一致的话说明改过表结构，需要重建
@ -101,9 +102,6 @@ func InitDatabase(forceRebuild bool) (err error) {
err = nil
}
}
if gulu.File.IsExist(util.BlockTreePath) {
treenode.InitBlockTree(true)
}
initDBConnection()
initDBTables()
@ -1278,6 +1276,11 @@ func CloseDatabase() {
logging.LogErrorf("close history database failed: %s", err)
return
}
if err := assetContentDB.Close(); nil != err {
logging.LogErrorf("close asset content database failed: %s", err)
return
}
treenode.CloseDatabase()
logging.LogInfof("closed database")
}

782
kernel/treenode/blocktree.go
View file

@ -17,33 +17,22 @@
package treenode
import (
"bytes"
"database/sql"
"errors"
"os"
"path/filepath"
"strings"
"runtime"
"runtime/debug"
"sync"
"sync/atomic"
"time"
"github.com/88250/go-humanize"
"github.com/88250/gulu"
"github.com/88250/lute/ast"
"github.com/88250/lute/parse"
"github.com/panjf2000/ants/v2"
util2 "github.com/siyuan-note/dejavu/util"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/util"
"github.com/vmihailenco/msgpack/v5"
)
var blockTrees = &sync.Map{}
type btSlice struct {
data map[string]*BlockTree
changed time.Time
m *sync.Mutex
}
type BlockTree struct {
ID string // 块 ID
RootID string // 根 ID
@ -55,131 +44,237 @@ type BlockTree struct {
Type string // 类型
}
func GetBlockTreesByType(typ string) (ret []*BlockTree) {
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if b.Type == typ {
ret = append(ret, b)
}
var (
db *sql.DB
)
func initDatabase(forceRebuild bool) (err error) {
initDBConnection()
if !forceRebuild {
if !gulu.File.IsExist(util.BlockTreeDBPath) {
forceRebuild = true
}
slice.m.Unlock()
return true
})
}
if !forceRebuild {
return
}
closeDatabase()
if gulu.File.IsExist(util.BlockTreeDBPath) {
if err = removeDatabaseFile(); nil != err {
logging.LogErrorf("remove database file [%s] failed: %s", util.BlockTreeDBPath, err)
err = nil
}
}
initDBConnection()
initDBTables()
logging.LogInfof("reinitialized database [%s]", util.BlockTreeDBPath)
return
}
func initDBTables() {
_, err := db.Exec("DROP TABLE IF EXISTS blocktrees")
if nil != err {
logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "drop table [blocks] failed: %s", err)
}
_, err = db.Exec("CREATE TABLE blocktrees (id, root_id, parent_id, box_id, path, hpath, updated, type)")
if nil != err {
logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create table [blocktrees] failed: %s", err)
}
}
func initDBConnection() {
if nil != db {
closeDatabase()
}
dsn := util.BlockTreeDBPath + "?_journal_mode=WAL" +
"&_synchronous=OFF" +
"&_mmap_size=2684354560" +
"&_secure_delete=OFF" +
"&_cache_size=-20480" +
"&_page_size=32768" +
"&_busy_timeout=7000" +
"&_ignore_check_constraints=ON" +
"&_temp_store=MEMORY" +
"&_case_sensitive_like=OFF"
var err error
db, err = sql.Open("sqlite3_extended", dsn)
if nil != err {
logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create database failed: %s", err)
}
db.SetMaxIdleConns(7)
db.SetMaxOpenConns(7)
db.SetConnMaxLifetime(365 * 24 * time.Hour)
}
func CloseDatabase() {
closeDatabase()
}
func closeDatabase() {
if nil == db {
return
}
if err := db.Close(); nil != err {
logging.LogErrorf("close database failed: %s", err)
}
debug.FreeOSMemory()
runtime.GC() // 没有这句的话文件句柄不会释放，后面就无法删除文件
return
}
func removeDatabaseFile() (err error) {
err = os.RemoveAll(util.BlockTreeDBPath)
if nil != err {
return
}
err = os.RemoveAll(util.BlockTreeDBPath + "-shm")
if nil != err {
return
}
err = os.RemoveAll(util.BlockTreeDBPath + "-wal")
if nil != err {
return
}
return
}
func GetBlockTreesByType(typ string) (ret []*BlockTree) {
sqlStmt := "SELECT * FROM blocktrees WHERE type = ?"
rows, err := db.Query(sqlStmt)
if nil != err {
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
return
}
defer rows.Close()
for rows.Next() {
var block BlockTree
if err = rows.Scan(&block.ID, &block.RootID, &block.ParentID, &block.BoxID, &block.Path, &block.HPath, &block.Updated, &block.Type); nil != err {
logging.LogErrorf("query scan field failed: %s", err)
return
}
ret = append(ret, &block)
}
return
}
func GetBlockTreeByPath(path string) (ret *BlockTree) {
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if b.Path == path {
ret = b
break
}
ret = &BlockTree{}
sqlStmt := "SELECT * FROM blocktrees WHERE path = ?"
err := db.QueryRow(sqlStmt, path).Scan(&ret.ID, &ret.RootID, &ret.ParentID, &ret.BoxID, &ret.Path, &ret.HPath, &ret.Updated, &ret.Type)
if nil != err {
ret = nil
if errors.Is(err, sql.ErrNoRows) {
return
}
slice.m.Unlock()
return nil == ret
})
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
return
}
return
}
func CountTrees() (ret int) {
roots := map[string]bool{}
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
roots[b.RootID] = true
sqlStmt := "SELECT COUNT(*) FROM blocktrees WHERE type = 'd'"
err := db.QueryRow(sqlStmt).Scan(&ret)
if nil != err {
if errors.Is(err, sql.ErrNoRows) {
return 0
}
slice.m.Unlock()
return true
})
ret = len(roots)
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
}
return
}
func CountBlocks() (ret int) {
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
ret += len(slice.data)
slice.m.Unlock()
return true
})
sqlStmt := "SELECT COUNT(*) FROM blocktrees"
err := db.QueryRow(sqlStmt).Scan(&ret)
if nil != err {
if errors.Is(err, sql.ErrNoRows) {
return 0
}
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
}
return
}
func GetBlockTreeRootByPath(boxID, path string) (ret *BlockTree) {
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if b.BoxID == boxID && b.Path == path && b.RootID == b.ID {
ret = b
break
}
ret = &BlockTree{}
sqlStmt := "SELECT * FROM blocktrees WHERE box_id = ? AND path = ?"
err := db.QueryRow(sqlStmt, boxID, path).Scan(&ret.ID, &ret.RootID, &ret.ParentID, &ret.BoxID, &ret.Path, &ret.HPath, &ret.Updated, &ret.Type)
if nil != err {
ret = nil
if errors.Is(err, sql.ErrNoRows) {
return
}
slice.m.Unlock()
return nil == ret
})
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
return
}
return
}
func GetBlockTreeRootByHPath(boxID, hPath string) (ret *BlockTree) {
ret = &BlockTree{}
hPath = gulu.Str.RemoveInvisible(hPath)
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if b.BoxID == boxID && b.HPath == hPath && b.RootID == b.ID {
ret = b
break
}
sqlStmt := "SELECT * FROM blocktrees WHERE box_id = ? AND hpath = ?"
err := db.QueryRow(sqlStmt, boxID, hPath).Scan(&ret.ID, &ret.RootID, &ret.ParentID, &ret.BoxID, &ret.Path, &ret.HPath, &ret.Updated, &ret.Type)
if nil != err {
ret = nil
if errors.Is(err, sql.ErrNoRows) {
return
}
slice.m.Unlock()
return nil == ret
})
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
return
}
return
}
func GetBlockTreeRootsByHPath(boxID, hPath string) (ret []*BlockTree) {
hPath = gulu.Str.RemoveInvisible(hPath)
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if b.BoxID == boxID && b.HPath == hPath && b.RootID == b.ID {
ret = append(ret, b)
}
sqlStmt := "SELECT * FROM blocktrees WHERE box_id = ? AND hpath = ?"
rows, err := db.Query(sqlStmt, boxID, hPath)
if nil != err {
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
return
}
defer rows.Close()
for rows.Next() {
var block BlockTree
if err = rows.Scan(&block.ID, &block.RootID, &block.ParentID, &block.BoxID, &block.Path, &block.HPath, &block.Updated, &block.Type); nil != err {
logging.LogErrorf("query scan field failed: %s", err)
return
}
slice.m.Unlock()
return true
})
ret = append(ret, &block)
}
return
}
func GetBlockTreeRootByHPathPreferredParentID(boxID, hPath, preferredParentID string) (ret *BlockTree) {
hPath = gulu.Str.RemoveInvisible(hPath)
var roots []*BlockTree
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if b.BoxID == boxID && b.HPath == hPath && b.RootID == b.ID {
if "" == preferredParentID {
ret = b
break
}
roots = append(roots, b)
}
sqlStmt := "SELECT * FROM blocktrees WHERE box_id = ? AND hpath = ? AND parent_id = ?"
rows, err := db.Query(sqlStmt, boxID, hPath, preferredParentID)
if nil != err {
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
return
}
defer rows.Close()
for rows.Next() {
var block BlockTree
if err = rows.Scan(&block.ID, &block.RootID, &block.ParentID, &block.BoxID, &block.Path, &block.HPath, &block.Updated, &block.Type); nil != err {
logging.LogErrorf("query scan field failed: %s", err)
return
}
slice.m.Unlock()
return nil == ret
})
if "" == preferredParentID {
ret = &block
return
}
roots = append(roots, &block)
}
if 1 > len(roots) {
return
}
@ -195,16 +290,17 @@ func GetBlockTreeRootByHPathPreferredParentID(boxID, hPath, preferredParentID st
}
func ExistBlockTree(id string) bool {
hash := btHash(id)
val, ok := blockTrees.Load(hash)
if !ok {
sqlStmt := "SELECT COUNT(*) FROM blocktrees WHERE id = ?"
var count int
err := db.QueryRow(sqlStmt, id).Scan(&count)
if nil != err {
if errors.Is(err, sql.ErrNoRows) {
return false
}
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
return false
}
slice := val.(*btSlice)
slice.m.Lock()
_, ok = slice.data[id]
slice.m.Unlock()
return ok
return 0 < count
}
func GetBlockTree(id string) (ret *BlockTree) {
@ -212,15 +308,17 @@ func GetBlockTree(id string) (ret *BlockTree) {
return
}
hash := btHash(id)
val, ok := blockTrees.Load(hash)
if !ok {
ret = &BlockTree{}
sqlStmt := "SELECT * FROM blocktrees WHERE id = ?"
err := db.QueryRow(sqlStmt, id).Scan(&ret.ID, &ret.RootID, &ret.ParentID, &ret.BoxID, &ret.Path, &ret.HPath, &ret.Updated, &ret.Type)
if nil != err {
ret = nil
if errors.Is(err, sql.ErrNoRows) {
return
}
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, logging.ShortStack())
return
}
slice := val.(*btSlice)
slice.m.Lock()
ret = slice.data[id]
slice.m.Unlock()
return
}
@ -230,170 +328,169 @@ func SetBlockTreePath(tree *parse.Tree) {
}
func RemoveBlockTreesByRootID(rootID string) {
var ids []string
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if b.RootID == rootID {
ids = append(ids, b.ID)
}
}
slice.m.Unlock()
return true
})
ids = gulu.Str.RemoveDuplicatedElem(ids)
for _, id := range ids {
val, ok := blockTrees.Load(btHash(id))
if !ok {
continue
}
slice := val.(*btSlice)
slice.m.Lock()
delete(slice.data, id)
slice.changed = time.Now()
slice.m.Unlock()
sqlStmt := "DELETE FROM blocktrees WHERE root_id = ?"
_, err := db.Exec(sqlStmt, rootID)
if nil != err {
logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err)
return
}
}
func GetBlockTreesByPathPrefix(pathPrefix string) (ret []*BlockTree) {
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if strings.HasPrefix(b.Path, pathPrefix) {
ret = append(ret, b)
}
sqlStmt := "SELECT * FROM blocktrees WHERE path LIKE ?"
rows, err := db.Query(sqlStmt, pathPrefix+"%")
if nil != err {
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
return
}
defer rows.Close()
for rows.Next() {
var block BlockTree
if err = rows.Scan(&block.ID, &block.RootID, &block.ParentID, &block.BoxID, &block.Path, &block.HPath, &block.Updated, &block.Type); nil != err {
logging.LogErrorf("query scan field failed: %s", err)
return
}
slice.m.Unlock()
return true
})
ret = append(ret, &block)
}
return
}
func GetBlockTreesByRootID(rootID string) (ret []*BlockTree) {
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if b.RootID == rootID {
ret = append(ret, b)
}
sqlStmt := "SELECT * FROM blocktrees WHERE root_id = ?"
rows, err := db.Query(sqlStmt, rootID)
if nil != err {
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
return
}
defer rows.Close()
for rows.Next() {
var block BlockTree
if err = rows.Scan(&block.ID, &block.RootID, &block.ParentID, &block.BoxID, &block.Path, &block.HPath, &block.Updated, &block.Type); nil != err {
logging.LogErrorf("query scan field failed: %s", err)
return
}
slice.m.Unlock()
return true
})
ret = append(ret, &block)
}
return
}
func RemoveBlockTreesByPathPrefix(pathPrefix string) {
var ids []string
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if strings.HasPrefix(b.Path, pathPrefix) {
ids = append(ids, b.ID)
}
}
slice.m.Unlock()
return true
})
ids = gulu.Str.RemoveDuplicatedElem(ids)
for _, id := range ids {
val, ok := blockTrees.Load(btHash(id))
if !ok {
continue
}
slice := val.(*btSlice)
slice.m.Lock()
delete(slice.data, id)
slice.changed = time.Now()
slice.m.Unlock()
sqlStmt := "DELETE FROM blocktrees WHERE path LIKE ?"
_, err := db.Exec(sqlStmt, pathPrefix+"%")
if nil != err {
logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err)
return
}
}
func GetBlockTreesByBoxID(boxID string) (ret []*BlockTree) {
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if b.BoxID == boxID {
ret = append(ret, b)
}
sqlStmt := "SELECT * FROM blocktrees WHERE box_id = ?"
rows, err := db.Query(sqlStmt, boxID)
if nil != err {
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
return
}
defer rows.Close()
for rows.Next() {
var block BlockTree
if err = rows.Scan(&block.ID, &block.RootID, &block.ParentID, &block.BoxID, &block.Path, &block.HPath, &block.Updated, &block.Type); nil != err {
logging.LogErrorf("query scan field failed: %s", err)
return
}
slice.m.Unlock()
return true
})
ret = append(ret, &block)
}
return
}
func RemoveBlockTreesByBoxID(boxID string) (ids []string) {
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if b.BoxID == boxID {
ids = append(ids, b.ID)
}
sqlStmt := "SELECT id FROM blocktrees WHERE box_id = ?"
rows, err := db.Query(sqlStmt, boxID)
if nil != err {
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
return
}
defer rows.Close()
for rows.Next() {
var id string
if err = rows.Scan(&id); nil != err {
logging.LogErrorf("query scan field failed: %s", err)
return
}
slice.m.Unlock()
return true
})
ids = append(ids, id)
}
ids = gulu.Str.RemoveDuplicatedElem(ids)
for _, id := range ids {
val, ok := blockTrees.Load(btHash(id))
if !ok {
continue
}
slice := val.(*btSlice)
slice.m.Lock()
delete(slice.data, id)
slice.changed = time.Now()
slice.m.Unlock()
sqlStmt = "DELETE FROM blocktrees WHERE box_id = ?"
_, err = db.Exec(sqlStmt, boxID)
if nil != err {
logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err)
return
}
return
}
func RemoveBlockTree(id string) {
val, ok := blockTrees.Load(btHash(id))
if !ok {
sqlStmt := "DELETE FROM blocktrees WHERE id = ?"
_, err := db.Exec(sqlStmt, id)
if nil != err {
logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err)
return
}
slice := val.(*btSlice)
slice.m.Lock()
delete(slice.data, id)
slice.changed = time.Now()
slice.m.Unlock()
}
var indexBlockTreeLock = sync.Mutex{}
func IndexBlockTree(tree *parse.Tree) {
var changedNodes []*ast.Node
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering || !n.IsBlock() {
return ast.WalkContinue
}
if "" == n.ID {
if !entering || !n.IsBlock() || "" == n.ID {
return ast.WalkContinue
}
hash := btHash(n.ID)
val, ok := blockTrees.Load(hash)
if !ok {
val = &btSlice{data: map[string]*BlockTree{}, changed: time.Time{}, m: &sync.Mutex{}}
blockTrees.Store(hash, val)
changedNodes = append(changedNodes, n)
return ast.WalkContinue
})
indexBlockTreeLock.Lock()
defer indexBlockTreeLock.Unlock()
tx, err := db.Begin()
if nil != err {
logging.LogErrorf("begin transaction failed: %s", err)
return
}
sqlStmt := "INSERT INTO blocktrees (id, root_id, parent_id, box_id, path, hpath, updated, type) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
for _, n := range changedNodes {
var parentID string
if nil != n.Parent {
parentID = n.Parent.ID
}
slice := val.(*btSlice)
if _, err = tx.Exec(sqlStmt, n.ID, tree.ID, parentID, tree.Box, tree.Path, tree.HPath, n.IALAttr("updated"), TypeAbbr(n.Type.String())); nil != err {
tx.Rollback()
logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err)
return
}
}
if err = tx.Commit(); nil != err {
logging.LogErrorf("commit transaction failed: %s", err)
}
}
slice.m.Lock()
bt := slice.data[n.ID]
slice.m.Unlock()
func UpsertBlockTree(tree *parse.Tree) {
oldBts := map[string]*BlockTree{}
bts := GetBlockTreesByRootID(tree.ID)
for _, bt := range bts {
oldBts[bt.ID] = bt
}
if nil != bt {
if bt.Updated != n.IALAttr("updated") || bt.Type != TypeAbbr(n.Type.String()) || bt.Path != tree.Path || bt.BoxID != tree.Box || bt.HPath != tree.HPath {
var changedNodes []*ast.Node
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering || !n.IsBlock() || "" == n.ID {
return ast.WalkContinue
}
if oldBt, found := oldBts[n.ID]; found {
if oldBt.Updated != n.IALAttr("updated") || oldBt.Type != TypeAbbr(n.Type.String()) || oldBt.Path != tree.Path || oldBt.BoxID != tree.Box || oldBt.HPath != tree.HPath {
children := ChildBlockNodes(n) // 需要考虑子块，因为一些操作（比如移动块）后需要同时更新子块
changedNodes = append(changedNodes, children...)
}
@ -404,179 +501,60 @@ func IndexBlockTree(tree *parse.Tree) {
return ast.WalkContinue
})
ids := bytes.Buffer{}
for i, n := range changedNodes {
ids.WriteString("'")
ids.WriteString(n.ID)
ids.WriteString("'")
if i < len(changedNodes)-1 {
ids.WriteString(",")
}
}
indexBlockTreeLock.Lock()
defer indexBlockTreeLock.Unlock()
tx, err := db.Begin()
if nil != err {
logging.LogErrorf("begin transaction failed: %s", err)
return
}
sqlStmt := "DELETE FROM blocktrees WHERE id IN (" + ids.String() + ")"
_, err = tx.Exec(sqlStmt)
if nil != err {
tx.Rollback()
logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err)
return
}
sqlStmt = "INSERT INTO blocktrees (id, root_id, parent_id, box_id, path, hpath, updated, type) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
for _, n := range changedNodes {
updateBtSlice(n, tree)
var parentID string
if nil != n.Parent {
parentID = n.Parent.ID
}
if _, err = tx.Exec(sqlStmt, n.ID, tree.ID, parentID, tree.Box, tree.Path, tree.HPath, n.IALAttr("updated"), TypeAbbr(n.Type.String())); nil != err {
tx.Rollback()
logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err)
return
}
}
if err = tx.Commit(); nil != err {
logging.LogErrorf("commit transaction failed: %s", err)
}
}
func updateBtSlice(n *ast.Node, tree *parse.Tree) {
var parentID string
if nil != n.Parent {
parentID = n.Parent.ID
}
hash := btHash(n.ID)
val, ok := blockTrees.Load(hash)
if !ok {
val = &btSlice{data: map[string]*BlockTree{}, changed: time.Time{}, m: &sync.Mutex{}}
blockTrees.Store(hash, val)
}
slice := val.(*btSlice)
slice.m.Lock()
slice.data[n.ID] = &BlockTree{ID: n.ID, ParentID: parentID, RootID: tree.ID, BoxID: tree.Box, Path: tree.Path, HPath: tree.HPath, Updated: n.IALAttr("updated"), Type: TypeAbbr(n.Type.String())}
slice.changed = time.Now()
slice.m.Unlock()
}
var blockTreeLock = sync.Mutex{}
func InitBlockTree(force bool) {
blockTreeLock.Lock()
defer blockTreeLock.Unlock()
start := time.Now()
if force {
err := os.RemoveAll(util.BlockTreePath)
if nil != err {
logging.LogErrorf("remove block tree file failed: %s", err)
}
blockTrees = &sync.Map{}
return
}
entries, err := os.ReadDir(util.BlockTreePath)
err := initDatabase(force)
if nil != err {
logging.LogErrorf("read block tree dir failed: %s", err)
os.Exit(logging.ExitCodeFileSysErr)
logging.LogErrorf("init database failed: %s", err)
os.Exit(logging.ExitCodeReadOnlyDatabase)
return
}
loadErr := atomic.Bool{}
size := atomic.Int64{}
waitGroup := &sync.WaitGroup{}
p, _ := ants.NewPoolWithFunc(4, func(arg interface{}) {
defer waitGroup.Done()
entry := arg.(os.DirEntry)
p := filepath.Join(util.BlockTreePath, entry.Name())
f, err := os.OpenFile(p, os.O_RDONLY, 0644)
if nil != err {
logging.LogErrorf("open block tree failed: %s", err)
loadErr.Store(true)
return
}
defer f.Close()
info, err := f.Stat()
if nil != err {
logging.LogErrorf("stat block tree failed: %s", err)
loadErr.Store(true)
return
}
size.Add(info.Size())
sliceData := map[string]*BlockTree{}
if err = msgpack.NewDecoder(f).Decode(&sliceData); nil != err {
logging.LogErrorf("unmarshal block tree failed: %s", err)
loadErr.Store(true)
return
}
name := entry.Name()[0:strings.Index(entry.Name(), ".")]
blockTrees.Store(name, &btSlice{data: sliceData, changed: time.Time{}, m: &sync.Mutex{}})
})
for _, entry := range entries {
if !strings.HasSuffix(entry.Name(), ".msgpack") {
continue
}
waitGroup.Add(1)
p.Invoke(entry)
}
waitGroup.Wait()
p.Release()
if loadErr.Load() {
logging.LogInfof("cause block tree load error, remove block tree file")
if removeErr := os.RemoveAll(util.BlockTreePath); nil != removeErr {
logging.LogErrorf("remove block tree file failed: %s", removeErr)
os.Exit(logging.ExitCodeFileSysErr)
return
}
blockTrees = &sync.Map{}
return
}
elapsed := time.Since(start).Seconds()
logging.LogInfof("read block tree [%s] to [%s], elapsed [%.2fs]", humanize.BytesCustomCeil(uint64(size.Load()), 2), util.BlockTreePath, elapsed)
return
}
func SaveBlockTreeJob() {
SaveBlockTree(false)
}
func SaveBlockTree(force bool) {
blockTreeLock.Lock()
defer blockTreeLock.Unlock()
if task.ContainIndexTask() {
//logging.LogInfof("skip saving block tree because indexing")
return
}
//logging.LogInfof("saving block tree")
start := time.Now()
if err := os.MkdirAll(util.BlockTreePath, 0755); nil != err {
logging.LogErrorf("create block tree dir [%s] failed: %s", util.BlockTreePath, err)
os.Exit(logging.ExitCodeFileSysErr)
return
}
size := uint64(0)
var count int
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
if !force && slice.changed.IsZero() {
slice.m.Unlock()
return true
}
data, err := msgpack.Marshal(slice.data)
if nil != err {
logging.LogErrorf("marshal block tree failed: %s", err)
os.Exit(logging.ExitCodeFileSysErr)
return false
}
slice.m.Unlock()
p := filepath.Join(util.BlockTreePath, key.(string)) + ".msgpack"
if err = gulu.File.WriteFileSafer(p, data, 0644); nil != err {
logging.LogErrorf("write block tree failed: %s", err)
os.Exit(logging.ExitCodeFileSysErr)
return false
}
slice.m.Lock()
slice.changed = time.Time{}
slice.m.Unlock()
size += uint64(len(data))
count++
return true
})
if 0 < count {
//logging.LogInfof("wrote block trees [%d]", count)
}
elapsed := time.Since(start).Seconds()
if 2 < elapsed {
logging.LogWarnf("save block tree [size=%s] to [%s], elapsed [%.2fs]", humanize.BytesCustomCeil(size, 2), util.BlockTreePath, elapsed)
}
}
func CeilTreeCount(count int) int {
if 100 > count {
return 100
@ -602,7 +580,3 @@ func CeilBlockCount(count int) int {
}
return 10000*100 + 1
}
func btHash(id string) string {
return util2.Hash([]byte(id))[0:2]
}

91
kernel/treenode/blocktree_fix.go
View file

@ -18,7 +18,7 @@ package treenode
import (
"github.com/88250/gulu"
"time"
"github.com/siyuan-note/logging"
)
func ClearRedundantBlockTrees(boxID string, paths []string) {
@ -35,17 +35,21 @@ func getRedundantPaths(boxID string, paths []string) (ret []string) {
}
btPathsMap := map[string]bool{}
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if b.BoxID == boxID {
btPathsMap[b.Path] = true
}
sqlStmt := "SELECT path FROM blocktrees WHERE box_id = ?"
rows, err := db.Query(sqlStmt, boxID)
if nil != err {
logging.LogErrorf("query block tree failed: %s", err)
return
}
defer rows.Close()
for rows.Next() {
var path string
if err = rows.Scan(&path); nil != err {
logging.LogErrorf("scan block tree failed: %s", err)
return
}
slice.m.Unlock()
return true
})
btPathsMap[path] = true
}
for p, _ := range btPathsMap {
if !pathsMap[p] {
@ -57,18 +61,11 @@ func getRedundantPaths(boxID string, paths []string) (ret []string) {
}
func removeBlockTreesByPath(boxID, path string) {
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if b.Path == path && b.BoxID == boxID {
delete(slice.data, b.ID)
slice.changed = time.Now()
}
}
slice.m.Unlock()
return true
})
sqlStmt := "DELETE FROM blocktrees WHERE box_id = ? AND path = ?"
_, err := db.Exec(sqlStmt, boxID, path)
if nil != err {
logging.LogErrorf("delete block tree failed: %s", err)
}
}
func GetNotExistPaths(boxID string, paths []string) (ret []string) {
@ -78,17 +75,21 @@ func GetNotExistPaths(boxID string, paths []string) (ret []string) {
}
btPathsMap := map[string]bool{}
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if b.BoxID == boxID {
btPathsMap[b.Path] = true
}
sqlStmt := "SELECT path FROM blocktrees WHERE box_id = ?"
rows, err := db.Query(sqlStmt, boxID)
if nil != err {
logging.LogErrorf("query block tree failed: %s", err)
return
}
defer rows.Close()
for rows.Next() {
var path string
if err = rows.Scan(&path); nil != err {
logging.LogErrorf("scan block tree failed: %s", err)
return
}
slice.m.Unlock()
return true
})
btPathsMap[path] = true
}
for p, _ := range pathsMap {
if !btPathsMap[p] {
@ -101,16 +102,20 @@ func GetNotExistPaths(boxID string, paths []string) (ret []string) {
func GetRootUpdated() (ret map[string]string) {
ret = map[string]string{}
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if b.RootID == b.ID {
ret[b.RootID] = b.Updated
}
sqlStmt := "SELECT root_id, updated FROM blocktrees WHERE root_id = id AND type = 'd'"
rows, err := db.Query(sqlStmt)
if nil != err {
logging.LogErrorf("query block tree failed: %s", err)
return
}
defer rows.Close()
for rows.Next() {
var rootID, updated string
if err = rows.Scan(&rootID, &updated); nil != err {
logging.LogErrorf("scan block tree failed: %s", err)
return
}
slice.m.Unlock()
return true
})
ret[rootID] = updated
}
return
}

4
kernel/util/working.go
View file

@ -209,7 +209,7 @@ var (
DBPath string // SQLite 数据库文件路径
HistoryDBPath string // SQLite 历史数据库文件路径
AssetContentDBPath string // SQLite 资源文件内容数据库文件路径
BlockTreePath string // 区块树文件路径
BlockTreeDBPath string // 区块树数据库文件路径
AppearancePath string // 配置目录下的外观目录 appearance/ 路径
ThemesPath string // 配置目录下的外观目录下的 themes/ 路径
IconsPath string // 配置目录下的外观目录下的 icons/ 路径
@ -287,7 +287,7 @@ func initWorkspaceDir(workspaceArg string) {
DBPath = filepath.Join(TempDir, DBName)
HistoryDBPath = filepath.Join(TempDir, "history.db")
AssetContentDBPath = filepath.Join(TempDir, "asset_content.db")
BlockTreePath = filepath.Join(TempDir, "blocktree")
BlockTreeDBPath = filepath.Join(TempDir, "blocktree.db")
SnippetsPath = filepath.Join(DataDir, "snippets")
}

2
kernel/util/working_mobile.go
View file

@ -159,7 +159,7 @@ func initWorkspaceDirMobile(workspaceBaseDir string) {
DBPath = filepath.Join(TempDir, DBName)
HistoryDBPath = filepath.Join(TempDir, "history.db")
AssetContentDBPath = filepath.Join(TempDir, "asset_content.db")
BlockTreePath = filepath.Join(TempDir, "blocktree")
BlockTreeDBPath = filepath.Join(TempDir, "blocktree.db")
SnippetsPath = filepath.Join(DataDir, "snippets")
AppearancePath = filepath.Join(ConfDir, "appearance")