Merge remote-tracking branch 'origin/dev' into dev
This commit is contained in:
Vanessa
commit
67d991aa22
27 changed files with 480 additions and 520 deletions
6
.github/CONTRIBUTING.md
vendored
6
.github/CONTRIBUTING.md
vendored
|
|
@ -14,10 +14,10 @@ Install pnpm: `npm install -g pnpm@9.1.1`
|
|||
|
||||
Set the Electron mirror environment variable and install Electron:
|
||||
|
||||
* macOS/Linux: `ELECTRON_MIRROR=https://npmmirror.com/mirrors/electron/ pnpm install electron@31.0.1 -D`
|
||||
* macOS/Linux: `ELECTRON_MIRROR=https://npmmirror.com/mirrors/electron/ pnpm install electron@31.0.2 -D`
|
||||
* Windows:
|
||||
* `SET ELECTRON_MIRROR=https://npmmirror.com/mirrors/electron/`
|
||||
* `pnpm install electron@31.0.1 -D`
|
||||
* `pnpm install electron@31.0.2 -D`
|
||||
|
||||
NPM mirror:
|
||||
|
||||
|
|
@ -28,7 +28,7 @@ NPM mirror:
|
|||
|
||||
On the desktop, go to the app folder to run:
|
||||
|
||||
* `pnpm install electron@31.0.1 -D`
|
||||
* `pnpm install electron@31.0.2 -D`
|
||||
* `pnpm run dev`
|
||||
* `pnpm run start`
|
||||
|
||||
|
|
|
|||
6
.github/CONTRIBUTING_zh_CN.md
vendored
6
.github/CONTRIBUTING_zh_CN.md
vendored
|
|
@ -16,11 +16,11 @@
|
|||
|
||||
* macOS/Linux:
|
||||
```
|
||||
ELECTRON_MIRROR=https://npmmirror.com/mirrors/electron/ pnpm install electron@31.0.1 -D
|
||||
ELECTRON_MIRROR=https://npmmirror.com/mirrors/electron/ pnpm install electron@31.0.2 -D
|
||||
```
|
||||
* Windows:
|
||||
* `SET ELECTRON_MIRROR=https://npmmirror.com/mirrors/electron/`
|
||||
* `pnpm install electron@31.0.1 -D`
|
||||
* `pnpm install electron@31.0.2 -D`
|
||||
|
||||
NPM 镜像：
|
||||
|
||||
|
|
@ -30,7 +30,7 @@ NPM 镜像：
|
|||
|
||||
桌面端进入 app 文件夹运行：
|
||||
|
||||
* `pnpm install electron@31.0.1 -D`
|
||||
* `pnpm install electron@31.0.2 -D`
|
||||
* `pnpm run dev`
|
||||
* `pnpm run start`
|
||||
|
||||
|
|
|
|||
|
|
@ -1377,7 +1377,7 @@
|
|||
"88": "Finished parsing [%d] data files, remaining to be processed [%d]",
|
||||
"89": "[%d/%d] Created [%d] of data indexes of block-level elements [%s]",
|
||||
"90": "[%d/%d] Created [%d] of search indexes of block-level elements [%s]",
|
||||
"91": "Reading block tree data...",
|
||||
"91": "TODO",
|
||||
"92": "Parsing document tree [%s]",
|
||||
"93": "[%d/%d] Cleaned up the index related to document [%s]",
|
||||
"94": "Upload failed: %s",
|
||||
|
|
|
|||
|
|
@ -1377,7 +1377,7 @@
|
|||
"88": "Se ha terminado de analizar [%d] archivos de datos, quedan por procesar [%d]",
|
||||
"89": "[%d/%d] Creado [%d] de índices de datos de elementos a nivel de bloque [%s]",
|
||||
"90": "[%d/%d] Creado [%d] de índices de búsqueda de elementos a nivel de bloque [%s]",
|
||||
"91": "Leyendo datos del árbol de bloques...",
|
||||
"91": "TODO",
|
||||
"92": "Analizando el árbol del documento [%s]",
|
||||
"93": "[%d/%d] ha limpiado el índice relacionado con el documento [%s]",
|
||||
"94": "Carga fallida: %s",
|
||||
|
|
|
|||
|
|
@ -1377,7 +1377,7 @@
|
|||
"88": "Fin de l'analyse des fichiers de données [%d], restant à traiter [%d]",
|
||||
"89": "[%d/%d] Créé [%d] d'index de données d'éléments de niveau bloc [%s]",
|
||||
"90": "[%d/%d] Création de [%d] index de recherche d'éléments de niveau bloc [%s]",
|
||||
"91": "Lecture des données de l'arborescence des blocs...",
|
||||
"91": "TODO",
|
||||
"92": "Analyse de l'arborescence du document [%s]",
|
||||
"93": "[%d/%d] a nettoyé l'index lié au document [%s]",
|
||||
"94": "Échec du téléchargement : %s",
|
||||
|
|
|
|||
|
|
@ -1377,7 +1377,7 @@
|
|||
"88": "[%d] 個のデータファイルの解析が完了し、処理待ちのデータファイルが [%d] 個残っています",
|
||||
"89": "[%d/%d] ブロックレベル要素 [%s] のデータインデックスを [%d] 個作成しました",
|
||||
"90": "[%d/%d] ブロックレベル要素 [%s] の検索インデックスを [%d] 個作成しました",
|
||||
"91": "ブロックツリーデータを読み込んでいます...",
|
||||
"91": "TODO",
|
||||
"92": "ドキュメントツリーを解析しています [%s]",
|
||||
"93": "[%d/%d] ドキュメント [%s] に関連するインデックスをクリーンアップしました",
|
||||
"94": "アップロードに失敗しました: %s",
|
||||
|
|
|
|||
|
|
@ -1377,7 +1377,7 @@
|
|||
"88": "已完成解析 [%d] 個資料文件，剩餘待處理 [%d]",
|
||||
"89": "[%d/%d] 已經創建 [%d] 個塊級元素的資料索引 [%s]",
|
||||
"90": "[%d/%d] 已經創建 [%d] 個塊級元素的搜索索引 [%s]",
|
||||
"91": "正在讀取塊樹資料...",
|
||||
"91": "TODO",
|
||||
"92": "正在解析文檔樹 [%s]",
|
||||
"93": "[%d/%d] 已經清理文檔 [%s] 相關的索引",
|
||||
"94": "上傳失敗：%s",
|
||||
|
|
|
|||
|
|
@ -1377,7 +1377,7 @@
|
|||
"88": "已完成解析 [%d] 个数据文件，剩余待处理 [%d]",
|
||||
"89": "[%d/%d] 已经创建 [%d] 个块级元素的数据索引 [%s]",
|
||||
"90": "[%d/%d] 已经创建 [%d] 个块级元素的搜索索引 [%s]",
|
||||
"91": "正在读取块树数据...",
|
||||
"91": "TODO",
|
||||
"92": "正在解析文档树 [%s]",
|
||||
"93": "[%d/%d] 已经清理文档 [%s] 相关的索引",
|
||||
"94": "上传失败：%s",
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@
|
|||
"clean-webpack-plugin": "^4.0.0",
|
||||
"css-loader": "^6.7.1",
|
||||
"dayjs": "^1.11.5",
|
||||
"electron": "31.0.1",
|
||||
"electron": "31.0.2",
|
||||
"electron-builder": "24.9.1",
|
||||
"encoding": "^0.1.13",
|
||||
"esbuild-loader": "^3.0.1",
|
||||
|
|
|
|||
16
app/pnpm-lock.yaml
generated
16
app/pnpm-lock.yaml
generated
|
|
@ -10,7 +10,7 @@ importers:
|
|||
dependencies:
|
||||
'@electron/remote':
|
||||
specifier: ^2.1.2
|
||||
version: 2.1.2(electron@31.0.1)
|
||||
version: 2.1.2(electron@31.0.2)
|
||||
devDependencies:
|
||||
'@types/node':
|
||||
specifier: ^18.13.0
|
||||
|
|
@ -34,8 +34,8 @@ importers:
|
|||
specifier: ^1.11.5
|
||||
version: 1.11.5
|
||||
electron:
|
||||
specifier: 31.0.1
|
||||
version: 31.0.1
|
||||
specifier: 31.0.2
|
||||
version: 31.0.2
|
||||
electron-builder:
|
||||
specifier: 24.9.1
|
||||
version: 24.9.1
|
||||
|
|
@ -925,8 +925,8 @@ packages:
|
|||
electron-to-chromium@1.4.186:
|
||||
resolution: {integrity: sha512-YoVeFrGd/7ROjz4R9uPoND1K/hSRC/xADy9639ZmIZeJSaBnKdYx3I6LMPsY7CXLpK7JFgKQVzeZ/dk2br6Eaw==}
|
||||
|
||||
electron@31.0.1:
|
||||
resolution: {integrity: sha512-2eBcp4iqLkTsml6mMq+iqrS5u3kJ/2mpOLP7Mj7lo0uNK3OyfNqRS9z1ArsHjBF2/HV250Te/O9nKrwQRTX/+g==}
|
||||
electron@31.0.2:
|
||||
resolution: {integrity: sha512-55efQ5yfLN+AQHcFC00AXQqtxC3iAGaxX2GQ3EDbFJ0ca9GHNOdSXkcrdBElLleiDrR2hpXNkQxN1bDn0oxe6w==}
|
||||
engines: {node: '>= 12.20.55'}
|
||||
hasBin: true
|
||||
|
||||
|
|
@ -2249,9 +2249,9 @@ snapshots:
|
|||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
'@electron/remote@2.1.2(electron@31.0.1)':
|
||||
'@electron/remote@2.1.2(electron@31.0.2)':
|
||||
dependencies:
|
||||
electron: 31.0.1
|
||||
electron: 31.0.2
|
||||
|
||||
'@electron/universal@1.4.1':
|
||||
dependencies:
|
||||
|
|
@ -3142,7 +3142,7 @@ snapshots:
|
|||
|
||||
electron-to-chromium@1.4.186: {}
|
||||
|
||||
electron@31.0.1:
|
||||
electron@31.0.2:
|
||||
dependencies:
|
||||
'@electron/get': 2.0.2
|
||||
'@types/node': 20.14.5
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ func LoadTreeByData(data []byte, boxID, p string, luteEngine *lute.Lute) (ret *p
|
|||
logging.LogErrorf("rebuild parent tree [%s] failed: %s", parentAbsPath, writeErr)
|
||||
} else {
|
||||
logging.LogInfof("rebuilt parent tree [%s]", parentAbsPath)
|
||||
treenode.IndexBlockTree(parentTree)
|
||||
treenode.UpsertBlockTree(parentTree)
|
||||
}
|
||||
} else {
|
||||
logging.LogWarnf("read parent tree data [%s] failed: %s", parentAbsPath, readErr)
|
||||
|
|
@ -137,7 +137,7 @@ func prepareWriteTree(tree *parse.Tree) (data []byte, filePath string, err error
|
|||
newP := treenode.NewParagraph()
|
||||
tree.Root.AppendChild(newP)
|
||||
tree.Root.SetIALAttr("updated", util.TimeFromID(newP.ID))
|
||||
treenode.IndexBlockTree(tree)
|
||||
treenode.UpsertBlockTree(tree)
|
||||
}
|
||||
|
||||
filePath = filepath.Join(util.DataDir, tree.Box, tree.Path)
|
||||
|
|
|
|||
|
|
@ -23,14 +23,12 @@ import (
|
|||
"github.com/siyuan-note/siyuan/kernel/model"
|
||||
"github.com/siyuan-note/siyuan/kernel/sql"
|
||||
"github.com/siyuan-note/siyuan/kernel/task"
|
||||
"github.com/siyuan-note/siyuan/kernel/treenode"
|
||||
"github.com/siyuan-note/siyuan/kernel/util"
|
||||
)
|
||||
|
||||
func StartCron() {
|
||||
go every(100*time.Millisecond, task.ExecTaskJob)
|
||||
go every(5*time.Second, task.StatusJob)
|
||||
go every(5*time.Second, treenode.SaveBlockTreeJob)
|
||||
go every(5*time.Second, model.SyncDataJob)
|
||||
go every(2*time.Hour, model.StatJob)
|
||||
go every(2*time.Hour, model.RefreshCheckJob)
|
||||
|
|
|
|||
|
|
@ -825,7 +825,7 @@ func RenameAsset(oldPath, newName string) (err error) {
|
|||
continue
|
||||
}
|
||||
|
||||
treenode.IndexBlockTree(tree)
|
||||
treenode.UpsertBlockTree(tree)
|
||||
sql.UpsertTreeQueue(tree)
|
||||
|
||||
util.PushEndlessProgress(fmt.Sprintf(Conf.Language(111), util.EscapeHTML(tree.Root.IALAttr("title"))))
|
||||
|
|
|
|||
|
|
@ -531,7 +531,6 @@ func fullReindex() {
|
|||
for _, openedBox := range openedBoxes {
|
||||
index(openedBox.ID)
|
||||
}
|
||||
treenode.SaveBlockTree(true)
|
||||
LoadFlashcards()
|
||||
debug.FreeOSMemory()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -627,7 +627,6 @@ func Close(force, setCurrentWorkspace bool, execInstallPkg int) (exitCode int) {
|
|||
|
||||
Conf.Close()
|
||||
sql.CloseDatabase()
|
||||
treenode.SaveBlockTree(false)
|
||||
util.SaveAssetsTexts()
|
||||
clearWorkspaceTemp()
|
||||
clearCorruptedNotebooks()
|
||||
|
|
@ -818,24 +817,7 @@ func (conf *AppConf) language(num int) (ret string) {
|
|||
}
|
||||
|
||||
func InitBoxes() {
|
||||
initialized := false
|
||||
if 1 > treenode.CountBlocks() {
|
||||
if gulu.File.IsExist(util.BlockTreePath) {
|
||||
util.IncBootProgress(20, Conf.Language(91))
|
||||
go func() {
|
||||
for i := 0; i < 40; i++ {
|
||||
util.RandomSleep(50, 100)
|
||||
util.IncBootProgress(1, Conf.Language(91))
|
||||
}
|
||||
}()
|
||||
|
||||
treenode.InitBlockTree(false)
|
||||
initialized = true
|
||||
}
|
||||
} else { // 大于 1 的话说明在同步阶段已经加载过了
|
||||
initialized = true
|
||||
}
|
||||
|
||||
initialized := 0 < treenode.CountBlocks()
|
||||
for _, box := range Conf.GetOpenedBoxes() {
|
||||
box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间
|
||||
|
||||
|
|
@ -844,10 +826,6 @@ func InitBoxes() {
|
|||
}
|
||||
}
|
||||
|
||||
if !initialized {
|
||||
treenode.SaveBlockTree(true)
|
||||
}
|
||||
|
||||
var dbSize string
|
||||
if dbFile, err := os.Stat(util.DBPath); nil == err {
|
||||
dbSize = humanize.BytesCustomCeil(uint64(dbFile.Size()), 2)
|
||||
|
|
@ -982,7 +960,8 @@ func clearWorkspaceTemp() {
|
|||
os.RemoveAll(filepath.Join(util.TempDir, "import"))
|
||||
os.RemoveAll(filepath.Join(util.TempDir, "repo"))
|
||||
os.RemoveAll(filepath.Join(util.TempDir, "os"))
|
||||
os.RemoveAll(filepath.Join(util.TempDir, "blocktree.msgpack")) // v2.7.2 前旧版的块数数据
|
||||
os.RemoveAll(filepath.Join(util.TempDir, "blocktree.msgpack")) // v2.7.2 前旧版的块树数据
|
||||
os.RemoveAll(filepath.Join(util.TempDir, "blocktree")) // v3.1.0 前旧版的块树数据
|
||||
|
||||
// 退出时自动删除超过 7 天的安装包 https://github.com/siyuan-note/siyuan/issues/6128
|
||||
install := filepath.Join(util.TempDir, "install")
|
||||
|
|
|
|||
|
|
@ -535,7 +535,7 @@ func ExportResources(resourcePaths []string, mainName string) (exportFilePath st
|
|||
|
||||
func Preview(id string) (retStdHTML string, retOutline []*Path) {
|
||||
tree, _ := LoadTreeByBlockID(id)
|
||||
tree = exportTree(tree, false, false,
|
||||
tree = exportTree(tree, false, false, true,
|
||||
Conf.Export.BlockRefMode, Conf.Export.BlockEmbedMode, Conf.Export.FileAnnotationRefMode,
|
||||
Conf.Export.TagOpenMarker, Conf.Export.TagCloseMarker,
|
||||
Conf.Export.BlockRefTextLeft, Conf.Export.BlockRefTextRight,
|
||||
|
|
@ -640,7 +640,7 @@ func ExportMarkdownHTML(id, savePath string, docx, merge bool) (name, dom string
|
|||
}
|
||||
}
|
||||
|
||||
tree = exportTree(tree, true, false,
|
||||
tree = exportTree(tree, true, false, !docx,
|
||||
Conf.Export.BlockRefMode, Conf.Export.BlockEmbedMode, Conf.Export.FileAnnotationRefMode,
|
||||
Conf.Export.TagOpenMarker, Conf.Export.TagCloseMarker,
|
||||
Conf.Export.BlockRefTextLeft, Conf.Export.BlockRefTextRight,
|
||||
|
|
@ -790,7 +790,7 @@ func ExportHTML(id, savePath string, pdf, image, keepFold, merge bool) (name, do
|
|||
}
|
||||
}
|
||||
|
||||
tree = exportTree(tree, true, keepFold,
|
||||
tree = exportTree(tree, true, keepFold, true,
|
||||
Conf.Export.BlockRefMode, Conf.Export.BlockEmbedMode, Conf.Export.FileAnnotationRefMode,
|
||||
Conf.Export.TagOpenMarker, Conf.Export.TagCloseMarker,
|
||||
Conf.Export.BlockRefTextLeft, Conf.Export.BlockRefTextRight,
|
||||
|
|
@ -1837,7 +1837,7 @@ func exportMarkdownContent0(tree *parse.Tree, cloudAssetsBase string, assetsDest
|
|||
blockRefTextLeft, blockRefTextRight string,
|
||||
addTitle bool,
|
||||
defBlockIDs []string) (ret string) {
|
||||
tree = exportTree(tree, false, false,
|
||||
tree = exportTree(tree, false, false, true,
|
||||
blockRefMode, blockEmbedMode, fileAnnotationRefMode,
|
||||
tagOpenMarker, tagCloseMarker,
|
||||
blockRefTextLeft, blockRefTextRight,
|
||||
|
|
@ -1926,7 +1926,7 @@ func exportMarkdownContent0(tree *parse.Tree, cloudAssetsBase string, assetsDest
|
|||
return
|
||||
}
|
||||
|
||||
func exportTree(tree *parse.Tree, wysiwyg, keepFold bool,
|
||||
func exportTree(tree *parse.Tree, wysiwyg, keepFold, pdfPageNum bool,
|
||||
blockRefMode, blockEmbedMode, fileAnnotationRefMode int,
|
||||
tagOpenMarker, tagCloseMarker string,
|
||||
blockRefTextLeft, blockRefTextRight string,
|
||||
|
|
@ -1975,7 +1975,7 @@ func exportTree(tree *parse.Tree, wysiwyg, keepFold bool,
|
|||
return ast.WalkSkipChildren
|
||||
}
|
||||
|
||||
status := processFileAnnotationRef(refID, n, fileAnnotationRefMode)
|
||||
status := processFileAnnotationRef(refID, n, fileAnnotationRefMode, pdfPageNum)
|
||||
unlinks = append(unlinks, n)
|
||||
return status
|
||||
} else if n.IsTextMarkType("tag") {
|
||||
|
|
@ -2551,7 +2551,7 @@ func exportRefTrees0(tree *parse.Tree, retTrees *map[string]*parse.Tree) {
|
|||
})
|
||||
}
|
||||
|
||||
func processFileAnnotationRef(refID string, n *ast.Node, fileAnnotationRefMode int) ast.WalkStatus {
|
||||
func processFileAnnotationRef(refID string, n *ast.Node, fileAnnotationRefMode int, pdfPageNum bool) ast.WalkStatus {
|
||||
p := refID[:strings.LastIndex(refID, "/")]
|
||||
absPath, err := GetAssetAbsPath(p)
|
||||
if nil != err {
|
||||
|
|
@ -2591,7 +2591,11 @@ func processFileAnnotationRef(refID string, n *ast.Node, fileAnnotationRefMode i
|
|||
}
|
||||
fileAnnotationRefLink.AppendChild(&ast.Node{Type: ast.NodeCloseBracket})
|
||||
fileAnnotationRefLink.AppendChild(&ast.Node{Type: ast.NodeOpenParen})
|
||||
fileAnnotationRefLink.AppendChild(&ast.Node{Type: ast.NodeLinkDest, Tokens: []byte(p + "?p=" + pageStr)})
|
||||
dest := p
|
||||
if pdfPageNum {
|
||||
dest += "?p=" + pageStr
|
||||
}
|
||||
fileAnnotationRefLink.AppendChild(&ast.Node{Type: ast.NodeLinkDest, Tokens: []byte(dest)})
|
||||
fileAnnotationRefLink.AppendChild(&ast.Node{Type: ast.NodeCloseParen})
|
||||
n.InsertBefore(fileAnnotationRefLink)
|
||||
return ast.WalkSkipChildren
|
||||
|
|
|
|||
|
|
@ -1086,7 +1086,7 @@ func indexWriteTreeIndexQueue(tree *parse.Tree) (err error) {
|
|||
}
|
||||
|
||||
func indexWriteTreeUpsertQueue(tree *parse.Tree) (err error) {
|
||||
treenode.IndexBlockTree(tree)
|
||||
treenode.UpsertBlockTree(tree)
|
||||
return writeTreeUpsertQueue(tree)
|
||||
}
|
||||
|
||||
|
|
@ -1095,7 +1095,7 @@ func renameWriteJSONQueue(tree *parse.Tree) (err error) {
|
|||
return
|
||||
}
|
||||
sql.RenameTreeQueue(tree)
|
||||
treenode.IndexBlockTree(tree)
|
||||
treenode.UpsertBlockTree(tree)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -478,7 +478,7 @@ func reindexTree0(tree *parse.Tree, i, size int) {
|
|||
tree.Root.SetIALAttr("updated", updated)
|
||||
indexWriteTreeUpsertQueue(tree)
|
||||
} else {
|
||||
treenode.IndexBlockTree(tree)
|
||||
treenode.UpsertBlockTree(tree)
|
||||
sql.IndexTreeQueue(tree)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ import (
|
|||
"github.com/88250/lute/ast"
|
||||
"github.com/siyuan-note/filelock"
|
||||
"github.com/siyuan-note/logging"
|
||||
"github.com/siyuan-note/siyuan/kernel/treenode"
|
||||
"github.com/siyuan-note/siyuan/kernel/util"
|
||||
)
|
||||
|
||||
|
|
@ -247,7 +246,6 @@ func Mount(boxID string) (alreadyMount bool, err error) {
|
|||
box.Index()
|
||||
// 缓存根一级的文档树展开
|
||||
ListDocTree(box.ID, "/", util.SortModeUnassigned, false, false, Conf.FileTree.MaxListCount)
|
||||
treenode.SaveBlockTree(false)
|
||||
util.ClearPushProgress(100)
|
||||
|
||||
if reMountGuide {
|
||||
|
|
|
|||
|
|
@ -352,7 +352,7 @@ func upsertIndexes(upsertFilePaths []string) (upsertRootIDs []string) {
|
|||
if nil != err0 {
|
||||
continue
|
||||
}
|
||||
treenode.IndexBlockTree(tree)
|
||||
treenode.UpsertBlockTree(tree)
|
||||
sql.UpsertTreeQueue(tree)
|
||||
|
||||
bts := treenode.GetBlockTreesByRootID(tree.ID)
|
||||
|
|
|
|||
|
|
@ -1378,7 +1378,7 @@ func (tx *Transaction) loadTree(id string) (ret *parse.Tree, err error) {
|
|||
|
||||
func (tx *Transaction) writeTree(tree *parse.Tree) (err error) {
|
||||
tx.trees[tree.ID] = tree
|
||||
treenode.IndexBlockTree(tree)
|
||||
treenode.UpsertBlockTree(tree)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -274,7 +274,7 @@ func searchTreeInFilesystem(rootID string) {
|
|||
return
|
||||
}
|
||||
|
||||
treenode.IndexBlockTree(tree)
|
||||
treenode.UpsertBlockTree(tree)
|
||||
sql.IndexTreeQueue(tree)
|
||||
logging.LogInfof("reindexed tree by filesystem [rootID=%s]", rootID)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@ func InitDatabase(forceRebuild bool) (err error) {
|
|||
}
|
||||
|
||||
initDBConnection()
|
||||
treenode.InitBlockTree(forceRebuild)
|
||||
|
||||
if !forceRebuild {
|
||||
// 检查数据库结构版本，如果版本不一致的话说明改过表结构，需要重建
|
||||
|
|
@ -101,9 +102,6 @@ func InitDatabase(forceRebuild bool) (err error) {
|
|||
err = nil
|
||||
}
|
||||
}
|
||||
if gulu.File.IsExist(util.BlockTreePath) {
|
||||
treenode.InitBlockTree(true)
|
||||
}
|
||||
|
||||
initDBConnection()
|
||||
initDBTables()
|
||||
|
|
@ -1278,6 +1276,11 @@ func CloseDatabase() {
|
|||
logging.LogErrorf("close history database failed: %s", err)
|
||||
return
|
||||
}
|
||||
if err := assetContentDB.Close(); nil != err {
|
||||
logging.LogErrorf("close asset content database failed: %s", err)
|
||||
return
|
||||
}
|
||||
treenode.CloseDatabase()
|
||||
logging.LogInfof("closed database")
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -17,33 +17,22 @@
|
|||
package treenode
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"runtime"
|
||||
"runtime/debug"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/88250/go-humanize"
|
||||
"github.com/88250/gulu"
|
||||
"github.com/88250/lute/ast"
|
||||
"github.com/88250/lute/parse"
|
||||
"github.com/panjf2000/ants/v2"
|
||||
util2 "github.com/siyuan-note/dejavu/util"
|
||||
"github.com/siyuan-note/logging"
|
||||
"github.com/siyuan-note/siyuan/kernel/task"
|
||||
"github.com/siyuan-note/siyuan/kernel/util"
|
||||
"github.com/vmihailenco/msgpack/v5"
|
||||
)
|
||||
|
||||
var blockTrees = &sync.Map{}
|
||||
|
||||
type btSlice struct {
|
||||
data map[string]*BlockTree
|
||||
changed time.Time
|
||||
m *sync.Mutex
|
||||
}
|
||||
|
||||
type BlockTree struct {
|
||||
ID string // 块 ID
|
||||
RootID string // 根 ID
|
||||
|
|
@ -55,131 +44,237 @@ type BlockTree struct {
|
|||
Type string // 类型
|
||||
}
|
||||
|
||||
func GetBlockTreesByType(typ string) (ret []*BlockTree) {
|
||||
blockTrees.Range(func(key, value interface{}) bool {
|
||||
slice := value.(*btSlice)
|
||||
slice.m.Lock()
|
||||
for _, b := range slice.data {
|
||||
if b.Type == typ {
|
||||
ret = append(ret, b)
|
||||
}
|
||||
var (
|
||||
db *sql.DB
|
||||
)
|
||||
|
||||
func initDatabase(forceRebuild bool) (err error) {
|
||||
initDBConnection()
|
||||
|
||||
if !forceRebuild {
|
||||
if !gulu.File.IsExist(util.BlockTreeDBPath) {
|
||||
forceRebuild = true
|
||||
}
|
||||
slice.m.Unlock()
|
||||
return true
|
||||
})
|
||||
}
|
||||
if !forceRebuild {
|
||||
return
|
||||
}
|
||||
|
||||
closeDatabase()
|
||||
if gulu.File.IsExist(util.BlockTreeDBPath) {
|
||||
if err = removeDatabaseFile(); nil != err {
|
||||
logging.LogErrorf("remove database file [%s] failed: %s", util.BlockTreeDBPath, err)
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
|
||||
initDBConnection()
|
||||
initDBTables()
|
||||
|
||||
logging.LogInfof("reinitialized database [%s]", util.BlockTreeDBPath)
|
||||
return
|
||||
}
|
||||
|
||||
func initDBTables() {
|
||||
_, err := db.Exec("DROP TABLE IF EXISTS blocktrees")
|
||||
if nil != err {
|
||||
logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "drop table [blocks] failed: %s", err)
|
||||
}
|
||||
_, err = db.Exec("CREATE TABLE blocktrees (id, root_id, parent_id, box_id, path, hpath, updated, type)")
|
||||
if nil != err {
|
||||
logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create table [blocktrees] failed: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
func initDBConnection() {
|
||||
if nil != db {
|
||||
closeDatabase()
|
||||
}
|
||||
dsn := util.BlockTreeDBPath + "?_journal_mode=WAL" +
|
||||
"&_synchronous=OFF" +
|
||||
"&_mmap_size=2684354560" +
|
||||
"&_secure_delete=OFF" +
|
||||
"&_cache_size=-20480" +
|
||||
"&_page_size=32768" +
|
||||
"&_busy_timeout=7000" +
|
||||
"&_ignore_check_constraints=ON" +
|
||||
"&_temp_store=MEMORY" +
|
||||
"&_case_sensitive_like=OFF"
|
||||
var err error
|
||||
db, err = sql.Open("sqlite3_extended", dsn)
|
||||
if nil != err {
|
||||
logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create database failed: %s", err)
|
||||
}
|
||||
db.SetMaxIdleConns(7)
|
||||
db.SetMaxOpenConns(7)
|
||||
db.SetConnMaxLifetime(365 * 24 * time.Hour)
|
||||
}
|
||||
|
||||
func CloseDatabase() {
|
||||
closeDatabase()
|
||||
}
|
||||
|
||||
func closeDatabase() {
|
||||
if nil == db {
|
||||
return
|
||||
}
|
||||
|
||||
if err := db.Close(); nil != err {
|
||||
logging.LogErrorf("close database failed: %s", err)
|
||||
}
|
||||
debug.FreeOSMemory()
|
||||
runtime.GC() // 没有这句的话文件句柄不会释放，后面就无法删除文件
|
||||
return
|
||||
}
|
||||
|
||||
func removeDatabaseFile() (err error) {
|
||||
err = os.RemoveAll(util.BlockTreeDBPath)
|
||||
if nil != err {
|
||||
return
|
||||
}
|
||||
err = os.RemoveAll(util.BlockTreeDBPath + "-shm")
|
||||
if nil != err {
|
||||
return
|
||||
}
|
||||
err = os.RemoveAll(util.BlockTreeDBPath + "-wal")
|
||||
if nil != err {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func GetBlockTreesByType(typ string) (ret []*BlockTree) {
|
||||
sqlStmt := "SELECT * FROM blocktrees WHERE type = ?"
|
||||
rows, err := db.Query(sqlStmt)
|
||||
if nil != err {
|
||||
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var block BlockTree
|
||||
if err = rows.Scan(&block.ID, &block.RootID, &block.ParentID, &block.BoxID, &block.Path, &block.HPath, &block.Updated, &block.Type); nil != err {
|
||||
logging.LogErrorf("query scan field failed: %s", err)
|
||||
return
|
||||
}
|
||||
ret = append(ret, &block)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func GetBlockTreeByPath(path string) (ret *BlockTree) {
|
||||
blockTrees.Range(func(key, value interface{}) bool {
|
||||
slice := value.(*btSlice)
|
||||
slice.m.Lock()
|
||||
for _, b := range slice.data {
|
||||
if b.Path == path {
|
||||
ret = b
|
||||
break
|
||||
}
|
||||
ret = &BlockTree{}
|
||||
sqlStmt := "SELECT * FROM blocktrees WHERE path = ?"
|
||||
err := db.QueryRow(sqlStmt, path).Scan(&ret.ID, &ret.RootID, &ret.ParentID, &ret.BoxID, &ret.Path, &ret.HPath, &ret.Updated, &ret.Type)
|
||||
if nil != err {
|
||||
ret = nil
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return
|
||||
}
|
||||
slice.m.Unlock()
|
||||
return nil == ret
|
||||
})
|
||||
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func CountTrees() (ret int) {
|
||||
roots := map[string]bool{}
|
||||
blockTrees.Range(func(key, value interface{}) bool {
|
||||
slice := value.(*btSlice)
|
||||
slice.m.Lock()
|
||||
for _, b := range slice.data {
|
||||
roots[b.RootID] = true
|
||||
sqlStmt := "SELECT COUNT(*) FROM blocktrees WHERE type = 'd'"
|
||||
err := db.QueryRow(sqlStmt).Scan(&ret)
|
||||
if nil != err {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return 0
|
||||
}
|
||||
slice.m.Unlock()
|
||||
return true
|
||||
})
|
||||
ret = len(roots)
|
||||
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func CountBlocks() (ret int) {
|
||||
blockTrees.Range(func(key, value interface{}) bool {
|
||||
slice := value.(*btSlice)
|
||||
slice.m.Lock()
|
||||
ret += len(slice.data)
|
||||
slice.m.Unlock()
|
||||
return true
|
||||
})
|
||||
sqlStmt := "SELECT COUNT(*) FROM blocktrees"
|
||||
err := db.QueryRow(sqlStmt).Scan(&ret)
|
||||
if nil != err {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return 0
|
||||
}
|
||||
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func GetBlockTreeRootByPath(boxID, path string) (ret *BlockTree) {
|
||||
blockTrees.Range(func(key, value interface{}) bool {
|
||||
slice := value.(*btSlice)
|
||||
slice.m.Lock()
|
||||
for _, b := range slice.data {
|
||||
if b.BoxID == boxID && b.Path == path && b.RootID == b.ID {
|
||||
ret = b
|
||||
break
|
||||
}
|
||||
ret = &BlockTree{}
|
||||
sqlStmt := "SELECT * FROM blocktrees WHERE box_id = ? AND path = ?"
|
||||
err := db.QueryRow(sqlStmt, boxID, path).Scan(&ret.ID, &ret.RootID, &ret.ParentID, &ret.BoxID, &ret.Path, &ret.HPath, &ret.Updated, &ret.Type)
|
||||
if nil != err {
|
||||
ret = nil
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return
|
||||
}
|
||||
slice.m.Unlock()
|
||||
return nil == ret
|
||||
})
|
||||
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func GetBlockTreeRootByHPath(boxID, hPath string) (ret *BlockTree) {
|
||||
ret = &BlockTree{}
|
||||
hPath = gulu.Str.RemoveInvisible(hPath)
|
||||
blockTrees.Range(func(key, value interface{}) bool {
|
||||
slice := value.(*btSlice)
|
||||
slice.m.Lock()
|
||||
for _, b := range slice.data {
|
||||
if b.BoxID == boxID && b.HPath == hPath && b.RootID == b.ID {
|
||||
ret = b
|
||||
break
|
||||
}
|
||||
sqlStmt := "SELECT * FROM blocktrees WHERE box_id = ? AND hpath = ?"
|
||||
err := db.QueryRow(sqlStmt, boxID, hPath).Scan(&ret.ID, &ret.RootID, &ret.ParentID, &ret.BoxID, &ret.Path, &ret.HPath, &ret.Updated, &ret.Type)
|
||||
if nil != err {
|
||||
ret = nil
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return
|
||||
}
|
||||
slice.m.Unlock()
|
||||
return nil == ret
|
||||
})
|
||||
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func GetBlockTreeRootsByHPath(boxID, hPath string) (ret []*BlockTree) {
|
||||
hPath = gulu.Str.RemoveInvisible(hPath)
|
||||
blockTrees.Range(func(key, value interface{}) bool {
|
||||
slice := value.(*btSlice)
|
||||
slice.m.Lock()
|
||||
for _, b := range slice.data {
|
||||
if b.BoxID == boxID && b.HPath == hPath && b.RootID == b.ID {
|
||||
ret = append(ret, b)
|
||||
}
|
||||
sqlStmt := "SELECT * FROM blocktrees WHERE box_id = ? AND hpath = ?"
|
||||
rows, err := db.Query(sqlStmt, boxID, hPath)
|
||||
if nil != err {
|
||||
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var block BlockTree
|
||||
if err = rows.Scan(&block.ID, &block.RootID, &block.ParentID, &block.BoxID, &block.Path, &block.HPath, &block.Updated, &block.Type); nil != err {
|
||||
logging.LogErrorf("query scan field failed: %s", err)
|
||||
return
|
||||
}
|
||||
slice.m.Unlock()
|
||||
return true
|
||||
})
|
||||
ret = append(ret, &block)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func GetBlockTreeRootByHPathPreferredParentID(boxID, hPath, preferredParentID string) (ret *BlockTree) {
|
||||
hPath = gulu.Str.RemoveInvisible(hPath)
|
||||
var roots []*BlockTree
|
||||
blockTrees.Range(func(key, value interface{}) bool {
|
||||
slice := value.(*btSlice)
|
||||
slice.m.Lock()
|
||||
for _, b := range slice.data {
|
||||
if b.BoxID == boxID && b.HPath == hPath && b.RootID == b.ID {
|
||||
if "" == preferredParentID {
|
||||
ret = b
|
||||
break
|
||||
}
|
||||
|
||||
roots = append(roots, b)
|
||||
}
|
||||
sqlStmt := "SELECT * FROM blocktrees WHERE box_id = ? AND hpath = ? AND parent_id = ?"
|
||||
rows, err := db.Query(sqlStmt, boxID, hPath, preferredParentID)
|
||||
if nil != err {
|
||||
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var block BlockTree
|
||||
if err = rows.Scan(&block.ID, &block.RootID, &block.ParentID, &block.BoxID, &block.Path, &block.HPath, &block.Updated, &block.Type); nil != err {
|
||||
logging.LogErrorf("query scan field failed: %s", err)
|
||||
return
|
||||
}
|
||||
slice.m.Unlock()
|
||||
return nil == ret
|
||||
})
|
||||
if "" == preferredParentID {
|
||||
ret = &block
|
||||
return
|
||||
}
|
||||
roots = append(roots, &block)
|
||||
}
|
||||
|
||||
if 1 > len(roots) {
|
||||
return
|
||||
}
|
||||
|
|
@ -195,16 +290,17 @@ func GetBlockTreeRootByHPathPreferredParentID(boxID, hPath, preferredParentID st
|
|||
}
|
||||
|
||||
func ExistBlockTree(id string) bool {
|
||||
hash := btHash(id)
|
||||
val, ok := blockTrees.Load(hash)
|
||||
if !ok {
|
||||
sqlStmt := "SELECT COUNT(*) FROM blocktrees WHERE id = ?"
|
||||
var count int
|
||||
err := db.QueryRow(sqlStmt, id).Scan(&count)
|
||||
if nil != err {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return false
|
||||
}
|
||||
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
|
||||
return false
|
||||
}
|
||||
slice := val.(*btSlice)
|
||||
slice.m.Lock()
|
||||
_, ok = slice.data[id]
|
||||
slice.m.Unlock()
|
||||
return ok
|
||||
return 0 < count
|
||||
}
|
||||
|
||||
func GetBlockTree(id string) (ret *BlockTree) {
|
||||
|
|
@ -212,15 +308,17 @@ func GetBlockTree(id string) (ret *BlockTree) {
|
|||
return
|
||||
}
|
||||
|
||||
hash := btHash(id)
|
||||
val, ok := blockTrees.Load(hash)
|
||||
if !ok {
|
||||
ret = &BlockTree{}
|
||||
sqlStmt := "SELECT * FROM blocktrees WHERE id = ?"
|
||||
err := db.QueryRow(sqlStmt, id).Scan(&ret.ID, &ret.RootID, &ret.ParentID, &ret.BoxID, &ret.Path, &ret.HPath, &ret.Updated, &ret.Type)
|
||||
if nil != err {
|
||||
ret = nil
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return
|
||||
}
|
||||
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, logging.ShortStack())
|
||||
return
|
||||
}
|
||||
slice := val.(*btSlice)
|
||||
slice.m.Lock()
|
||||
ret = slice.data[id]
|
||||
slice.m.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -230,170 +328,169 @@ func SetBlockTreePath(tree *parse.Tree) {
|
|||
}
|
||||
|
||||
func RemoveBlockTreesByRootID(rootID string) {
|
||||
var ids []string
|
||||
blockTrees.Range(func(key, value interface{}) bool {
|
||||
slice := value.(*btSlice)
|
||||
slice.m.Lock()
|
||||
for _, b := range slice.data {
|
||||
if b.RootID == rootID {
|
||||
ids = append(ids, b.ID)
|
||||
}
|
||||
}
|
||||
slice.m.Unlock()
|
||||
return true
|
||||
})
|
||||
|
||||
ids = gulu.Str.RemoveDuplicatedElem(ids)
|
||||
for _, id := range ids {
|
||||
val, ok := blockTrees.Load(btHash(id))
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
slice := val.(*btSlice)
|
||||
slice.m.Lock()
|
||||
delete(slice.data, id)
|
||||
slice.changed = time.Now()
|
||||
slice.m.Unlock()
|
||||
sqlStmt := "DELETE FROM blocktrees WHERE root_id = ?"
|
||||
_, err := db.Exec(sqlStmt, rootID)
|
||||
if nil != err {
|
||||
logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func GetBlockTreesByPathPrefix(pathPrefix string) (ret []*BlockTree) {
|
||||
blockTrees.Range(func(key, value interface{}) bool {
|
||||
slice := value.(*btSlice)
|
||||
slice.m.Lock()
|
||||
for _, b := range slice.data {
|
||||
if strings.HasPrefix(b.Path, pathPrefix) {
|
||||
ret = append(ret, b)
|
||||
}
|
||||
sqlStmt := "SELECT * FROM blocktrees WHERE path LIKE ?"
|
||||
rows, err := db.Query(sqlStmt, pathPrefix+"%")
|
||||
if nil != err {
|
||||
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var block BlockTree
|
||||
if err = rows.Scan(&block.ID, &block.RootID, &block.ParentID, &block.BoxID, &block.Path, &block.HPath, &block.Updated, &block.Type); nil != err {
|
||||
logging.LogErrorf("query scan field failed: %s", err)
|
||||
return
|
||||
}
|
||||
slice.m.Unlock()
|
||||
return true
|
||||
})
|
||||
ret = append(ret, &block)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func GetBlockTreesByRootID(rootID string) (ret []*BlockTree) {
|
||||
blockTrees.Range(func(key, value interface{}) bool {
|
||||
slice := value.(*btSlice)
|
||||
slice.m.Lock()
|
||||
for _, b := range slice.data {
|
||||
if b.RootID == rootID {
|
||||
ret = append(ret, b)
|
||||
}
|
||||
sqlStmt := "SELECT * FROM blocktrees WHERE root_id = ?"
|
||||
rows, err := db.Query(sqlStmt, rootID)
|
||||
if nil != err {
|
||||
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var block BlockTree
|
||||
if err = rows.Scan(&block.ID, &block.RootID, &block.ParentID, &block.BoxID, &block.Path, &block.HPath, &block.Updated, &block.Type); nil != err {
|
||||
logging.LogErrorf("query scan field failed: %s", err)
|
||||
return
|
||||
}
|
||||
slice.m.Unlock()
|
||||
return true
|
||||
})
|
||||
ret = append(ret, &block)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func RemoveBlockTreesByPathPrefix(pathPrefix string) {
|
||||
var ids []string
|
||||
blockTrees.Range(func(key, value interface{}) bool {
|
||||
slice := value.(*btSlice)
|
||||
slice.m.Lock()
|
||||
for _, b := range slice.data {
|
||||
if strings.HasPrefix(b.Path, pathPrefix) {
|
||||
ids = append(ids, b.ID)
|
||||
}
|
||||
}
|
||||
slice.m.Unlock()
|
||||
return true
|
||||
})
|
||||
|
||||
ids = gulu.Str.RemoveDuplicatedElem(ids)
|
||||
for _, id := range ids {
|
||||
val, ok := blockTrees.Load(btHash(id))
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
slice := val.(*btSlice)
|
||||
slice.m.Lock()
|
||||
delete(slice.data, id)
|
||||
slice.changed = time.Now()
|
||||
slice.m.Unlock()
|
||||
sqlStmt := "DELETE FROM blocktrees WHERE path LIKE ?"
|
||||
_, err := db.Exec(sqlStmt, pathPrefix+"%")
|
||||
if nil != err {
|
||||
logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func GetBlockTreesByBoxID(boxID string) (ret []*BlockTree) {
|
||||
blockTrees.Range(func(key, value interface{}) bool {
|
||||
slice := value.(*btSlice)
|
||||
slice.m.Lock()
|
||||
for _, b := range slice.data {
|
||||
if b.BoxID == boxID {
|
||||
ret = append(ret, b)
|
||||
}
|
||||
sqlStmt := "SELECT * FROM blocktrees WHERE box_id = ?"
|
||||
rows, err := db.Query(sqlStmt, boxID)
|
||||
if nil != err {
|
||||
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var block BlockTree
|
||||
if err = rows.Scan(&block.ID, &block.RootID, &block.ParentID, &block.BoxID, &block.Path, &block.HPath, &block.Updated, &block.Type); nil != err {
|
||||
logging.LogErrorf("query scan field failed: %s", err)
|
||||
return
|
||||
}
|
||||
slice.m.Unlock()
|
||||
return true
|
||||
})
|
||||
ret = append(ret, &block)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func RemoveBlockTreesByBoxID(boxID string) (ids []string) {
|
||||
blockTrees.Range(func(key, value interface{}) bool {
|
||||
slice := value.(*btSlice)
|
||||
slice.m.Lock()
|
||||
for _, b := range slice.data {
|
||||
if b.BoxID == boxID {
|
||||
ids = append(ids, b.ID)
|
||||
}
|
||||
sqlStmt := "SELECT id FROM blocktrees WHERE box_id = ?"
|
||||
rows, err := db.Query(sqlStmt, boxID)
|
||||
if nil != err {
|
||||
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err = rows.Scan(&id); nil != err {
|
||||
logging.LogErrorf("query scan field failed: %s", err)
|
||||
return
|
||||
}
|
||||
slice.m.Unlock()
|
||||
return true
|
||||
})
|
||||
ids = append(ids, id)
|
||||
}
|
||||
|
||||
ids = gulu.Str.RemoveDuplicatedElem(ids)
|
||||
for _, id := range ids {
|
||||
val, ok := blockTrees.Load(btHash(id))
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
slice := val.(*btSlice)
|
||||
slice.m.Lock()
|
||||
delete(slice.data, id)
|
||||
slice.changed = time.Now()
|
||||
slice.m.Unlock()
|
||||
sqlStmt = "DELETE FROM blocktrees WHERE box_id = ?"
|
||||
_, err = db.Exec(sqlStmt, boxID)
|
||||
if nil != err {
|
||||
logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err)
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func RemoveBlockTree(id string) {
|
||||
val, ok := blockTrees.Load(btHash(id))
|
||||
if !ok {
|
||||
sqlStmt := "DELETE FROM blocktrees WHERE id = ?"
|
||||
_, err := db.Exec(sqlStmt, id)
|
||||
if nil != err {
|
||||
logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err)
|
||||
return
|
||||
}
|
||||
slice := val.(*btSlice)
|
||||
slice.m.Lock()
|
||||
delete(slice.data, id)
|
||||
slice.changed = time.Now()
|
||||
slice.m.Unlock()
|
||||
}
|
||||
|
||||
var indexBlockTreeLock = sync.Mutex{}
|
||||
|
||||
func IndexBlockTree(tree *parse.Tree) {
|
||||
var changedNodes []*ast.Node
|
||||
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
|
||||
if !entering || !n.IsBlock() {
|
||||
return ast.WalkContinue
|
||||
}
|
||||
if "" == n.ID {
|
||||
if !entering || !n.IsBlock() || "" == n.ID {
|
||||
return ast.WalkContinue
|
||||
}
|
||||
|
||||
hash := btHash(n.ID)
|
||||
val, ok := blockTrees.Load(hash)
|
||||
if !ok {
|
||||
val = &btSlice{data: map[string]*BlockTree{}, changed: time.Time{}, m: &sync.Mutex{}}
|
||||
blockTrees.Store(hash, val)
|
||||
changedNodes = append(changedNodes, n)
|
||||
return ast.WalkContinue
|
||||
})
|
||||
|
||||
indexBlockTreeLock.Lock()
|
||||
defer indexBlockTreeLock.Unlock()
|
||||
|
||||
tx, err := db.Begin()
|
||||
if nil != err {
|
||||
logging.LogErrorf("begin transaction failed: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
sqlStmt := "INSERT INTO blocktrees (id, root_id, parent_id, box_id, path, hpath, updated, type) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
|
||||
for _, n := range changedNodes {
|
||||
var parentID string
|
||||
if nil != n.Parent {
|
||||
parentID = n.Parent.ID
|
||||
}
|
||||
slice := val.(*btSlice)
|
||||
if _, err = tx.Exec(sqlStmt, n.ID, tree.ID, parentID, tree.Box, tree.Path, tree.HPath, n.IALAttr("updated"), TypeAbbr(n.Type.String())); nil != err {
|
||||
tx.Rollback()
|
||||
logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if err = tx.Commit(); nil != err {
|
||||
logging.LogErrorf("commit transaction failed: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
slice.m.Lock()
|
||||
bt := slice.data[n.ID]
|
||||
slice.m.Unlock()
|
||||
func UpsertBlockTree(tree *parse.Tree) {
|
||||
oldBts := map[string]*BlockTree{}
|
||||
bts := GetBlockTreesByRootID(tree.ID)
|
||||
for _, bt := range bts {
|
||||
oldBts[bt.ID] = bt
|
||||
}
|
||||
|
||||
if nil != bt {
|
||||
if bt.Updated != n.IALAttr("updated") || bt.Type != TypeAbbr(n.Type.String()) || bt.Path != tree.Path || bt.BoxID != tree.Box || bt.HPath != tree.HPath {
|
||||
var changedNodes []*ast.Node
|
||||
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
|
||||
if !entering || !n.IsBlock() || "" == n.ID {
|
||||
return ast.WalkContinue
|
||||
}
|
||||
|
||||
if oldBt, found := oldBts[n.ID]; found {
|
||||
if oldBt.Updated != n.IALAttr("updated") || oldBt.Type != TypeAbbr(n.Type.String()) || oldBt.Path != tree.Path || oldBt.BoxID != tree.Box || oldBt.HPath != tree.HPath {
|
||||
children := ChildBlockNodes(n) // 需要考虑子块，因为一些操作（比如移动块）后需要同时更新子块
|
||||
changedNodes = append(changedNodes, children...)
|
||||
}
|
||||
|
|
@ -404,179 +501,60 @@ func IndexBlockTree(tree *parse.Tree) {
|
|||
return ast.WalkContinue
|
||||
})
|
||||
|
||||
ids := bytes.Buffer{}
|
||||
for i, n := range changedNodes {
|
||||
ids.WriteString("'")
|
||||
ids.WriteString(n.ID)
|
||||
ids.WriteString("'")
|
||||
if i < len(changedNodes)-1 {
|
||||
ids.WriteString(",")
|
||||
}
|
||||
}
|
||||
|
||||
indexBlockTreeLock.Lock()
|
||||
defer indexBlockTreeLock.Unlock()
|
||||
|
||||
tx, err := db.Begin()
|
||||
if nil != err {
|
||||
logging.LogErrorf("begin transaction failed: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
sqlStmt := "DELETE FROM blocktrees WHERE id IN (" + ids.String() + ")"
|
||||
|
||||
_, err = tx.Exec(sqlStmt)
|
||||
if nil != err {
|
||||
tx.Rollback()
|
||||
logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err)
|
||||
return
|
||||
}
|
||||
sqlStmt = "INSERT INTO blocktrees (id, root_id, parent_id, box_id, path, hpath, updated, type) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
|
||||
for _, n := range changedNodes {
|
||||
updateBtSlice(n, tree)
|
||||
var parentID string
|
||||
if nil != n.Parent {
|
||||
parentID = n.Parent.ID
|
||||
}
|
||||
if _, err = tx.Exec(sqlStmt, n.ID, tree.ID, parentID, tree.Box, tree.Path, tree.HPath, n.IALAttr("updated"), TypeAbbr(n.Type.String())); nil != err {
|
||||
tx.Rollback()
|
||||
logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if err = tx.Commit(); nil != err {
|
||||
logging.LogErrorf("commit transaction failed: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
func updateBtSlice(n *ast.Node, tree *parse.Tree) {
|
||||
var parentID string
|
||||
if nil != n.Parent {
|
||||
parentID = n.Parent.ID
|
||||
}
|
||||
|
||||
hash := btHash(n.ID)
|
||||
val, ok := blockTrees.Load(hash)
|
||||
if !ok {
|
||||
val = &btSlice{data: map[string]*BlockTree{}, changed: time.Time{}, m: &sync.Mutex{}}
|
||||
blockTrees.Store(hash, val)
|
||||
}
|
||||
slice := val.(*btSlice)
|
||||
slice.m.Lock()
|
||||
slice.data[n.ID] = &BlockTree{ID: n.ID, ParentID: parentID, RootID: tree.ID, BoxID: tree.Box, Path: tree.Path, HPath: tree.HPath, Updated: n.IALAttr("updated"), Type: TypeAbbr(n.Type.String())}
|
||||
slice.changed = time.Now()
|
||||
slice.m.Unlock()
|
||||
}
|
||||
|
||||
var blockTreeLock = sync.Mutex{}
|
||||
|
||||
func InitBlockTree(force bool) {
|
||||
blockTreeLock.Lock()
|
||||
defer blockTreeLock.Unlock()
|
||||
|
||||
start := time.Now()
|
||||
if force {
|
||||
err := os.RemoveAll(util.BlockTreePath)
|
||||
if nil != err {
|
||||
logging.LogErrorf("remove block tree file failed: %s", err)
|
||||
}
|
||||
blockTrees = &sync.Map{}
|
||||
return
|
||||
}
|
||||
|
||||
entries, err := os.ReadDir(util.BlockTreePath)
|
||||
err := initDatabase(force)
|
||||
if nil != err {
|
||||
logging.LogErrorf("read block tree dir failed: %s", err)
|
||||
os.Exit(logging.ExitCodeFileSysErr)
|
||||
logging.LogErrorf("init database failed: %s", err)
|
||||
os.Exit(logging.ExitCodeReadOnlyDatabase)
|
||||
return
|
||||
}
|
||||
|
||||
loadErr := atomic.Bool{}
|
||||
size := atomic.Int64{}
|
||||
waitGroup := &sync.WaitGroup{}
|
||||
p, _ := ants.NewPoolWithFunc(4, func(arg interface{}) {
|
||||
defer waitGroup.Done()
|
||||
|
||||
entry := arg.(os.DirEntry)
|
||||
p := filepath.Join(util.BlockTreePath, entry.Name())
|
||||
|
||||
f, err := os.OpenFile(p, os.O_RDONLY, 0644)
|
||||
if nil != err {
|
||||
logging.LogErrorf("open block tree failed: %s", err)
|
||||
loadErr.Store(true)
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
info, err := f.Stat()
|
||||
if nil != err {
|
||||
logging.LogErrorf("stat block tree failed: %s", err)
|
||||
loadErr.Store(true)
|
||||
return
|
||||
}
|
||||
size.Add(info.Size())
|
||||
|
||||
sliceData := map[string]*BlockTree{}
|
||||
if err = msgpack.NewDecoder(f).Decode(&sliceData); nil != err {
|
||||
logging.LogErrorf("unmarshal block tree failed: %s", err)
|
||||
loadErr.Store(true)
|
||||
return
|
||||
}
|
||||
|
||||
name := entry.Name()[0:strings.Index(entry.Name(), ".")]
|
||||
blockTrees.Store(name, &btSlice{data: sliceData, changed: time.Time{}, m: &sync.Mutex{}})
|
||||
})
|
||||
for _, entry := range entries {
|
||||
if !strings.HasSuffix(entry.Name(), ".msgpack") {
|
||||
continue
|
||||
}
|
||||
|
||||
waitGroup.Add(1)
|
||||
p.Invoke(entry)
|
||||
}
|
||||
|
||||
waitGroup.Wait()
|
||||
p.Release()
|
||||
|
||||
if loadErr.Load() {
|
||||
logging.LogInfof("cause block tree load error, remove block tree file")
|
||||
if removeErr := os.RemoveAll(util.BlockTreePath); nil != removeErr {
|
||||
logging.LogErrorf("remove block tree file failed: %s", removeErr)
|
||||
os.Exit(logging.ExitCodeFileSysErr)
|
||||
return
|
||||
}
|
||||
blockTrees = &sync.Map{}
|
||||
return
|
||||
}
|
||||
|
||||
elapsed := time.Since(start).Seconds()
|
||||
logging.LogInfof("read block tree [%s] to [%s], elapsed [%.2fs]", humanize.BytesCustomCeil(uint64(size.Load()), 2), util.BlockTreePath, elapsed)
|
||||
return
|
||||
}
|
||||
|
||||
func SaveBlockTreeJob() {
|
||||
SaveBlockTree(false)
|
||||
}
|
||||
|
||||
func SaveBlockTree(force bool) {
|
||||
blockTreeLock.Lock()
|
||||
defer blockTreeLock.Unlock()
|
||||
|
||||
if task.ContainIndexTask() {
|
||||
//logging.LogInfof("skip saving block tree because indexing")
|
||||
return
|
||||
}
|
||||
//logging.LogInfof("saving block tree")
|
||||
|
||||
start := time.Now()
|
||||
if err := os.MkdirAll(util.BlockTreePath, 0755); nil != err {
|
||||
logging.LogErrorf("create block tree dir [%s] failed: %s", util.BlockTreePath, err)
|
||||
os.Exit(logging.ExitCodeFileSysErr)
|
||||
return
|
||||
}
|
||||
|
||||
size := uint64(0)
|
||||
var count int
|
||||
blockTrees.Range(func(key, value interface{}) bool {
|
||||
slice := value.(*btSlice)
|
||||
slice.m.Lock()
|
||||
if !force && slice.changed.IsZero() {
|
||||
slice.m.Unlock()
|
||||
return true
|
||||
}
|
||||
|
||||
data, err := msgpack.Marshal(slice.data)
|
||||
if nil != err {
|
||||
logging.LogErrorf("marshal block tree failed: %s", err)
|
||||
os.Exit(logging.ExitCodeFileSysErr)
|
||||
return false
|
||||
}
|
||||
slice.m.Unlock()
|
||||
|
||||
p := filepath.Join(util.BlockTreePath, key.(string)) + ".msgpack"
|
||||
if err = gulu.File.WriteFileSafer(p, data, 0644); nil != err {
|
||||
logging.LogErrorf("write block tree failed: %s", err)
|
||||
os.Exit(logging.ExitCodeFileSysErr)
|
||||
return false
|
||||
}
|
||||
|
||||
slice.m.Lock()
|
||||
slice.changed = time.Time{}
|
||||
slice.m.Unlock()
|
||||
size += uint64(len(data))
|
||||
count++
|
||||
return true
|
||||
})
|
||||
if 0 < count {
|
||||
//logging.LogInfof("wrote block trees [%d]", count)
|
||||
}
|
||||
|
||||
elapsed := time.Since(start).Seconds()
|
||||
if 2 < elapsed {
|
||||
logging.LogWarnf("save block tree [size=%s] to [%s], elapsed [%.2fs]", humanize.BytesCustomCeil(size, 2), util.BlockTreePath, elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func CeilTreeCount(count int) int {
|
||||
if 100 > count {
|
||||
return 100
|
||||
|
|
@ -602,7 +580,3 @@ func CeilBlockCount(count int) int {
|
|||
}
|
||||
return 10000*100 + 1
|
||||
}
|
||||
|
||||
func btHash(id string) string {
|
||||
return util2.Hash([]byte(id))[0:2]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ package treenode
|
|||
|
||||
import (
|
||||
"github.com/88250/gulu"
|
||||
"time"
|
||||
"github.com/siyuan-note/logging"
|
||||
)
|
||||
|
||||
func ClearRedundantBlockTrees(boxID string, paths []string) {
|
||||
|
|
@ -35,17 +35,21 @@ func getRedundantPaths(boxID string, paths []string) (ret []string) {
|
|||
}
|
||||
|
||||
btPathsMap := map[string]bool{}
|
||||
blockTrees.Range(func(key, value interface{}) bool {
|
||||
slice := value.(*btSlice)
|
||||
slice.m.Lock()
|
||||
for _, b := range slice.data {
|
||||
if b.BoxID == boxID {
|
||||
btPathsMap[b.Path] = true
|
||||
}
|
||||
sqlStmt := "SELECT path FROM blocktrees WHERE box_id = ?"
|
||||
rows, err := db.Query(sqlStmt, boxID)
|
||||
if nil != err {
|
||||
logging.LogErrorf("query block tree failed: %s", err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var path string
|
||||
if err = rows.Scan(&path); nil != err {
|
||||
logging.LogErrorf("scan block tree failed: %s", err)
|
||||
return
|
||||
}
|
||||
slice.m.Unlock()
|
||||
return true
|
||||
})
|
||||
btPathsMap[path] = true
|
||||
}
|
||||
|
||||
for p, _ := range btPathsMap {
|
||||
if !pathsMap[p] {
|
||||
|
|
@ -57,18 +61,11 @@ func getRedundantPaths(boxID string, paths []string) (ret []string) {
|
|||
}
|
||||
|
||||
func removeBlockTreesByPath(boxID, path string) {
|
||||
blockTrees.Range(func(key, value interface{}) bool {
|
||||
slice := value.(*btSlice)
|
||||
slice.m.Lock()
|
||||
for _, b := range slice.data {
|
||||
if b.Path == path && b.BoxID == boxID {
|
||||
delete(slice.data, b.ID)
|
||||
slice.changed = time.Now()
|
||||
}
|
||||
}
|
||||
slice.m.Unlock()
|
||||
return true
|
||||
})
|
||||
sqlStmt := "DELETE FROM blocktrees WHERE box_id = ? AND path = ?"
|
||||
_, err := db.Exec(sqlStmt, boxID, path)
|
||||
if nil != err {
|
||||
logging.LogErrorf("delete block tree failed: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
func GetNotExistPaths(boxID string, paths []string) (ret []string) {
|
||||
|
|
@ -78,17 +75,21 @@ func GetNotExistPaths(boxID string, paths []string) (ret []string) {
|
|||
}
|
||||
|
||||
btPathsMap := map[string]bool{}
|
||||
blockTrees.Range(func(key, value interface{}) bool {
|
||||
slice := value.(*btSlice)
|
||||
slice.m.Lock()
|
||||
for _, b := range slice.data {
|
||||
if b.BoxID == boxID {
|
||||
btPathsMap[b.Path] = true
|
||||
}
|
||||
sqlStmt := "SELECT path FROM blocktrees WHERE box_id = ?"
|
||||
rows, err := db.Query(sqlStmt, boxID)
|
||||
if nil != err {
|
||||
logging.LogErrorf("query block tree failed: %s", err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var path string
|
||||
if err = rows.Scan(&path); nil != err {
|
||||
logging.LogErrorf("scan block tree failed: %s", err)
|
||||
return
|
||||
}
|
||||
slice.m.Unlock()
|
||||
return true
|
||||
})
|
||||
btPathsMap[path] = true
|
||||
}
|
||||
|
||||
for p, _ := range pathsMap {
|
||||
if !btPathsMap[p] {
|
||||
|
|
@ -101,16 +102,20 @@ func GetNotExistPaths(boxID string, paths []string) (ret []string) {
|
|||
|
||||
func GetRootUpdated() (ret map[string]string) {
|
||||
ret = map[string]string{}
|
||||
blockTrees.Range(func(key, value interface{}) bool {
|
||||
slice := value.(*btSlice)
|
||||
slice.m.Lock()
|
||||
for _, b := range slice.data {
|
||||
if b.RootID == b.ID {
|
||||
ret[b.RootID] = b.Updated
|
||||
}
|
||||
sqlStmt := "SELECT root_id, updated FROM blocktrees WHERE root_id = id AND type = 'd'"
|
||||
rows, err := db.Query(sqlStmt)
|
||||
if nil != err {
|
||||
logging.LogErrorf("query block tree failed: %s", err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var rootID, updated string
|
||||
if err = rows.Scan(&rootID, &updated); nil != err {
|
||||
logging.LogErrorf("scan block tree failed: %s", err)
|
||||
return
|
||||
}
|
||||
slice.m.Unlock()
|
||||
return true
|
||||
})
|
||||
ret[rootID] = updated
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
|
|||
|
|
@ -209,7 +209,7 @@ var (
|
|||
DBPath string // SQLite 数据库文件路径
|
||||
HistoryDBPath string // SQLite 历史数据库文件路径
|
||||
AssetContentDBPath string // SQLite 资源文件内容数据库文件路径
|
||||
BlockTreePath string // 区块树文件路径
|
||||
BlockTreeDBPath string // 区块树数据库文件路径
|
||||
AppearancePath string // 配置目录下的外观目录 appearance/ 路径
|
||||
ThemesPath string // 配置目录下的外观目录下的 themes/ 路径
|
||||
IconsPath string // 配置目录下的外观目录下的 icons/ 路径
|
||||
|
|
@ -287,7 +287,7 @@ func initWorkspaceDir(workspaceArg string) {
|
|||
DBPath = filepath.Join(TempDir, DBName)
|
||||
HistoryDBPath = filepath.Join(TempDir, "history.db")
|
||||
AssetContentDBPath = filepath.Join(TempDir, "asset_content.db")
|
||||
BlockTreePath = filepath.Join(TempDir, "blocktree")
|
||||
BlockTreeDBPath = filepath.Join(TempDir, "blocktree.db")
|
||||
SnippetsPath = filepath.Join(DataDir, "snippets")
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -159,7 +159,7 @@ func initWorkspaceDirMobile(workspaceBaseDir string) {
|
|||
DBPath = filepath.Join(TempDir, DBName)
|
||||
HistoryDBPath = filepath.Join(TempDir, "history.db")
|
||||
AssetContentDBPath = filepath.Join(TempDir, "asset_content.db")
|
||||
BlockTreePath = filepath.Join(TempDir, "blocktree")
|
||||
BlockTreeDBPath = filepath.Join(TempDir, "blocktree.db")
|
||||
SnippetsPath = filepath.Join(DataDir, "snippets")
|
||||
|
||||
AppearancePath = filepath.Join(ConfDir, "appearance")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue