0ct0pu5/siyuan
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/dev' into dev

Browse source
This commit is contained in:
Vanessa 2023-01-24 11:18:35 +08:00
commit a8bab8ce62
20 changed files with 180 additions and 367 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
app/appearance/langs/en_US.json
View file

@ -928,10 +928,10 @@
"50": "Resolving reference [%s]",
"51": "Cache content block [%d]",
"52": "Every time you open the user guide notebook data will be reset, so please do not save any data in it",
"53": "The index of [%d] documents has been completed, and [%d] remain to be processed",
"53": "TODO",
"54": "Indexing references...",
"55": "Indexed references of [%d] documents",
"56": "Indexed [%d] documents",
"56": "TODO",
"57": "Failed to create temp key",
"58": "After the index is rebuilt, the interface will be automatically refreshed later...",
"59": "Failed to set sync ignore list",
@ -968,7 +968,7 @@
"90": "Created [%d] of search indexes of block-level elements [%s]",
"91": "Reading block tree data...",
"92": "Parsing document tree [%s]",
"93": "Indexing document tree [%s]",
"93": "TODO",
"94": "Upload failed: %s",
"95": "Exiting...",
"96": "Synchronization failed when exiting. Please manually perform a synchronization to ensure that the local data is consistent with the cloud data",
@ -983,7 +983,7 @@
"105": "Corrupted data repo have been automatically reset",
"106": "Maximum length is limited to 512 characters",
"107": "Moving document [%s]",
"108": "Cleaning obsolete indexes...",
"108": "TODO",
"109": "Remove reminder completed [%s]",
"110": "Renaming...",
"111": "Saving document [%s]...",

8
app/appearance/langs/es_ES.json
View file

@ -928,10 +928,10 @@
"50": "Resolviendo referencia [%s]",
"51": "Bloque de contenido de la caché [%d]",
"52": "Cada vez que abra la guía del usuario se restablecerán los datos del cuaderno, así que no guarde ningún dato en ella",
"53": "Se ha completado el índice de [%d] documentos y quedan [%d] por procesar",
"53": "TODO",
"54": "Indexando referencias...",
"55": "Referencias indexadas de [%d] documentos",
"56": "Documentos indexados [%d]",
"56": "TODO",
"57": "Fallo en la creación de la clave temporal",
"58": "Después de reconstruir el índice, la interfaz se actualizará automáticamente más tarde...",
"59": " Falló la configuración de sincronización de la lista de ignorados",
@ -968,7 +968,7 @@
"90": "Creado [%d] de índices de búsqueda de elementos a nivel de bloque [%s]",
"91": "Leyendo datos del árbol de bloques...",
"92": "Analizando el árbol del documento [%s]",
"93": "Árbol de documentos de indexación [%s]",
"93": "TODO",
"94": "Carga fallida: %s",
"95": "Saliendo...",
"96": "La sincronización falló al salir. Por favor, realice manualmente una sincronización para asegurarse de que los datos locales son coherentes con los datos de la nube",
@ -983,7 +983,7 @@
"105": "El repositorio de datos corruptos se ha restablecido automáticamente",
"106": "La longitud máxima está limitada a 512 caracteres",
"107": "Moviendo documento [%s]",
"108": "Limpiando índices obsoletos...",
"108": "TODO",
"109": "Eliminación de recordatorios completada [%s]",
"110": "Renombrar...",
"111": "Guardando documento [%s]...",

8
app/appearance/langs/fr_FR.json
View file

@ -928,10 +928,10 @@
"50": "Résolution des référence [%s]",
"51": "Blocage du contenu du cache [%d]",
"52": "Chaque fois que vous ouvrez le guide de l'utilisateur, les données du notebook seront réinitialisées, veuillez donc ne pas y enregistrer de données.",
"53": "L'indexation de [%d] documents est terminée, et [%d] restent à traiter.",
"53": "TODO",
"54": "Indexation des références...",
"55": "Références indexées de [%d] documents",
"56": "Indexé [%d] documents",
"56": "TODO",
"57": "Échec de la création d'une clé temporaire",
"58": "Une fois l'index reconstruit, l'interface sera automatiquement rafraîchie ultérieurement...",
"59": "Échec de la définition de la liste des ignores de synchronisation",
@ -968,7 +968,7 @@
"90": "Création de [%d] index de recherche d'éléments de niveau bloc [%s]",
"91": "Lecture des données de l'arborescence des blocs...",
"92": "Analyse de l'arborescence du document [%s]",
"93": "Indexation de l'arborescence du document [%s]",
"93": "TODO",
"94": "Échec du téléchargement : %s",
"95": "Quitter le programme...",
"96": "La synchronisation a échoué lors de la sortie. Veuillez effectuer une synchronisation manuellement pour vous assurer que les données locales sont cohérentes avec les données du cloud",
@ -983,7 +983,7 @@
"105": "Le référentiel de données corrompu a été automatiquement réinitialisé",
"106": "La longueur maximale est limitée à 512 caractères",
"107": "Déplacement du document [%s]",
"108": "Nettoyage des index obsolètes...",
"108": "TODO",
"109": "Supprimer le rappel terminé [%s]",
"110": "Renommer...",
"111": "Enregistrement du document [%s]...",

8
app/appearance/langs/zh_CHT.json
View file

@ -928,10 +928,10 @@
"50": "正在解析引用 [%s]",
"51": "正在快取內容塊 [%d]",
"52": "每次打開使用者指南筆記本資料都會被重置，所以請勿在其中保存任何資料",
"53": "已完成索引 [%d] 篇文檔，剩餘待處理 [%d]",
"53": "TODO",
"54": "正在索引引用關係...",
"55": "已完成索引 [%d] 篇文檔的引用關係",
"56": "已完成索引 [%d] 篇文檔",
"56": "TODO",
"57": "創建臨時金鑰失敗",
"58": "重建索引完畢，稍後將自動重新整理介面...",
"59": "設置同步忽略列表失敗",
@ -968,7 +968,7 @@
"90": "已經建立 [%d] 個塊級元素的搜索索引 [%s]",
"91": "正在讀取塊樹數據...",
"92": "正在解析文檔樹 [%s]",
"93": "正在索引文檔樹 [%s]",
"93": "TODO",
"94": "上傳失敗：%s",
"95": "正在退出...",
"96": "退出時同步失敗，請手動執行一次同步以確保本地資料和雲端資料一致",
@ -983,7 +983,7 @@
"105": "已經自動重置損壞的數據倉庫",
"106": "最大長度限制為 512 字元",
"107": "正在移動文檔 [%s]",
"108": "正在清理已過時的索引...",
"108": "TODO",
"109": "移除提醒完畢 [%s]",
"110": "正在重命名...",
"111": "正在保存文檔 [%s]...",

8
app/appearance/langs/zh_CN.json
View file

@ -928,10 +928,10 @@
"50": "正在解析引用 [%s]",
"51": "正在缓存内容块 [%d]",
"52": "每次打开用户指南笔记本数据都会被重置，所以请勿在其中保存任何数据",
"53": "已完成索引 [%d] 篇文档，剩余待处理 [%d]",
"53": "TODO",
"54": "正在索引引用关系...",
"55": "已完成索引 [%d] 篇文档的引用关系",
"56": "已完成索引 [%d] 篇文档",
"56": "TODO",
"57": "创建临时密钥失败",
"58": "重建索引完毕，稍后将自动刷新界面...",
"59": "设置同步忽略列表失败",
@ -968,7 +968,7 @@
"90": "已经建立 [%d] 个块级元素的搜索索引 [%s]",
"91": "正在读取块树数据...",
"92": "正在解析文档树 [%s]",
"93": "正在索引文档树 [%s]",
"93": "TODO",
"94": "上传失败：%s",
"95": "正在退出...",
"96": "退出时同步失败，请手动执行一次同步以确保本地数据和云端数据一致",
@ -983,7 +983,7 @@
"105": "已经自动重置损坏的数据仓库",
"106": "最大长度限制为 512 字符",
"107": "正在移动文档 [%s]",
"108": "正在清理已过时的索引...",
"108": "TODO",
"109": "移除提醒完毕 [%s]",
"110": "正在重命名...",
"111": "正在保存文档 [%s]...",

13
kernel/api/transaction.go
View file

@ -25,9 +25,7 @@ import (
"github.com/88250/gulu"
"github.com/gin-gonic/gin"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/model"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/util"
)
@ -67,17 +65,6 @@ func performTransactions(c *gin.Context) {
ret.Code = 1
return
}
if nil != err {
tx, txErr := sql.BeginTx()
if nil != txErr {
logging.LogFatalf("transaction failed: %s", txErr)
return
}
sql.ClearBoxHash(tx)
sql.CommitTx(tx)
logging.LogFatalf("transaction failed: %s", err)
return
}
ret.Data = transactions

15
kernel/model/backlink.go
View file

@ -40,24 +40,19 @@ import (
func RefreshBacklink(id string) {
WaitForWritingFiles()
tx, err := sql.BeginTx()
if nil != err {
return
}
defer sql.CommitTx(tx)
refs := sql.QueryRefsByDefID(id, false)
trees := map[string]*parse.Tree{}
for _, ref := range refs {
tree := trees[ref.RootID]
if nil == tree {
tree, err = loadTreeByBlockID(ref.RootID)
if nil != err {
logging.LogErrorf("refresh tree refs failed: %s", err)
var loadErr error
tree, loadErr = loadTreeByBlockID(ref.RootID)
if nil != loadErr {
logging.LogErrorf("refresh tree refs failed: %s", loadErr)
continue
}
trees[ref.RootID] = tree
sql.UpsertRefs(tx, tree)
sql.UpdateRefsTreeQueue(tree)
}
}
}

23
kernel/model/box.go
View file

@ -339,22 +339,6 @@ func (box *Box) Remove(path string) error {
return nil
}
func (box *Box) Unindex() {
task.PrependTask(task.DatabaseIndex, unindex, box.ID)
}
func unindex(boxID string) {
tx, err := sql.BeginTx()
if nil != err {
return
}
sql.RemoveBoxHash(tx, boxID)
sql.DeleteByBoxTx(tx, boxID)
sql.CommitTx(tx)
ids := treenode.RemoveBlockTreesByBoxID(boxID)
RemoveRecentDoc(ids)
}
func (box *Box) ListFiles(path string) (ret []*FileInfo) {
fis, _, err := box.Ls(path)
if nil != err {
@ -498,6 +482,7 @@ func genTreeID(tree *parse.Tree) {
func FullReindex() {
task.PrependTask(task.DatabaseIndexFull, fullReindex)
task.AppendTask(task.DatabaseIndexRef, IndexRefs)
}
func fullReindex() {
@ -510,11 +495,13 @@ func fullReindex() {
}
treenode.InitBlockTree(true)
sql.IndexMode()
defer sql.NormalMode()
openedBoxes := Conf.GetOpenedBoxes()
for _, openedBox := range openedBoxes {
openedBox.Index(true)
index(openedBox.ID)
}
IndexRefs()
treenode.SaveBlockTree(true)
LoadFlashcards()

2
kernel/model/conf.go
View file

@ -599,7 +599,7 @@ func InitBoxes() {
box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间
if !initialized {
box.Index(true)
index(box.ID)
}
}

15
kernel/model/file.go
View file

@ -912,15 +912,7 @@ func createTreeTx(tree *parse.Tree) {
transaction := &Transaction{DoOperations: []*Operation{{Action: "create", Data: tree}}}
err := PerformTransactions(&[]*Transaction{transaction})
if nil != err {
tx, txErr := sql.BeginTx()
if nil != txErr {
logging.LogFatalf("transaction failed: %s", txErr)
return
}
sql.ClearBoxHash(tx)
sql.CommitTx(tx)
logging.LogFatalf("transaction failed: %s", err)
return
}
}
@ -1467,13 +1459,6 @@ func createDoc(boxID, p, title, dom string) (err error) {
transaction := &Transaction{DoOperations: []*Operation{{Action: "create", Data: tree}}}
err = PerformTransactions(&[]*Transaction{transaction})
if nil != err {
tx, txErr := sql.BeginTx()
if nil != txErr {
logging.LogFatalf("transaction failed: %s", txErr)
return
}
sql.ClearBoxHash(tx)
sql.CommitTx(tx)
logging.LogFatalf("transaction failed: %s", err)
return
}

2
kernel/model/import.go
View file

@ -393,7 +393,7 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
continue
}
treenode.IndexBlockTree(tree)
treenode.ReindexBlockTree(tree)
sql.UpsertTreeQueue(tree)
}

188
kernel/model/index.go
View file

@ -17,11 +17,7 @@
package model
import (
"bytes"
"crypto/sha256"
"fmt"
"runtime/debug"
"sort"
"strings"
"time"
@ -33,20 +29,31 @@ import (
"github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
)
func (box *Box) Index(fullRebuildIndex bool) (treeCount int, treeSize int64) {
defer debug.FreeOSMemory()
func (box *Box) Unindex() {
task.PrependTask(task.DatabaseIndex, unindex, box.ID)
}
sql.IndexMode()
defer sql.NormalMode()
func unindex(boxID string) {
ids := treenode.RemoveBlockTreesByBoxID(boxID)
RemoveRecentDoc(ids)
sql.DeleteBoxQueue(boxID)
}
//os.MkdirAll("pprof", 0755)
//cpuProfile, _ := os.Create("pprof/cpu_profile_index")
//pprof.StartCPUProfile(cpuProfile)
//defer pprof.StopCPUProfile()
func (box *Box) Index() {
task.PrependTask(task.DatabaseIndex, index, box.ID)
task.AppendTask(task.DatabaseIndexRef, IndexRefs)
}
func index(boxID string) {
box := Conf.Box(boxID)
if nil == box {
return
}
util.SetBootDetails("Listing files...")
files := box.ListFiles("/")
@ -54,104 +61,17 @@ func (box *Box) Index(fullRebuildIndex bool) (treeCount int, treeSize int64) {
if 1 > boxLen {
boxLen = 1
}
bootProgressPart := 10.0 / float64(boxLen) / float64(len(files))
luteEngine := NewLute()
idTitleMap := map[string]string{}
idHashMap := map[string]string{}
util.PushEndlessProgress(fmt.Sprintf("["+box.Name+"] "+Conf.Language(64), len(files)))
i := 0
// 读取并缓存路径映射
for _, file := range files {
if file.isdir || !strings.HasSuffix(file.name, ".sy") {
continue
}
p := file.path
tree, err := filesys.LoadTree(box.ID, p, luteEngine)
if nil != err {
logging.LogErrorf("read box [%s] tree [%s] failed: %s", box.ID, p, err)
continue
}
docIAL := parse.IAL2MapUnEsc(tree.Root.KramdownIAL)
if "" == docIAL["updated"] {
updated := util.TimeFromID(tree.Root.ID)
tree.Root.SetIALAttr("updated", updated)
docIAL["updated"] = updated
writeJSONQueue(tree)
}
cache.PutDocIAL(p, docIAL)
util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(92), util.ShortPathForBootingDisplay(tree.Path)))
treeSize += file.size
treeCount++
// 缓存文档标题，后面做 Path -> HPath 路径映射时需要
idTitleMap[tree.ID] = tree.Root.IALAttr("title")
// 缓存块树
treenode.IndexBlockTree(tree)
// 缓存 ID-Hash，后面需要用于判断是否要重建库
idHashMap[tree.ID] = tree.Hash
if 1 < i && 0 == i%64 {
util.PushEndlessProgress(fmt.Sprintf(Conf.Language(88), i, len(files)-i))
}
i++
}
box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间
// 检查是否需要重新建库
util.SetBootDetails("Checking data hashes...")
var ids []string
for id := range idTitleMap {
ids = append(ids, id)
}
sort.Slice(ids, func(i, j int) bool { return ids[i] >= ids[j] })
buf := bytes.Buffer{}
for _, id := range ids {
hash, _ := idHashMap[id]
buf.WriteString(hash)
util.SetBootDetails("Checking hash " + hash)
}
boxHash := fmt.Sprintf("%x", sha256.Sum256(buf.Bytes()))
dbBoxHash := sql.GetBoxHash(box.ID)
if boxHash == dbBoxHash {
//logging.LogInfof("use existing database for box [%s]", box.ID)
util.SetBootDetails("Use existing database for notebook " + box.ID)
return
}
// 开始重建库
sql.DisableCache()
defer sql.EnableCache()
bootProgressPart := 30.0 / float64(boxLen) / float64(len(files))
start := time.Now()
if !fullRebuildIndex {
tx, err := sql.BeginTx()
if nil != err {
return
}
sql.PutBoxHash(tx, box.ID, boxHash)
util.SetBootDetails("Cleaning obsolete indexes...")
util.PushEndlessProgress(Conf.Language(108))
sql.DeleteByBoxTx(tx, box.ID)
if err = sql.CommitTx(tx); nil != err {
return
}
}
luteEngine := NewLute()
var treeCount int
var treeSize int64
i := 0
bootProgressPart = 20.0 / float64(boxLen) / float64(treeCount)
util.PushEndlessProgress(fmt.Sprintf("["+box.Name+"] "+Conf.Language(64), len(files)))
defer util.PushClearProgress()
context := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress}
i = 0
// 块级行级入库，缓存块
// 这里不能并行插入，因为 SQLite 不支持
for _, file := range files {
if file.isdir || !strings.HasSuffix(file.name, ".sy") {
continue
@ -163,29 +83,33 @@ func (box *Box) Index(fullRebuildIndex bool) (treeCount int, treeSize int64) {
continue
}
util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(93), util.ShortPathForBootingDisplay(tree.Path)))
tx, err := sql.BeginTx()
if nil != err {
continue
}
if err = sql.InsertBlocksSpans(tx, tree, context); nil != err {
sql.RollbackTx(tx)
continue
}
if err = sql.CommitTx(tx); nil != err {
continue
docIAL := parse.IAL2MapUnEsc(tree.Root.KramdownIAL)
if "" == docIAL["updated"] { // 早期的数据可能没有 updated 属性，这里进行订正
updated := util.TimeFromID(tree.Root.ID)
tree.Root.SetIALAttr("updated", updated)
docIAL["updated"] = updated
if writeErr := filesys.WriteTree(tree); nil != writeErr {
logging.LogErrorf("write tree [%s] failed: %s", tree.Path, writeErr)
}
}
cache.PutDocIAL(file.path, docIAL)
treenode.ReindexBlockTree(tree)
sql.UpsertTreeQueue(tree)
util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(92), util.ShortPathForBootingDisplay(tree.Path)))
treeSize += file.size
treeCount++
if 1 < i && 0 == i%64 {
util.PushEndlessProgress(fmt.Sprintf("["+box.Name+"] "+Conf.Language(53), i, treeCount-i))
util.PushEndlessProgress(fmt.Sprintf(Conf.Language(88), i, len(files)-i))
}
i++
}
box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间
end := time.Now()
elapsed := end.Sub(start).Seconds()
logging.LogInfof("rebuilt database for notebook [%s] in [%.2fs], tree [count=%d, size=%s]", box.ID, elapsed, treeCount, humanize.Bytes(uint64(treeSize)))
util.PushEndlessProgress(fmt.Sprintf(Conf.Language(56), treeCount))
return
}
@ -195,7 +119,7 @@ func IndexRefs() {
start := time.Now()
util.SetBootDetails("Resolving refs...")
util.PushEndlessProgress(Conf.Language(54))
util.PushStatusBar(Conf.Language(54))
// 引用入库
util.SetBootDetails("Indexing refs...")
@ -204,19 +128,15 @@ func IndexRefs() {
for _, refBlock := range refBlocks {
refTreeIDs.Add(refBlock.RootID)
}
i := 0
if 0 < refTreeIDs.Size() {
luteEngine := NewLute()
bootProgressPart := 10.0 / float64(refTreeIDs.Size())
for _, box := range Conf.GetOpenedBoxes() {
tx, err := sql.BeginTx()
if nil != err {
return
}
sql.DeleteRefsByBoxTx(tx, box.ID)
sql.CommitTx(tx)
sql.DeleteBoxRefsQueue(box.ID)
files := box.ListFiles("/")
i := 0
for _, file := range files {
if file.isdir || !strings.HasSuffix(file.name, ".sy") {
continue
@ -239,22 +159,16 @@ func IndexRefs() {
continue
}
tx, err = sql.BeginTx()
if nil != err {
continue
}
sql.InsertRefs(tx, tree)
if err = sql.CommitTx(tx); nil != err {
continue
}
sql.InsertRefsTreeQueue(tree)
if 1 < i && 0 == i%64 {
util.PushEndlessProgress(fmt.Sprintf(Conf.Language(55), i))
util.PushStatusBar(fmt.Sprintf(Conf.Language(55), i))
}
i++
}
}
}
logging.LogInfof("resolved refs [%d] in [%dms]", len(refBlocks), time.Now().Sub(start).Milliseconds())
util.PushStatusBar(fmt.Sprintf(Conf.Language(55), i))
}
func init() {

3
kernel/model/mount.go
View file

@ -195,8 +195,7 @@ func Mount(boxID string) (alreadyMount bool, err error) {
boxConf.Closed = false
box.SaveConf(boxConf)
box.Index(false)
IndexRefs()
box.Index()
// 缓存根一级的文档树展开
ListDocTree(box.ID, "/", Conf.FileTree.Sort)
treenode.SaveBlockTree(false)

6
kernel/model/sync.go
View file

@ -150,7 +150,11 @@ func syncData(boot, exit, byHand bool) {
msg := fmt.Sprintf(Conf.Language(82), synced)
Conf.Sync.Stat = msg
Conf.Save()
util.BroadcastByType("main", "syncing", 1, msg, nil)
code := 1
if nil != err {
code = 2
}
util.BroadcastByType("main", "syncing", code, msg, nil)
return
}

4
kernel/sql/block.go
View file

@ -65,10 +65,6 @@ func updateRootContent(tx *sql.Tx, content, updated, id string) {
cache.RemoveBlockIAL(id)
}
func InsertBlock(tx *sql.Tx, block *Block, context map[string]interface{}) (err error) {
return insertBlocks(tx, []*Block{block}, context)
}
func UpdateBlockContent(block *Block) {
tx, err := BeginTx()
if nil != err {

168
kernel/sql/queue.go
View file

@ -17,17 +17,11 @@
package sql
import (
"bytes"
"crypto/sha256"
"database/sql"
"fmt"
"path"
"sync"
"time"
"github.com/88250/lute/ast"
"github.com/88250/lute/parse"
"github.com/emirpasic/gods/sets/hashset"
"github.com/siyuan-note/eventbus"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/task"
@ -35,19 +29,20 @@ import (
)
var (
operationQueue []*treeQueueOperation
upsertTreeQueueLock = sync.Mutex{}
operationQueue []*dbQueueOperation
dbQueueLock = sync.Mutex{}
txLock = sync.Mutex{}
)
type treeQueueOperation struct {
type dbQueueOperation struct {
inQueueTime time.Time
action string // upsert/delete/delete_id/rename
action string // upsert/delete/delete_id/rename/delete_box/delete_box_refs/insert_refs
upsertTree *parse.Tree // upsert
upsertTree *parse.Tree // upsert/insert_refs
removeTreeBox, removeTreePath string // delete
removeTreeIDBox, removeTreeID string // delete_id
box string // delete_box/delete_box_refs
renameTree *parse.Tree // rename
renameTreeOldHPath string // rename
}
@ -88,8 +83,8 @@ func IsEmptyQueue() bool {
}
func ClearQueue() {
upsertTreeQueueLock.Lock()
defer upsertTreeQueueLock.Unlock()
dbQueueLock.Lock()
defer dbQueueLock.Unlock()
operationQueue = nil
}
@ -108,7 +103,6 @@ func FlushQueue() {
}
context := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar}
boxes := hashset.New()
for _, op := range ops {
switch op.action {
case "upsert":
@ -116,17 +110,21 @@ func FlushQueue() {
if err = upsertTree(tx, tree, context); nil != err {
logging.LogErrorf("upsert tree [%s] into database failed: %s", tree.Box+tree.Path, err)
}
boxes.Add(op.upsertTree.Box)
case "delete":
batchDeleteByPathPrefix(tx, op.removeTreeBox, op.removeTreePath)
boxes.Add(op.removeTreeBox)
case "delete_id":
DeleteByRootID(tx, op.removeTreeID)
boxes.Add(op.removeTreeIDBox)
case "rename":
batchUpdateHPath(tx, op.renameTree.Box, op.renameTree.ID, op.renameTreeOldHPath, op.renameTree.HPath)
updateRootContent(tx, path.Base(op.renameTree.HPath), op.renameTree.Root.IALAttr("updated"), op.renameTree.ID)
boxes.Add(op.renameTree.Box)
case "delete_box":
DeleteByBoxTx(tx, op.box)
case "delete_box_refs":
DeleteRefsByBoxTx(tx, op.box)
case "insert_refs":
InsertRefs(tx, op.upsertTree)
case "update_refs":
UpsertRefs(tx, op.upsertTree)
default:
logging.LogErrorf("unknown operation [%s]", op.action)
}
@ -136,39 +134,78 @@ func FlushQueue() {
if 5000 < elapsed {
logging.LogInfof("op tx [%dms]", elapsed)
}
start = time.Now()
tx, err = BeginTx()
if nil != err {
return
}
for _, box := range boxes.Values() {
if !ast.IsNodeIDPattern(box.(string)) {
continue
}
updateBoxHash(tx, box.(string))
}
CommitTx(tx)
elapsed = time.Now().Sub(start).Milliseconds()
if 1000 < elapsed {
logging.LogInfof("hash tx [%dms]", elapsed)
}
}
func mergeUpsertTrees() (ops []*treeQueueOperation) {
upsertTreeQueueLock.Lock()
defer upsertTreeQueueLock.Unlock()
func mergeUpsertTrees() (ops []*dbQueueOperation) {
dbQueueLock.Lock()
defer dbQueueLock.Unlock()
ops = operationQueue
operationQueue = nil
return
}
func UpsertTreeQueue(tree *parse.Tree) {
upsertTreeQueueLock.Lock()
defer upsertTreeQueueLock.Unlock()
func UpdateRefsTreeQueue(tree *parse.Tree) {
dbQueueLock.Lock()
defer dbQueueLock.Unlock()
newOp := &treeQueueOperation{upsertTree: tree, inQueueTime: time.Now(), action: "upsert"}
newOp := &dbQueueOperation{upsertTree: tree, inQueueTime: time.Now(), action: "update_refs"}
for i, op := range operationQueue {
if "update_refs" == op.action && op.upsertTree.ID == tree.ID {
operationQueue[i] = newOp
return
}
}
operationQueue = append(operationQueue, newOp)
}
func InsertRefsTreeQueue(tree *parse.Tree) {
dbQueueLock.Lock()
defer dbQueueLock.Unlock()
newOp := &dbQueueOperation{upsertTree: tree, inQueueTime: time.Now(), action: "insert_refs"}
for i, op := range operationQueue {
if "insert_refs" == op.action && op.upsertTree.ID == tree.ID {
operationQueue[i] = newOp
return
}
}
operationQueue = append(operationQueue, newOp)
}
func DeleteBoxRefsQueue(boxID string) {
dbQueueLock.Lock()
defer dbQueueLock.Unlock()
newOp := &dbQueueOperation{box: boxID, inQueueTime: time.Now(), action: "delete_box_refs"}
for i, op := range operationQueue {
if "delete_box_refs" == op.action && op.box == boxID {
operationQueue[i] = newOp
return
}
}
operationQueue = append(operationQueue, newOp)
}
func DeleteBoxQueue(boxID string) {
dbQueueLock.Lock()
defer dbQueueLock.Unlock()
newOp := &dbQueueOperation{box: boxID, inQueueTime: time.Now(), action: "delete_box"}
for i, op := range operationQueue {
if "delete_box" == op.action && op.box == boxID {
operationQueue[i] = newOp
return
}
}
operationQueue = append(operationQueue, newOp)
}
func UpsertTreeQueue(tree *parse.Tree) {
dbQueueLock.Lock()
defer dbQueueLock.Unlock()
newOp := &dbQueueOperation{upsertTree: tree, inQueueTime: time.Now(), action: "upsert"}
for i, op := range operationQueue {
if "upsert" == op.action && op.upsertTree.ID == tree.ID { // 相同树则覆盖
operationQueue[i] = newOp
@ -179,10 +216,10 @@ func UpsertTreeQueue(tree *parse.Tree) {
}
func RenameTreeQueue(tree *parse.Tree, oldHPath string) {
upsertTreeQueueLock.Lock()
defer upsertTreeQueueLock.Unlock()
dbQueueLock.Lock()
defer dbQueueLock.Unlock()
newOp := &treeQueueOperation{
newOp := &dbQueueOperation{
renameTree: tree,
renameTreeOldHPath: oldHPath,
inQueueTime: time.Now(),
@ -197,10 +234,10 @@ func RenameTreeQueue(tree *parse.Tree, oldHPath string) {
}
func RemoveTreeQueue(box, rootID string) {
upsertTreeQueueLock.Lock()
defer upsertTreeQueueLock.Unlock()
dbQueueLock.Lock()
defer dbQueueLock.Unlock()
var tmp []*treeQueueOperation
var tmp []*dbQueueOperation
// 将已有的 upsert 操作去重
for _, op := range operationQueue {
if "upsert" == op.action && op.upsertTree.ID != rootID {
@ -209,15 +246,15 @@ func RemoveTreeQueue(box, rootID string) {
}
operationQueue = tmp
newOp := &treeQueueOperation{removeTreeIDBox: box, removeTreeID: rootID, inQueueTime: time.Now(), action: "delete_id"}
newOp := &dbQueueOperation{removeTreeIDBox: box, removeTreeID: rootID, inQueueTime: time.Now(), action: "delete_id"}
operationQueue = append(operationQueue, newOp)
}
func RemoveTreePathQueue(treeBox, treePathPrefix string) {
upsertTreeQueueLock.Lock()
defer upsertTreeQueueLock.Unlock()
dbQueueLock.Lock()
defer dbQueueLock.Unlock()
var tmp []*treeQueueOperation
var tmp []*dbQueueOperation
// 将已有的 upsert 操作去重
for _, op := range operationQueue {
if "upsert" == op.action && (op.removeTreeBox != treeBox || op.upsertTree.Path != treePathPrefix) {
@ -226,31 +263,6 @@ func RemoveTreePathQueue(treeBox, treePathPrefix string) {
}
operationQueue = tmp
newOp := &treeQueueOperation{removeTreeBox: treeBox, removeTreePath: treePathPrefix, inQueueTime: time.Now(), action: "delete"}
newOp := &dbQueueOperation{removeTreeBox: treeBox, removeTreePath: treePathPrefix, inQueueTime: time.Now(), action: "delete"}
operationQueue = append(operationQueue, newOp)
}
func updateBoxHash(tx *sql.Tx, boxID string) {
sum := boxChecksum(boxID)
PutBoxHash(tx, boxID, sum)
}
func boxChecksum(box string) (ret string) {
rows, err := query("SELECT hash FROM blocks WHERE type = 'd' AND box = ? ORDER BY id DESC", box)
if nil != err {
logging.LogErrorf("sql query failed: %s", err)
return
}
defer rows.Close()
buf := bytes.Buffer{}
for rows.Next() {
var hash string
if err = rows.Scan(&hash); nil != err {
logging.LogErrorf("query scan field failed: %s", err)
return
}
buf.WriteString(hash)
}
ret = fmt.Sprintf("%x", sha256.Sum256(buf.Bytes()))
return
}

21
kernel/sql/stat.go
View file

@ -54,27 +54,6 @@ func setDatabaseVer() {
CommitTx(tx)
}
func ClearBoxHash(tx *sql.Tx) {
stmt := "DELETE FROM stat WHERE `key` LIKE '%_hash'"
execStmtTx(tx, stmt)
}
func RemoveBoxHash(tx *sql.Tx, box string) {
key := box + "_hash"
stmt := "DELETE FROM stat WHERE `key` = '" + key + "'"
execStmtTx(tx, stmt)
}
func PutBoxHash(tx *sql.Tx, box, hash string) {
key := box + "_hash"
putStat(tx, key, hash)
}
func GetBoxHash(box string) string {
key := box + "_hash"
return getStat(key)
}
func putStat(tx *sql.Tx, key, value string) (err error) {
stmt := "DELETE FROM stat WHERE `key` = '" + key + "'"
if err = execStmtTx(tx, stmt); nil != err {

24
kernel/sql/upsert.go
View file

@ -36,13 +36,6 @@ func init() {
luteEngine.RenderOptions.KramdownBlockIAL = false // 数据库 markdown 字段为标准 md，但是要保留 span block ial
}
func InsertBlocksSpans(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) {
if err = insertBlocksSpans(tx, tree, context); nil != err {
logging.LogErrorf("insert tree [%s] into database failed: %s", tree.Box+tree.Path, err)
}
return
}
func InsertRefs(tx *sql.Tx, tree *parse.Tree) {
if err := insertRef(tx, tree); nil != err {
logging.LogErrorf("insert refs tree [%s] into database failed: %s", tree.Box+tree.Path, err)
@ -395,23 +388,6 @@ func insertFileAnnotationRefs0(tx *sql.Tx, bulk []*FileAnnotationRef) (err error
return
}
func insertBlocksSpans(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) {
blocks, spans, assets, attributes := fromTree(tree.Root, tree)
if err = insertBlocks(tx, blocks, context); nil != err {
return
}
if err = insertSpans(tx, spans); nil != err {
return
}
if err = insertAssets(tx, assets); nil != err {
return
}
if err = insertAttributes(tx, attributes); nil != err {
return
}
return
}
func insertRef(tx *sql.Tx, tree *parse.Tree) (err error) {
refs, fileAnnotationRefs := refsFromTree(tree)
if err = insertRefs(tx, refs); nil != err {

1
kernel/task/queue.go
View file

@ -102,6 +102,7 @@ const (
RepoCheckout = "task.repo.checkout" // 从快照中检出
DatabaseIndexFull = "task.database.index.full" // 重建索引
DatabaseIndex = "task.database.index" // 数据库索引队列
DatabaseIndexRef = "task.database.index.ref" // 数据库索引引用
DatabaseIndexFix = "task.database.index.fix" // 数据库索引订正
OCRImage = "task.ocr.image" // 图片 OCR 提取文本
HistoryGenerateDoc = "task.history.generateDoc" // 生成文件历史

22
kernel/treenode/blocktree.go
View file

@ -320,28 +320,6 @@ func ReindexBlockTree(tree *parse.Tree) {
blockTreesChanged = true
}
func IndexBlockTree(tree *parse.Tree) {
blockTreesLock.Lock()
defer blockTreesLock.Unlock()
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering || !n.IsBlock() {
return ast.WalkContinue
}
var parentID string
if nil != n.Parent {
parentID = n.Parent.ID
}
if "" == n.ID {
return ast.WalkContinue
}
blockTrees[n.ID] = &BlockTree{ID: n.ID, ParentID: parentID, RootID: tree.ID, BoxID: tree.Box, Path: tree.Path, HPath: tree.HPath, Updated: tree.Root.IALAttr("updated")}
return ast.WalkContinue
})
// 新建索引不变更持久化文件，调用处会负责调用 SaveBlockTree()
}
func AutoFlushBlockTree() {
for {
SaveBlockTree(false)