Merge remote-tracking branch 'origin/dev' into dev

app/appearance/langs/en_US.json

@@ -928,10 +928,10 @@
     "50": "Resolving reference [%s]",
     "51": "Cache content block [%d]",
     "52": "Every time you open the user guide notebook data will be reset, so please do not save any data in it",
-    "53": "The index of [%d] documents has been completed, and [%d] remain to be processed",
+    "53": "TODO",
     "54": "Indexing references...",
     "55": "Indexed references of [%d] documents",
-    "56": "Indexed [%d] documents",
+    "56": "TODO",
     "57": "Failed to create temp key",
     "58": "After the index is rebuilt, the interface will be automatically refreshed later...",
     "59": "Failed to set sync ignore list",
@@ -968,7 +968,7 @@
     "90": "Created [%d] of search indexes of block-level elements [%s]",
     "91": "Reading block tree data...",
     "92": "Parsing document tree [%s]",
-    "93": "Indexing document tree [%s]",
+    "93": "TODO",
     "94": "Upload failed: %s",
     "95": "Exiting...",
     "96": "Synchronization failed when exiting. Please manually perform a synchronization to ensure that the local data is consistent with the cloud data",
@@ -983,7 +983,7 @@
     "105": "Corrupted data repo have been automatically reset",
     "106": "Maximum length is limited to 512 characters",
     "107": "Moving document [%s]",
-    "108": "Cleaning obsolete indexes...",
+    "108": "TODO",
     "109": "Remove reminder completed [%s]",
     "110": "Renaming...",
     "111": "Saving document [%s]...",

app/appearance/langs/es_ES.json

@@ -928,10 +928,10 @@
     "50": "Resolviendo referencia [%s]",
     "51": "Bloque de contenido de la caché [%d]",
     "52": "Cada vez que abra la guía del usuario se restablecerán los datos del cuaderno, así que no guarde ningún dato en ella",
-    "53": "Se ha completado el índice de [%d] documentos y quedan [%d] por procesar",
+    "53": "TODO",
     "54": "Indexando referencias...",
     "55": "Referencias indexadas de [%d] documentos",
-    "56": "Documentos indexados [%d]",
+    "56": "TODO",
     "57": "Fallo en la creación de la clave temporal",
     "58": "Después de reconstruir el índice, la interfaz se actualizará automáticamente más tarde...",
     "59": " Falló la configuración de sincronización de la lista de ignorados",
@@ -968,7 +968,7 @@
     "90": "Creado [%d] de índices de búsqueda de elementos a nivel de bloque [%s]",
     "91": "Leyendo datos del árbol de bloques...",
     "92": "Analizando el árbol del documento [%s]",
-    "93": "Árbol de documentos de indexación [%s]",
+    "93": "TODO",
     "94": "Carga fallida: %s",
     "95": "Saliendo...",
     "96": "La sincronización falló al salir. Por favor, realice manualmente una sincronización para asegurarse de que los datos locales son coherentes con los datos de la nube",
@@ -983,7 +983,7 @@
     "105": "El repositorio de datos corruptos se ha restablecido automáticamente",
     "106": "La longitud máxima está limitada a 512 caracteres",
     "107": "Moviendo documento [%s]",
-    "108": "Limpiando índices obsoletos...",
+    "108": "TODO",
     "109": "Eliminación de recordatorios completada [%s]",
     "110": "Renombrar...",
     "111": "Guardando documento [%s]...",

app/appearance/langs/fr_FR.json

@@ -928,10 +928,10 @@
     "50": "Résolution des référence [%s]",
     "51": "Blocage du contenu du cache [%d]",
     "52": "Chaque fois que vous ouvrez le guide de l'utilisateur, les données du notebook seront réinitialisées, veuillez donc ne pas y enregistrer de données.",
-    "53": "L'indexation de [%d] documents est terminée, et [%d] restent à traiter.",
+    "53": "TODO",
     "54": "Indexation des références...",
     "55": "Références indexées de [%d] documents",
-    "56": "Indexé [%d] documents",
+    "56": "TODO",
     "57": "Échec de la création d'une clé temporaire",
     "58": "Une fois l'index reconstruit, l'interface sera automatiquement rafraîchie ultérieurement...",
     "59": "Échec de la définition de la liste des ignores de synchronisation",
@@ -968,7 +968,7 @@
     "90": "Création de [%d] index de recherche d'éléments de niveau bloc [%s]",
     "91": "Lecture des données de l'arborescence des blocs...",
     "92": "Analyse de l'arborescence du document [%s]",
-    "93": "Indexation de l'arborescence du document [%s]",
+    "93": "TODO",
     "94": "Échec du téléchargement : %s",
     "95": "Quitter le programme...",
     "96": "La synchronisation a échoué lors de la sortie. Veuillez effectuer une synchronisation manuellement pour vous assurer que les données locales sont cohérentes avec les données du cloud",
@@ -983,7 +983,7 @@
     "105": "Le référentiel de données corrompu a été automatiquement réinitialisé",
     "106": "La longueur maximale est limitée à 512 caractères",
     "107": "Déplacement du document [%s]",
-    "108": "Nettoyage des index obsolètes...",
+    "108": "TODO",
     "109": "Supprimer le rappel terminé [%s]",
     "110": "Renommer...",
     "111": "Enregistrement du document [%s]...",

app/appearance/langs/zh_CHT.json

@@ -928,10 +928,10 @@
     "50": "正在解析引用 [%s]",
     "51": "正在快取內容塊 [%d]",
     "52": "每次打開使用者指南筆記本資料都會被重置，所以請勿在其中保存任何資料",
-    "53": "已完成索引 [%d] 篇文檔，剩餘待處理 [%d]",
+    "53": "TODO",
     "54": "正在索引引用關係...",
     "55": "已完成索引 [%d] 篇文檔的引用關係",
-    "56": "已完成索引 [%d] 篇文檔",
+    "56": "TODO",
     "57": "創建臨時金鑰失敗",
     "58": "重建索引完畢，稍後將自動重新整理介面...",
     "59": "設置同步忽略列表失敗",
@@ -968,7 +968,7 @@
     "90": "已經建立 [%d] 個塊級元素的搜索索引 [%s]",
     "91": "正在讀取塊樹數據...",
     "92": "正在解析文檔樹 [%s]",
-    "93": "正在索引文檔樹 [%s]",
+    "93": "TODO",
     "94": "上傳失敗：%s",
     "95": "正在退出...",
     "96": "退出時同步失敗，請手動執行一次同步以確保本地資料和雲端資料一致",
@@ -983,7 +983,7 @@
     "105": "已經自動重置損壞的數據倉庫",
     "106": "最大長度限制為 512 字元",
     "107": "正在移動文檔 [%s]",
-    "108": "正在清理已過時的索引...",
+    "108": "TODO",
     "109": "移除提醒完畢 [%s]",
     "110": "正在重命名...",
     "111": "正在保存文檔 [%s]...",

app/appearance/langs/zh_CN.json

@@ -928,10 +928,10 @@
     "50": "正在解析引用 [%s]",
     "51": "正在缓存内容块 [%d]",
     "52": "每次打开用户指南笔记本数据都会被重置，所以请勿在其中保存任何数据",
-    "53": "已完成索引 [%d] 篇文档，剩余待处理 [%d]",
+    "53": "TODO",
     "54": "正在索引引用关系...",
     "55": "已完成索引 [%d] 篇文档的引用关系",
-    "56": "已完成索引 [%d] 篇文档",
+    "56": "TODO",
     "57": "创建临时密钥失败",
     "58": "重建索引完毕，稍后将自动刷新界面...",
     "59": "设置同步忽略列表失败",
@@ -968,7 +968,7 @@
     "90": "已经建立 [%d] 个块级元素的搜索索引 [%s]",
     "91": "正在读取块树数据...",
     "92": "正在解析文档树 [%s]",
-    "93": "正在索引文档树 [%s]",
+    "93": "TODO",
     "94": "上传失败：%s",
     "95": "正在退出...",
     "96": "退出时同步失败，请手动执行一次同步以确保本地数据和云端数据一致",
@@ -983,7 +983,7 @@
     "105": "已经自动重置损坏的数据仓库",
     "106": "最大长度限制为 512 字符",
     "107": "正在移动文档 [%s]",
-    "108": "正在清理已过时的索引...",
+    "108": "TODO",
     "109": "移除提醒完毕 [%s]",
     "110": "正在重命名...",
     "111": "正在保存文档 [%s]...",

kernel/api/transaction.go

@@ -25,9 +25,7 @@ import (
 	"github.com/88250/gulu"
 	"github.com/gin-gonic/gin"
 	"github.com/siyuan-note/filelock"
-	"github.com/siyuan-note/logging"
 	"github.com/siyuan-note/siyuan/kernel/model"
-	"github.com/siyuan-note/siyuan/kernel/sql"
 	"github.com/siyuan-note/siyuan/kernel/util"
 )
 
@@ -67,17 +65,6 @@ func performTransactions(c *gin.Context) {
 		ret.Code = 1
 		return
 	}
-	if nil != err {
-		tx, txErr := sql.BeginTx()
-		if nil != txErr {
-			logging.LogFatalf("transaction failed: %s", txErr)
-			return
-		}
-		sql.ClearBoxHash(tx)
-		sql.CommitTx(tx)
-		logging.LogFatalf("transaction failed: %s", err)
-		return
-	}
 
 	ret.Data = transactions
 

kernel/model/backlink.go

@@ -40,24 +40,19 @@ import (
 func RefreshBacklink(id string) {
 	WaitForWritingFiles()
 
-	tx, err := sql.BeginTx()
-	if nil != err {
-		return
-	}
-	defer sql.CommitTx(tx)
-
 	refs := sql.QueryRefsByDefID(id, false)
 	trees := map[string]*parse.Tree{}
 	for _, ref := range refs {
 		tree := trees[ref.RootID]
 		if nil == tree {
-			tree, err = loadTreeByBlockID(ref.RootID)
-			if nil != err {
-				logging.LogErrorf("refresh tree refs failed: %s", err)
+			var loadErr error
+			tree, loadErr = loadTreeByBlockID(ref.RootID)
+			if nil != loadErr {
+				logging.LogErrorf("refresh tree refs failed: %s", loadErr)
 				continue
 			}
 			trees[ref.RootID] = tree
-			sql.UpsertRefs(tx, tree)
+			sql.UpdateRefsTreeQueue(tree)
 		}
 	}
 }

kernel/model/box.go

@@ -339,22 +339,6 @@ func (box *Box) Remove(path string) error {
 	return nil
 }
 
-func (box *Box) Unindex() {
-	task.PrependTask(task.DatabaseIndex, unindex, box.ID)
-}
-
-func unindex(boxID string) {
-	tx, err := sql.BeginTx()
-	if nil != err {
-		return
-	}
-	sql.RemoveBoxHash(tx, boxID)
-	sql.DeleteByBoxTx(tx, boxID)
-	sql.CommitTx(tx)
-	ids := treenode.RemoveBlockTreesByBoxID(boxID)
-	RemoveRecentDoc(ids)
-}
-
 func (box *Box) ListFiles(path string) (ret []*FileInfo) {
 	fis, _, err := box.Ls(path)
 	if nil != err {
@@ -498,6 +482,7 @@ func genTreeID(tree *parse.Tree) {
 
 func FullReindex() {
 	task.PrependTask(task.DatabaseIndexFull, fullReindex)
+	task.AppendTask(task.DatabaseIndexRef, IndexRefs)
 }
 
 func fullReindex() {
@@ -510,11 +495,13 @@ func fullReindex() {
 	}
 	treenode.InitBlockTree(true)
 
+	sql.IndexMode()
+	defer sql.NormalMode()
+
 	openedBoxes := Conf.GetOpenedBoxes()
 	for _, openedBox := range openedBoxes {
-		openedBox.Index(true)
+		index(openedBox.ID)
 	}
-	IndexRefs()
 	treenode.SaveBlockTree(true)
 	LoadFlashcards()
 

kernel/model/conf.go

@@ -599,7 +599,7 @@ func InitBoxes() {
 		box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间
 
 		if !initialized {
-			box.Index(true)
+			index(box.ID)
 		}
 	}
 

kernel/model/file.go

@@ -912,15 +912,7 @@ func createTreeTx(tree *parse.Tree) {
 	transaction := &Transaction{DoOperations: []*Operation{{Action: "create", Data: tree}}}
 	err := PerformTransactions(&[]*Transaction{transaction})
 	if nil != err {
-		tx, txErr := sql.BeginTx()
-		if nil != txErr {
-			logging.LogFatalf("transaction failed: %s", txErr)
-			return
-		}
-		sql.ClearBoxHash(tx)
-		sql.CommitTx(tx)
 		logging.LogFatalf("transaction failed: %s", err)
-		return
 	}
 }
 
@@ -1467,13 +1459,6 @@ func createDoc(boxID, p, title, dom string) (err error) {
 	transaction := &Transaction{DoOperations: []*Operation{{Action: "create", Data: tree}}}
 	err = PerformTransactions(&[]*Transaction{transaction})
 	if nil != err {
-		tx, txErr := sql.BeginTx()
-		if nil != txErr {
-			logging.LogFatalf("transaction failed: %s", txErr)
-			return
-		}
-		sql.ClearBoxHash(tx)
-		sql.CommitTx(tx)
 		logging.LogFatalf("transaction failed: %s", err)
 		return
 	}

kernel/model/import.go

@@ -393,7 +393,7 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
 			continue
 		}
 
-		treenode.IndexBlockTree(tree)
+		treenode.ReindexBlockTree(tree)
 		sql.UpsertTreeQueue(tree)
 	}
 

kernel/model/index.go

@@ -17,11 +17,7 @@
 package model
 
 import (
-	"bytes"
-	"crypto/sha256"
 	"fmt"
-	"runtime/debug"
-	"sort"
 	"strings"
 	"time"
 
@@ -33,20 +29,31 @@ import (
 	"github.com/siyuan-note/siyuan/kernel/cache"
 	"github.com/siyuan-note/siyuan/kernel/filesys"
 	"github.com/siyuan-note/siyuan/kernel/sql"
+	"github.com/siyuan-note/siyuan/kernel/task"
 	"github.com/siyuan-note/siyuan/kernel/treenode"
 	"github.com/siyuan-note/siyuan/kernel/util"
 )
 
-func (box *Box) Index(fullRebuildIndex bool) (treeCount int, treeSize int64) {
-	defer debug.FreeOSMemory()
+func (box *Box) Unindex() {
+	task.PrependTask(task.DatabaseIndex, unindex, box.ID)
+}
+
+func unindex(boxID string) {
+	ids := treenode.RemoveBlockTreesByBoxID(boxID)
+	RemoveRecentDoc(ids)
+	sql.DeleteBoxQueue(boxID)
+}
 
-	sql.IndexMode()
-	defer sql.NormalMode()
+func (box *Box) Index() {
+	task.PrependTask(task.DatabaseIndex, index, box.ID)
+	task.AppendTask(task.DatabaseIndexRef, IndexRefs)
+}
 
-	//os.MkdirAll("pprof", 0755)
-	//cpuProfile, _ := os.Create("pprof/cpu_profile_index")
-	//pprof.StartCPUProfile(cpuProfile)
-	//defer pprof.StopCPUProfile()
+func index(boxID string) {
+	box := Conf.Box(boxID)
+	if nil == box {
+		return
+	}
 
 	util.SetBootDetails("Listing files...")
 	files := box.ListFiles("/")
@@ -54,48 +61,45 @@ func (box *Box) Index(fullRebuildIndex bool) (treeCount int, treeSize int64) {
 	if 1 > boxLen {
 		boxLen = 1
 	}
-	bootProgressPart := 10.0 / float64(boxLen) / float64(len(files))
+	bootProgressPart := 30.0 / float64(boxLen) / float64(len(files))
 
+	start := time.Now()
 	luteEngine := NewLute()
-	idTitleMap := map[string]string{}
-	idHashMap := map[string]string{}
+	var treeCount int
+	var treeSize int64
+	i := 0
 
 	util.PushEndlessProgress(fmt.Sprintf("["+box.Name+"] "+Conf.Language(64), len(files)))
+	defer util.PushClearProgress()
 
-	i := 0
-	// 读取并缓存路径映射
 	for _, file := range files {
 		if file.isdir || !strings.HasSuffix(file.name, ".sy") {
 			continue
 		}
 
-		p := file.path
-
-		tree, err := filesys.LoadTree(box.ID, p, luteEngine)
+		tree, err := filesys.LoadTree(box.ID, file.path, luteEngine)
 		if nil != err {
-			logging.LogErrorf("read box [%s] tree [%s] failed: %s", box.ID, p, err)
+			logging.LogErrorf("read box [%s] tree [%s] failed: %s", box.ID, file.path, err)
 			continue
 		}
 
 		docIAL := parse.IAL2MapUnEsc(tree.Root.KramdownIAL)
-		if "" == docIAL["updated"] {
+		if "" == docIAL["updated"] { // 早期的数据可能没有 updated 属性，这里进行订正
 			updated := util.TimeFromID(tree.Root.ID)
 			tree.Root.SetIALAttr("updated", updated)
 			docIAL["updated"] = updated
-			writeJSONQueue(tree)
+			if writeErr := filesys.WriteTree(tree); nil != writeErr {
+				logging.LogErrorf("write tree [%s] failed: %s", tree.Path, writeErr)
+			}
 		}
 
-		cache.PutDocIAL(p, docIAL)
+		cache.PutDocIAL(file.path, docIAL)
+		treenode.ReindexBlockTree(tree)
+		sql.UpsertTreeQueue(tree)
 
 		util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(92), util.ShortPathForBootingDisplay(tree.Path)))
 		treeSize += file.size
 		treeCount++
-		// 缓存文档标题，后面做 Path -> HPath 路径映射时需要
-		idTitleMap[tree.ID] = tree.Root.IALAttr("title")
-		// 缓存块树
-		treenode.IndexBlockTree(tree)
-		// 缓存 ID-Hash，后面需要用于判断是否要重建库
-		idHashMap[tree.ID] = tree.Hash
 		if 1 < i && 0 == i%64 {
 			util.PushEndlessProgress(fmt.Sprintf(Conf.Language(88), i, len(files)-i))
 		}
@@ -103,89 +107,9 @@ func (box *Box) Index(fullRebuildIndex bool) (treeCount int, treeSize int64) {
 	}
 
 	box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间
-
-	// 检查是否需要重新建库
-	util.SetBootDetails("Checking data hashes...")
-	var ids []string
-	for id := range idTitleMap {
-		ids = append(ids, id)
-	}
-	sort.Slice(ids, func(i, j int) bool { return ids[i] >= ids[j] })
-	buf := bytes.Buffer{}
-	for _, id := range ids {
-		hash, _ := idHashMap[id]
-		buf.WriteString(hash)
-		util.SetBootDetails("Checking hash " + hash)
-	}
-	boxHash := fmt.Sprintf("%x", sha256.Sum256(buf.Bytes()))
-
-	dbBoxHash := sql.GetBoxHash(box.ID)
-	if boxHash == dbBoxHash {
-		//logging.LogInfof("use existing database for box [%s]", box.ID)
-		util.SetBootDetails("Use existing database for notebook " + box.ID)
-		return
-	}
-
-	// 开始重建库
-
-	sql.DisableCache()
-	defer sql.EnableCache()
-
-	start := time.Now()
-	if !fullRebuildIndex {
-		tx, err := sql.BeginTx()
-		if nil != err {
-			return
-		}
-		sql.PutBoxHash(tx, box.ID, boxHash)
-		util.SetBootDetails("Cleaning obsolete indexes...")
-		util.PushEndlessProgress(Conf.Language(108))
-		sql.DeleteByBoxTx(tx, box.ID)
-		if err = sql.CommitTx(tx); nil != err {
-			return
-		}
-	}
-
-	bootProgressPart = 20.0 / float64(boxLen) / float64(treeCount)
-
-	context := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress}
-	i = 0
-	// 块级行级入库，缓存块
-	// 这里不能并行插入，因为 SQLite 不支持
-	for _, file := range files {
-		if file.isdir || !strings.HasSuffix(file.name, ".sy") {
-			continue
-		}
-
-		tree, err := filesys.LoadTree(box.ID, file.path, luteEngine)
-		if nil != err {
-			logging.LogErrorf("read box [%s] tree [%s] failed: %s", box.ID, file.path, err)
-			continue
-		}
-
-		util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(93), util.ShortPathForBootingDisplay(tree.Path)))
-		tx, err := sql.BeginTx()
-		if nil != err {
-			continue
-		}
-		if err = sql.InsertBlocksSpans(tx, tree, context); nil != err {
-			sql.RollbackTx(tx)
-			continue
-		}
-		if err = sql.CommitTx(tx); nil != err {
-			continue
-		}
-		if 1 < i && 0 == i%64 {
-			util.PushEndlessProgress(fmt.Sprintf("["+box.Name+"] "+Conf.Language(53), i, treeCount-i))
-		}
-		i++
-	}
-
 	end := time.Now()
 	elapsed := end.Sub(start).Seconds()
 	logging.LogInfof("rebuilt database for notebook [%s] in [%.2fs], tree [count=%d, size=%s]", box.ID, elapsed, treeCount, humanize.Bytes(uint64(treeSize)))
-
-	util.PushEndlessProgress(fmt.Sprintf(Conf.Language(56), treeCount))
 	return
 }
 
@@ -195,7 +119,7 @@ func IndexRefs() {
 
 	start := time.Now()
 	util.SetBootDetails("Resolving refs...")
-	util.PushEndlessProgress(Conf.Language(54))
+	util.PushStatusBar(Conf.Language(54))
 
 	// 引用入库
 	util.SetBootDetails("Indexing refs...")
@@ -204,19 +128,15 @@ func IndexRefs() {
 	for _, refBlock := range refBlocks {
 		refTreeIDs.Add(refBlock.RootID)
 	}
+
+	i := 0
 	if 0 < refTreeIDs.Size() {
 		luteEngine := NewLute()
 		bootProgressPart := 10.0 / float64(refTreeIDs.Size())
 		for _, box := range Conf.GetOpenedBoxes() {
-			tx, err := sql.BeginTx()
-			if nil != err {
-				return
-			}
-			sql.DeleteRefsByBoxTx(tx, box.ID)
-			sql.CommitTx(tx)
+			sql.DeleteBoxRefsQueue(box.ID)
 
 			files := box.ListFiles("/")
-			i := 0
 			for _, file := range files {
 				if file.isdir || !strings.HasSuffix(file.name, ".sy") {
 					continue
@@ -239,22 +159,16 @@ func IndexRefs() {
 					continue
 				}
 
-				tx, err = sql.BeginTx()
-				if nil != err {
-					continue
-				}
-				sql.InsertRefs(tx, tree)
-				if err = sql.CommitTx(tx); nil != err {
-					continue
-				}
+				sql.InsertRefsTreeQueue(tree)
 				if 1 < i && 0 == i%64 {
-					util.PushEndlessProgress(fmt.Sprintf(Conf.Language(55), i))
+					util.PushStatusBar(fmt.Sprintf(Conf.Language(55), i))
 				}
 				i++
 			}
 		}
 	}
 	logging.LogInfof("resolved refs [%d] in [%dms]", len(refBlocks), time.Now().Sub(start).Milliseconds())
+	util.PushStatusBar(fmt.Sprintf(Conf.Language(55), i))
 }
 
 func init() {

kernel/model/mount.go

@@ -195,8 +195,7 @@ func Mount(boxID string) (alreadyMount bool, err error) {
 	boxConf.Closed = false
 	box.SaveConf(boxConf)
 
-	box.Index(false)
-	IndexRefs()
+	box.Index()
 	// 缓存根一级的文档树展开
 	ListDocTree(box.ID, "/", Conf.FileTree.Sort)
 	treenode.SaveBlockTree(false)

kernel/model/sync.go

@@ -150,7 +150,11 @@ func syncData(boot, exit, byHand bool) {
 	msg := fmt.Sprintf(Conf.Language(82), synced)
 	Conf.Sync.Stat = msg
 	Conf.Save()
-	util.BroadcastByType("main", "syncing", 1, msg, nil)
+	code := 1
+	if nil != err {
+		code = 2
+	}
+	util.BroadcastByType("main", "syncing", code, msg, nil)
 	return
 }
 

kernel/sql/block.go

@@ -65,10 +65,6 @@ func updateRootContent(tx *sql.Tx, content, updated, id string) {
 	cache.RemoveBlockIAL(id)
 }
 
-func InsertBlock(tx *sql.Tx, block *Block, context map[string]interface{}) (err error) {
-	return insertBlocks(tx, []*Block{block}, context)
-}
-
 func UpdateBlockContent(block *Block) {
 	tx, err := BeginTx()
 	if nil != err {

kernel/sql/queue.go

@@ -17,17 +17,11 @@
 package sql
 
 import (
-	"bytes"
-	"crypto/sha256"
-	"database/sql"
-	"fmt"
 	"path"
 	"sync"
 	"time"
 
-	"github.com/88250/lute/ast"
 	"github.com/88250/lute/parse"
-	"github.com/emirpasic/gods/sets/hashset"
 	"github.com/siyuan-note/eventbus"
 	"github.com/siyuan-note/logging"
 	"github.com/siyuan-note/siyuan/kernel/task"
@@ -35,19 +29,20 @@ import (
 )
 
 var (
-	operationQueue      []*treeQueueOperation
-	upsertTreeQueueLock = sync.Mutex{}
+	operationQueue []*dbQueueOperation
+	dbQueueLock    = sync.Mutex{}
 
 	txLock = sync.Mutex{}
 )
 
-type treeQueueOperation struct {
+type dbQueueOperation struct {
 	inQueueTime time.Time
-	action      string // upsert/delete/delete_id/rename
+	action      string // upsert/delete/delete_id/rename/delete_box/delete_box_refs/insert_refs
 
-	upsertTree                    *parse.Tree // upsert
+	upsertTree                    *parse.Tree // upsert/insert_refs
 	removeTreeBox, removeTreePath string      // delete
 	removeTreeIDBox, removeTreeID string      // delete_id
+	box                           string      // delete_box/delete_box_refs
 	renameTree                    *parse.Tree // rename
 	renameTreeOldHPath            string      // rename
 }
@@ -88,8 +83,8 @@ func IsEmptyQueue() bool {
 }
 
 func ClearQueue() {
-	upsertTreeQueueLock.Lock()
-	defer upsertTreeQueueLock.Unlock()
+	dbQueueLock.Lock()
+	defer dbQueueLock.Unlock()
 	operationQueue = nil
 }
 
@@ -108,7 +103,6 @@ func FlushQueue() {
 	}
 
 	context := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar}
-	boxes := hashset.New()
 	for _, op := range ops {
 		switch op.action {
 		case "upsert":
@@ -116,17 +110,21 @@ func FlushQueue() {
 			if err = upsertTree(tx, tree, context); nil != err {
 				logging.LogErrorf("upsert tree [%s] into database failed: %s", tree.Box+tree.Path, err)
 			}
-			boxes.Add(op.upsertTree.Box)
 		case "delete":
 			batchDeleteByPathPrefix(tx, op.removeTreeBox, op.removeTreePath)
-			boxes.Add(op.removeTreeBox)
 		case "delete_id":
 			DeleteByRootID(tx, op.removeTreeID)
-			boxes.Add(op.removeTreeIDBox)
 		case "rename":
 			batchUpdateHPath(tx, op.renameTree.Box, op.renameTree.ID, op.renameTreeOldHPath, op.renameTree.HPath)
 			updateRootContent(tx, path.Base(op.renameTree.HPath), op.renameTree.Root.IALAttr("updated"), op.renameTree.ID)
-			boxes.Add(op.renameTree.Box)
+		case "delete_box":
+			DeleteByBoxTx(tx, op.box)
+		case "delete_box_refs":
+			DeleteRefsByBoxTx(tx, op.box)
+		case "insert_refs":
+			InsertRefs(tx, op.upsertTree)
+		case "update_refs":
+			UpsertRefs(tx, op.upsertTree)
 		default:
 			logging.LogErrorf("unknown operation [%s]", op.action)
 		}
@@ -136,39 +134,78 @@ func FlushQueue() {
 	if 5000 < elapsed {
 		logging.LogInfof("op tx [%dms]", elapsed)
 	}
+}
 
-	start = time.Now()
-	tx, err = BeginTx()
-	if nil != err {
-		return
+func mergeUpsertTrees() (ops []*dbQueueOperation) {
+	dbQueueLock.Lock()
+	defer dbQueueLock.Unlock()
+
+	ops = operationQueue
+	operationQueue = nil
+	return
+}
+
+func UpdateRefsTreeQueue(tree *parse.Tree) {
+	dbQueueLock.Lock()
+	defer dbQueueLock.Unlock()
+
+	newOp := &dbQueueOperation{upsertTree: tree, inQueueTime: time.Now(), action: "update_refs"}
+	for i, op := range operationQueue {
+		if "update_refs" == op.action && op.upsertTree.ID == tree.ID {
+			operationQueue[i] = newOp
+			return
+		}
 	}
-	for _, box := range boxes.Values() {
-		if !ast.IsNodeIDPattern(box.(string)) {
-			continue
+	operationQueue = append(operationQueue, newOp)
+}
+
+func InsertRefsTreeQueue(tree *parse.Tree) {
+	dbQueueLock.Lock()
+	defer dbQueueLock.Unlock()
+
+	newOp := &dbQueueOperation{upsertTree: tree, inQueueTime: time.Now(), action: "insert_refs"}
+	for i, op := range operationQueue {
+		if "insert_refs" == op.action && op.upsertTree.ID == tree.ID {
+			operationQueue[i] = newOp
+			return
 		}
-		updateBoxHash(tx, box.(string))
 	}
-	CommitTx(tx)
-	elapsed = time.Now().Sub(start).Milliseconds()
-	if 1000 < elapsed {
-		logging.LogInfof("hash tx [%dms]", elapsed)
+	operationQueue = append(operationQueue, newOp)
+}
+
+func DeleteBoxRefsQueue(boxID string) {
+	dbQueueLock.Lock()
+	defer dbQueueLock.Unlock()
+
+	newOp := &dbQueueOperation{box: boxID, inQueueTime: time.Now(), action: "delete_box_refs"}
+	for i, op := range operationQueue {
+		if "delete_box_refs" == op.action && op.box == boxID {
+			operationQueue[i] = newOp
+			return
+		}
 	}
+	operationQueue = append(operationQueue, newOp)
 }
 
-func mergeUpsertTrees() (ops []*treeQueueOperation) {
-	upsertTreeQueueLock.Lock()
-	defer upsertTreeQueueLock.Unlock()
+func DeleteBoxQueue(boxID string) {
+	dbQueueLock.Lock()
+	defer dbQueueLock.Unlock()
 
-	ops = operationQueue
-	operationQueue = nil
-	return
+	newOp := &dbQueueOperation{box: boxID, inQueueTime: time.Now(), action: "delete_box"}
+	for i, op := range operationQueue {
+		if "delete_box" == op.action && op.box == boxID {
+			operationQueue[i] = newOp
+			return
+		}
+	}
+	operationQueue = append(operationQueue, newOp)
 }
 
 func UpsertTreeQueue(tree *parse.Tree) {
-	upsertTreeQueueLock.Lock()
-	defer upsertTreeQueueLock.Unlock()
+	dbQueueLock.Lock()
+	defer dbQueueLock.Unlock()
 
-	newOp := &treeQueueOperation{upsertTree: tree, inQueueTime: time.Now(), action: "upsert"}
+	newOp := &dbQueueOperation{upsertTree: tree, inQueueTime: time.Now(), action: "upsert"}
 	for i, op := range operationQueue {
 		if "upsert" == op.action && op.upsertTree.ID == tree.ID { // 相同树则覆盖
 			operationQueue[i] = newOp
@@ -179,10 +216,10 @@ func UpsertTreeQueue(tree *parse.Tree) {
 }
 
 func RenameTreeQueue(tree *parse.Tree, oldHPath string) {
-	upsertTreeQueueLock.Lock()
-	defer upsertTreeQueueLock.Unlock()
+	dbQueueLock.Lock()
+	defer dbQueueLock.Unlock()
 
-	newOp := &treeQueueOperation{
+	newOp := &dbQueueOperation{
 		renameTree:         tree,
 		renameTreeOldHPath: oldHPath,
 		inQueueTime:        time.Now(),
@@ -197,10 +234,10 @@ func RenameTreeQueue(tree *parse.Tree, oldHPath string) {
 }
 
 func RemoveTreeQueue(box, rootID string) {
-	upsertTreeQueueLock.Lock()
-	defer upsertTreeQueueLock.Unlock()
+	dbQueueLock.Lock()
+	defer dbQueueLock.Unlock()
 
-	var tmp []*treeQueueOperation
+	var tmp []*dbQueueOperation
 	// 将已有的 upsert 操作去重
 	for _, op := range operationQueue {
 		if "upsert" == op.action && op.upsertTree.ID != rootID {
@@ -209,15 +246,15 @@ func RemoveTreeQueue(box, rootID string) {
 	}
 	operationQueue = tmp
 
-	newOp := &treeQueueOperation{removeTreeIDBox: box, removeTreeID: rootID, inQueueTime: time.Now(), action: "delete_id"}
+	newOp := &dbQueueOperation{removeTreeIDBox: box, removeTreeID: rootID, inQueueTime: time.Now(), action: "delete_id"}
 	operationQueue = append(operationQueue, newOp)
 }
 
 func RemoveTreePathQueue(treeBox, treePathPrefix string) {
-	upsertTreeQueueLock.Lock()
-	defer upsertTreeQueueLock.Unlock()
+	dbQueueLock.Lock()
+	defer dbQueueLock.Unlock()
 
-	var tmp []*treeQueueOperation
+	var tmp []*dbQueueOperation
 	// 将已有的 upsert 操作去重
 	for _, op := range operationQueue {
 		if "upsert" == op.action && (op.removeTreeBox != treeBox || op.upsertTree.Path != treePathPrefix) {
@@ -226,31 +263,6 @@ func RemoveTreePathQueue(treeBox, treePathPrefix string) {
 	}
 	operationQueue = tmp
 
-	newOp := &treeQueueOperation{removeTreeBox: treeBox, removeTreePath: treePathPrefix, inQueueTime: time.Now(), action: "delete"}
+	newOp := &dbQueueOperation{removeTreeBox: treeBox, removeTreePath: treePathPrefix, inQueueTime: time.Now(), action: "delete"}
 	operationQueue = append(operationQueue, newOp)
 }
-
-func updateBoxHash(tx *sql.Tx, boxID string) {
-	sum := boxChecksum(boxID)
-	PutBoxHash(tx, boxID, sum)
-}
-
-func boxChecksum(box string) (ret string) {
-	rows, err := query("SELECT hash FROM blocks WHERE type = 'd' AND box = ? ORDER BY id DESC", box)
-	if nil != err {
-		logging.LogErrorf("sql query failed: %s", err)
-		return
-	}
-	defer rows.Close()
-	buf := bytes.Buffer{}
-	for rows.Next() {
-		var hash string
-		if err = rows.Scan(&hash); nil != err {
-			logging.LogErrorf("query scan field failed: %s", err)
-			return
-		}
-		buf.WriteString(hash)
-	}
-	ret = fmt.Sprintf("%x", sha256.Sum256(buf.Bytes()))
-	return
-}

kernel/sql/stat.go

@@ -54,27 +54,6 @@ func setDatabaseVer() {
 	CommitTx(tx)
 }
 
-func ClearBoxHash(tx *sql.Tx) {
-	stmt := "DELETE FROM stat WHERE `key` LIKE '%_hash'"
-	execStmtTx(tx, stmt)
-}
-
-func RemoveBoxHash(tx *sql.Tx, box string) {
-	key := box + "_hash"
-	stmt := "DELETE FROM stat WHERE `key` = '" + key + "'"
-	execStmtTx(tx, stmt)
-}
-
-func PutBoxHash(tx *sql.Tx, box, hash string) {
-	key := box + "_hash"
-	putStat(tx, key, hash)
-}
-
-func GetBoxHash(box string) string {
-	key := box + "_hash"
-	return getStat(key)
-}
-
 func putStat(tx *sql.Tx, key, value string) (err error) {
 	stmt := "DELETE FROM stat WHERE `key` = '" + key + "'"
 	if err = execStmtTx(tx, stmt); nil != err {

kernel/sql/upsert.go

@@ -36,13 +36,6 @@ func init() {
 	luteEngine.RenderOptions.KramdownBlockIAL = false // 数据库 markdown 字段为标准 md，但是要保留 span block ial
 }
 
-func InsertBlocksSpans(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) {
-	if err = insertBlocksSpans(tx, tree, context); nil != err {
-		logging.LogErrorf("insert tree [%s] into database failed: %s", tree.Box+tree.Path, err)
-	}
-	return
-}
-
 func InsertRefs(tx *sql.Tx, tree *parse.Tree) {
 	if err := insertRef(tx, tree); nil != err {
 		logging.LogErrorf("insert refs tree [%s] into database failed: %s", tree.Box+tree.Path, err)
@@ -395,23 +388,6 @@ func insertFileAnnotationRefs0(tx *sql.Tx, bulk []*FileAnnotationRef) (err error
 	return
 }
 
-func insertBlocksSpans(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) {
-	blocks, spans, assets, attributes := fromTree(tree.Root, tree)
-	if err = insertBlocks(tx, blocks, context); nil != err {
-		return
-	}
-	if err = insertSpans(tx, spans); nil != err {
-		return
-	}
-	if err = insertAssets(tx, assets); nil != err {
-		return
-	}
-	if err = insertAttributes(tx, attributes); nil != err {
-		return
-	}
-	return
-}
-
 func insertRef(tx *sql.Tx, tree *parse.Tree) (err error) {
 	refs, fileAnnotationRefs := refsFromTree(tree)
 	if err = insertRefs(tx, refs); nil != err {

kernel/task/queue.go

@@ -102,6 +102,7 @@ const (
 	RepoCheckout            = "task.repo.checkout"             // 从快照中检出
 	DatabaseIndexFull       = "task.database.index.full"       // 重建索引
 	DatabaseIndex           = "task.database.index"            // 数据库索引队列
+	DatabaseIndexRef        = "task.database.index.ref"        // 数据库索引引用
 	DatabaseIndexFix        = "task.database.index.fix"        // 数据库索引订正
 	OCRImage                = "task.ocr.image"                 // 图片 OCR 提取文本
 	HistoryGenerateDoc      = "task.history.generateDoc"       // 生成文件历史

kernel/treenode/blocktree.go

@@ -320,28 +320,6 @@ func ReindexBlockTree(tree *parse.Tree) {
 	blockTreesChanged = true
 }
 
-func IndexBlockTree(tree *parse.Tree) {
-	blockTreesLock.Lock()
-	defer blockTreesLock.Unlock()
-
-	ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
-		if !entering || !n.IsBlock() {
-			return ast.WalkContinue
-		}
-		var parentID string
-		if nil != n.Parent {
-			parentID = n.Parent.ID
-		}
-		if "" == n.ID {
-			return ast.WalkContinue
-		}
-		blockTrees[n.ID] = &BlockTree{ID: n.ID, ParentID: parentID, RootID: tree.ID, BoxID: tree.Box, Path: tree.Path, HPath: tree.HPath, Updated: tree.Root.IALAttr("updated")}
-		return ast.WalkContinue
-	})
-
-	// 新建索引不变更持久化文件，调用处会负责调用 SaveBlockTree()
-}
-
 func AutoFlushBlockTree() {
 	for {
 		SaveBlockTree(false)