:art: 历史数据库索引队列化 https://github.com/siyuan-note/siyuan/issues/7386

app/appearance/langs/en_US.json

@@ -885,6 +885,7 @@
     "task.database.index.fix": "Execute database index fix",
     "task.ocr.image": "Execute image OCR to extract text",
     "task.history.generateDoc": "Execute GenerateDoc History",
+    "task.history.database.index.commit": "Execute history database index commit",
     "task.database.index.embedBlock": "Execute database index embed block",
     "task.reload.ui": "Execute reload UI"
   },
@@ -1089,6 +1090,7 @@
     "187": "Unlocking cloud sync directory",
     "188": "Failed to lock the cloud sync directory, please try again later",
     "189": "Cloud sync directory is still locked by other devices, please try again later",
-    "190": "A problem was found while validating the index, which has been automatically fixed"
+    "190": "A problem was found while validating the index, which has been automatically fixed",
+    "191": "[%d/%d] Created historical data index"
   }
 }

app/appearance/langs/es_ES.json

@@ -885,6 +885,7 @@
     "task.database.index.fix": "Ejecutar corrección del índice de la base de datos",
     "task.ocr.image": "Ejecutar OCR de imagen para extraer texto",
     "task.history.generateDoc": "Ejecutar Historial GenerateDoc",
+    "task.history.database.index.commit": "Ejecutar la confirmación del índice de la base de datos del historial",
     "task.database.index.embedBlock": "Ejecutar bloque de incrustación de índice de base de datos",
     "task.reload.ui": "IU de recarga de tareas"
   },
@@ -1089,6 +1090,7 @@
     "187": "Desbloqueando el directorio de sincronización en la nube",
     "188": "Error al bloquear el directorio de sincronizaci\u00f3n en la nube, int\u00e1ntelo de nuevo m\u00e1s tarde",
     "189": "El directorio de sincronización en la nube todavía está bloqueado por otros dispositivos, inténtalo de nuevo más tarde",
-    "190": "Se encontro un problema al validar el indice, el cual se soluciono automaticamente"
+    "190": "Se encontro un problema al validar el indice, el cual se soluciono automaticamente",
+    "191": "[%d/%d] Índice de datos históricos creado"
   }
 }

app/appearance/langs/fr_FR.json

@@ -885,6 +885,7 @@
     "task.database.index.fix": "Effectuer la correction de l'index de la base de données",
     "task.ocr.image": "Exécute l'OCR d'image pour extraire le texte",
     "task.history.generateDoc": "Exécuter l'historique de GenerateDoc",
+    "task.history.database.index.commit": "Effectuer la validation de l'index de la base de données d'historique",
     "task.database.index.embedBlock": "Exécuter le bloc d'intégration d'index de base de données",
     "task.reload.ui": "Interface utilisateur de rechargement de tâche"
   },
@@ -1089,6 +1090,7 @@
     "187": "Déverrouillage du répertoire de synchronisation cloud",
     "188": "Échec du verrouillage du répertoire de synchronisation cloud, veuillez réessayer plus tard",
     "189": "Le répertoire de synchronisation cloud est toujours verrouillé par d'autres appareils, veuillez réessayer plus tard",
-    "190": "Un problème a été trouvé lors de la validation de l'index, qui a été automatiquement corrigé"
+    "190": "Un problème a été trouvé lors de la validation de l'index, qui a été automatiquement corrigé",
+    "191": "[%d/%d] Création d'un index de données historiques"
   }
 }

app/appearance/langs/zh_CHT.json

@@ -885,6 +885,7 @@
     "task.database.index.fix": "執行數據庫索引訂正",
     "task.ocr.image": "執行圖片 OCR 提取文本",
     "task.history.generateDoc": "執行生成文件歷史",
+    "task.history.database.index.commit": "執行歷史數據庫索引提交",
     "task.database.index.embedBlock": "執行數據庫索引嵌入塊",
     "task.reload.ui": "執行重載界面"
   },
@@ -1089,6 +1090,7 @@
     "187": "正在解鎖雲端同步目錄",
     "188": "鎖定雲端同步目錄失敗，請稍後再試",
     "189": "雲端同步目錄還在被其他設備鎖定，請稍後再試",
-    "190": "校驗索引時發現一個問題，已經自動修復"
+    "190": "校驗索引時發現一個問題，已經自動修復",
+    "191": "[%d/%d] 已经建立条历史数据索引"
   }
 }

app/appearance/langs/zh_CN.json

@@ -885,6 +885,7 @@
     "task.database.index.fix": "执行数据库索引订正",
     "task.ocr.image": "执行图片 OCR 提取文本",
     "task.history.generateDoc": "执行生成文件历史",
+    "task.history.database.index.commit": "执行历史数据库索引提交",
     "task.database.index.embedBlock": "执行数据库索引嵌入块",
     "task.reload.ui": "执行重载界面"
   },
@@ -1089,6 +1090,7 @@
     "187": "正在解锁云端同步目录",
     "188": "锁定云端同步目录失败，请稍后再试",
     "189": "云端同步目录还在被其他设备锁定，请稍后再试",
-    "190": "校验索引时发现一个问题，已经自动修复"
+    "190": "校验索引时发现一个问题，已经自动修复",
+    "191": "[%d/%d] 已经建立条历史数据索引"
   }
 }

kernel/go.mod

@@ -42,7 +42,7 @@ require (
 	github.com/shirou/gopsutil/v3 v3.23.1
 	github.com/siyuan-note/dejavu v0.0.0-20230212031819-32964d704bd2
 	github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75
-	github.com/siyuan-note/eventbus v0.0.0-20230203085647-fb624740be03
+	github.com/siyuan-note/eventbus v0.0.0-20230216103454-41885eac6c2b
 	github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e
 	github.com/siyuan-note/httpclient v0.0.0-20230211023949-b9d36c2da3ea
 	github.com/siyuan-note/logging v0.0.0-20221031125421-9b7234d79d8a

kernel/go.sum

@@ -272,6 +272,10 @@ github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 h1:Bi7/7f29
 github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw=
 github.com/siyuan-note/eventbus v0.0.0-20230203085647-fb624740be03 h1:W7nGGluE6sBrFSVkmucGsh2NruleOPsQle7fcAW115o=
 github.com/siyuan-note/eventbus v0.0.0-20230203085647-fb624740be03/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI=
+github.com/siyuan-note/eventbus v0.0.0-20230216101534-15f8e2f2fb12 h1:DjZa4jP3J+cZK9BuCXXzY4kr37QXXaCX+IHt8JP8UXQ=
+github.com/siyuan-note/eventbus v0.0.0-20230216101534-15f8e2f2fb12/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI=
+github.com/siyuan-note/eventbus v0.0.0-20230216103454-41885eac6c2b h1:828lTUW2C0uNiolODqoACu7J8sDUzswD4Xo04mUombg=
+github.com/siyuan-note/eventbus v0.0.0-20230216103454-41885eac6c2b/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI=
 github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e h1:i3RKrdrddr4AuaHJtoWYAEVNuR7Y9wIsEqPmuFFbJC4=
 github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e/go.mod h1:NmpSIVtIGy8eNWapjDIiiCw5+5r5wxC76k40oG+WRXQ=
 github.com/siyuan-note/httpclient v0.0.0-20230211023949-b9d36c2da3ea h1:Q6cuN3L4zWR+MwQVdMKeUnfusd+W0LLu1KeqSB3vfdQ=

kernel/job/cron.go

@@ -37,6 +37,7 @@ func StartCron() {
 	go every(3*time.Second, model.FlushUpdateRefTextRenameDocJob)
 	go every(50*time.Millisecond, model.FlushTxJob)
 	go every(util.SQLFlushInterval, sql.FlushTxJob)
+	go every(util.SQLFlushInterval, sql.FlushHistoryTxJob)
 	go every(10*time.Minute, model.FixIndexJob)
 	go every(10*time.Minute, model.IndexEmbedBlockJob)
 	go every(10*time.Minute, model.CacheVirtualBlockRefJob)

kernel/model/history.go

@@ -70,15 +70,12 @@ func generateDocHistory() {
 	clearOutdatedHistoryDir(historyDir)
 
 	// 以下部分是老版本的历史数据，不再保留
-
 	for _, box := range Conf.GetBoxes() {
 		historyDir = filepath.Join(util.DataDir, box.ID, ".siyuan", "history")
 		os.RemoveAll(historyDir)
 	}
-
 	historyDir = filepath.Join(util.DataDir, "assets", ".siyuan", "history")
 	os.RemoveAll(historyDir)
-
 	historyDir = filepath.Join(util.DataDir, ".siyuan", "history")
 	os.RemoveAll(historyDir)
 }
@@ -517,23 +514,7 @@ func clearOutdatedHistoryDir(historyDir string) {
 		//logging.LogInfof("auto removed history dir [%s]", dir)
 
 		// 清理历史库
-
-		tx, txErr := sql.BeginHistoryTx()
-		if nil != txErr {
-			logging.LogErrorf("begin history tx failed: %s", txErr)
-			return
-		}
-
-		p := strings.TrimPrefix(dir, util.HistoryDir)
-		p = filepath.ToSlash(p[1:])
-		if txErr = sql.DeleteHistoriesByPathPrefix(tx, dir); nil != txErr {
-			logging.LogErrorf("delete history [%s] failed: %s", dir, txErr)
-			return
-		}
-		if txErr = sql.CommitHistoryTx(tx); nil != txErr {
-			logging.LogErrorf("commit history tx failed: %s", txErr)
-			return
-		}
+		sql.DeleteHistoriesByPathPrefixQueue(dir)
 	}
 }
 
@@ -678,19 +659,7 @@ func indexHistoryDir(name string, luteEngine *lute.Lute) {
 		})
 	}
 
-	tx, txErr := sql.BeginHistoryTx()
-	if nil != txErr {
-		logging.LogErrorf("begin transaction failed: %s", txErr)
-		return
-	}
-	if err := sql.InsertHistories(tx, histories); nil != err {
-		logging.LogErrorf("insert histories failed: %s", err)
-		return
-	}
-	if err := sql.CommitHistoryTx(tx); nil != err {
-		logging.LogErrorf("commit transaction failed: %s", err)
-		return
-	}
+	sql.IndexHistoriesQueue(histories)
 	return
 }
 

kernel/model/index.go

@@ -286,4 +286,16 @@ func init() {
 		util.SetBootDetails(msg)
 		util.ContextPushMsg(context, msg)
 	})
+
+	eventbus.Subscribe(eventbus.EvtSQLInsertHistory, func(context map[string]interface{}) {
+		if util.ContainerAndroid == util.Container || util.ContainerIOS == util.Container {
+			return
+		}
+
+		current := context["current"].(int) + 1
+		total := context["total"]
+		msg := fmt.Sprintf(Conf.Language(191), current, total)
+		util.SetBootDetails(msg)
+		util.ContextPushMsg(context, msg)
+	})
 }

kernel/sql/database.go

@@ -1142,7 +1142,7 @@ func beginTx() (tx *sql.Tx, err error) {
 	return
 }
 
-func BeginHistoryTx() (tx *sql.Tx, err error) {
+func beginHistoryTx() (tx *sql.Tx, err error) {
 	if tx, err = historyDB.Begin(); nil != err {
 		logging.LogErrorf("begin history tx failed: %s\n  %s", err, logging.ShortStack())
 		if strings.Contains(err.Error(), "database is locked") {
@@ -1152,7 +1152,7 @@ func BeginHistoryTx() (tx *sql.Tx, err error) {
 	return
 }
 
-func CommitHistoryTx(tx *sql.Tx) (err error) {
+func commitHistoryTx(tx *sql.Tx) (err error) {
 	if nil == tx {
 		logging.LogErrorf("tx is nil")
 		return errors.New("tx is nil")

kernel/sql/history.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/siyuan-note/eventbus"
 	"github.com/siyuan-note/logging"
 )
 
@@ -102,7 +103,7 @@ func queryHistory(query string, args ...interface{}) (*sql.Rows, error) {
 	return historyDB.Query(query, args...)
 }
 
-func DeleteHistoriesByPathPrefix(tx *sql.Tx, pathPrefix string) (err error) {
+func deleteHistoriesByPathPrefix(tx *sql.Tx, pathPrefix string, context map[string]interface{}) (err error) {
 	stmt := "DELETE FROM histories_fts_case_insensitive WHERE path LIKE ?"
 	if err = execStmtTx(tx, stmt, pathPrefix+"%"); nil != err {
 		return
@@ -115,7 +116,7 @@ const (
 	HistoriesPlaceholder              = "(?, ?, ?, ?, ?, ?)"
 )
 
-func InsertHistories(tx *sql.Tx, histories []*History) (err error) {
+func insertHistories(tx *sql.Tx, histories []*History, context map[string]interface{}) (err error) {
 	if 1 > len(histories) {
 		return
 	}
@@ -127,20 +128,20 @@ func InsertHistories(tx *sql.Tx, histories []*History) (err error) {
 			continue
 		}
 
-		if err = insertHistories0(tx, bulk); nil != err {
+		if err = insertHistories0(tx, bulk, context); nil != err {
 			return
 		}
 		bulk = []*History{}
 	}
 	if 0 < len(bulk) {
-		if err = insertHistories0(tx, bulk); nil != err {
+		if err = insertHistories0(tx, bulk, context); nil != err {
 			return
 		}
 	}
 	return
 }
 
-func insertHistories0(tx *sql.Tx, bulk []*History) (err error) {
+func insertHistories0(tx *sql.Tx, bulk []*History, context map[string]interface{}) (err error) {
 	valueStrings := make([]string, 0, len(bulk))
 	valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(HistoriesPlaceholder, "?"))
 	for _, b := range bulk {
@@ -157,5 +158,7 @@ func insertHistories0(tx *sql.Tx, bulk []*History) (err error) {
 	if err = prepareExecInsertTx(tx, stmt, valueArgs); nil != err {
 		return
 	}
+
+	eventbus.Publish(eventbus.EvtSQLInsertHistory, context)
 	return
 }

kernel/sql/queue_history.go

@@ -0,0 +1,139 @@
+// SiYuan - Build Your Eternal Digital Garden
+// Copyright (c) 2020-present, b3log.org
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package sql
+
+import (
+	"database/sql"
+	"errors"
+	"fmt"
+	"runtime/debug"
+	"sync"
+	"time"
+
+	"github.com/siyuan-note/eventbus"
+	"github.com/siyuan-note/logging"
+	"github.com/siyuan-note/siyuan/kernel/task"
+	"github.com/siyuan-note/siyuan/kernel/util"
+)
+
+var (
+	historyOperationQueue []*historyDBQueueOperation
+	historyDBQueueLock    = sync.Mutex{}
+
+	historyTxLock = sync.Mutex{}
+)
+
+type historyDBQueueOperation struct {
+	inQueueTime time.Time
+	action      string // index/deletePathPrefix
+
+	histories  []*History // index
+	pathPrefix string     // deletePathPrefix
+}
+
+func FlushHistoryTxJob() {
+	task.AppendTask(task.HistoryDatabaseIndexCommit, FlushHistoryQueue)
+}
+
+func FlushHistoryQueue() {
+	ops := getHistoryOperations()
+	if 1 > len(ops) {
+		return
+	}
+
+	txLock.Lock()
+	defer txLock.Unlock()
+	start := time.Now()
+
+	context := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar}
+	total := len(ops)
+	for i, op := range ops {
+		if util.IsExiting {
+			return
+		}
+
+		tx, err := beginHistoryTx()
+		if nil != err {
+			return
+		}
+
+		context["current"] = i
+		context["total"] = total
+		if err = execHistoryOp(op, tx, context); nil != err {
+			tx.Rollback()
+			logging.LogErrorf("queue operation failed: %s", err)
+			continue
+		}
+
+		if err = commitHistoryTx(tx); nil != err {
+			logging.LogErrorf("commit tx failed: %s", err)
+			return
+		}
+
+		if 16 < i && 0 == i%128 {
+			debug.FreeOSMemory()
+		}
+	}
+
+	if 128 < len(ops) {
+		debug.FreeOSMemory()
+	}
+
+	elapsed := time.Now().Sub(start).Milliseconds()
+	if 7000 < elapsed {
+		logging.LogInfof("database history op tx [%dms]", elapsed)
+	}
+}
+
+func execHistoryOp(op *historyDBQueueOperation, tx *sql.Tx, context map[string]interface{}) (err error) {
+	switch op.action {
+	case "index":
+		err = insertHistories(tx, op.histories, context)
+	case "deletePathPrefix":
+		err = deleteHistoriesByPathPrefix(tx, op.pathPrefix, context)
+	default:
+		msg := fmt.Sprintf("unknown history operation [%s]", op.action)
+		logging.LogErrorf(msg)
+		err = errors.New(msg)
+	}
+	return
+}
+
+func DeleteHistoriesByPathPrefixQueue(pathPrefix string) {
+	historyDBQueueLock.Lock()
+	defer historyDBQueueLock.Unlock()
+
+	newOp := &historyDBQueueOperation{inQueueTime: time.Now(), action: "deletePathPrefix", pathPrefix: pathPrefix}
+	historyOperationQueue = append(historyOperationQueue, newOp)
+}
+
+func IndexHistoriesQueue(histories []*History) {
+	historyDBQueueLock.Lock()
+	defer historyDBQueueLock.Unlock()
+
+	newOp := &historyDBQueueOperation{inQueueTime: time.Now(), action: "index", histories: histories}
+	historyOperationQueue = append(historyOperationQueue, newOp)
+}
+
+func getHistoryOperations() (ops []*historyDBQueueOperation) {
+	historyDBQueueLock.Lock()
+	defer historyDBQueueLock.Unlock()
+
+	ops = historyOperationQueue
+	historyOperationQueue = nil
+	return
+}

kernel/task/queue.go

@@ -82,16 +82,17 @@ func getCurrentActions() (ret []string) {
 }
 
 const (
-	RepoCheckout            = "task.repo.checkout"             // 从快照中检出
-	DatabaseIndexFull       = "task.database.index.full"       // 重建索引
-	DatabaseIndex           = "task.database.index"            // 数据库索引
-	DatabaseIndexCommit     = "task.database.index.commit"     // 数据库索引提交
-	DatabaseIndexRef        = "task.database.index.ref"        // 数据库索引引用
-	DatabaseIndexFix        = "task.database.index.fix"        // 数据库索引订正
-	OCRImage                = "task.ocr.image"                 // 图片 OCR 提取文本
-	HistoryGenerateDoc      = "task.history.generateDoc"       // 生成文件历史
-	DatabaseIndexEmbedBlock = "task.database.index.embedBlock" // 数据库索引嵌入块
-	ReloadUI                = "task.reload.ui"                 // 重载 UI
+	RepoCheckout               = "task.repo.checkout"                 // 从快照中检出
+	DatabaseIndexFull          = "task.database.index.full"           // 重建索引
+	DatabaseIndex              = "task.database.index"                // 数据库索引
+	DatabaseIndexCommit        = "task.database.index.commit"         // 数据库索引提交
+	DatabaseIndexRef           = "task.database.index.ref"            // 数据库索引引用
+	DatabaseIndexFix           = "task.database.index.fix"            // 数据库索引订正
+	OCRImage                   = "task.ocr.image"                     // 图片 OCR 提取文本
+	HistoryGenerateDoc         = "task.history.generateDoc"           // 生成文件历史
+	HistoryDatabaseIndexCommit = "task.history.database.index.commit" // 历史数据库索引提交
+	DatabaseIndexEmbedBlock    = "task.database.index.embedBlock"     // 数据库索引嵌入块
+	ReloadUI                   = "task.reload.ui"                     // 重载 UI
 )
 
 // uniqueActions 描述了唯一的任务，即队列中只能存在一个在执行的任务。
@@ -101,6 +102,7 @@ var uniqueActions = []string{
 	DatabaseIndexCommit,
 	OCRImage,
 	HistoryGenerateDoc,
+	HistoryDatabaseIndexCommit,
 	DatabaseIndexEmbedBlock,
 }