0ct0pu5/siyuan
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

🎨 历史数据库索引队列化 https://github.com/siyuan-note/siyuan/issues/7386

Browse source
This commit is contained in:
Liang Ding 2023-02-16 18:42:19 +08:00
parent 4d8b826e29
commit f581b34ea8
No known key found for this signature in database
GPG key ID: 136F30F901A2231D
14 changed files with 196 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
app/appearance/langs/en_US.json
View file

@ -885,6 +885,7 @@
"task.database.index.fix": "Execute database index fix",
"task.ocr.image": "Execute image OCR to extract text",
"task.history.generateDoc": "Execute GenerateDoc History",
"task.history.database.index.commit": "Execute history database index commit",
"task.database.index.embedBlock": "Execute database index embed block",
"task.reload.ui": "Execute reload UI"
},
@ -1089,6 +1090,7 @@
"187": "Unlocking cloud sync directory",
"188": "Failed to lock the cloud sync directory, please try again later",
"189": "Cloud sync directory is still locked by other devices, please try again later",
"190": "A problem was found while validating the index, which has been automatically fixed"
"190": "A problem was found while validating the index, which has been automatically fixed",
"191": "[%d/%d] Created historical data index"
}
}

4
app/appearance/langs/es_ES.json
View file

@ -885,6 +885,7 @@
"task.database.index.fix": "Ejecutar corrección del índice de la base de datos",
"task.ocr.image": "Ejecutar OCR de imagen para extraer texto",
"task.history.generateDoc": "Ejecutar Historial GenerateDoc",
"task.history.database.index.commit": "Ejecutar la confirmación del índice de la base de datos del historial",
"task.database.index.embedBlock": "Ejecutar bloque de incrustación de índice de base de datos",
"task.reload.ui": "IU de recarga de tareas"
},
@ -1089,6 +1090,7 @@
"187": "Desbloqueando el directorio de sincronización en la nube",
"188": "Error al bloquear el directorio de sincronizaci\u00f3n en la nube, int\u00e1ntelo de nuevo m\u00e1s tarde",
"189": "El directorio de sincronización en la nube todavía está bloqueado por otros dispositivos, inténtalo de nuevo más tarde",
"190": "Se encontro un problema al validar el indice, el cual se soluciono automaticamente"
"190": "Se encontro un problema al validar el indice, el cual se soluciono automaticamente",
"191": "[%d/%d] Índice de datos históricos creado"
}
}

4
app/appearance/langs/fr_FR.json
View file

@ -885,6 +885,7 @@
"task.database.index.fix": "Effectuer la correction de l'index de la base de données",
"task.ocr.image": "Exécute l'OCR d'image pour extraire le texte",
"task.history.generateDoc": "Exécuter l'historique de GenerateDoc",
"task.history.database.index.commit": "Effectuer la validation de l'index de la base de données d'historique",
"task.database.index.embedBlock": "Exécuter le bloc d'intégration d'index de base de données",
"task.reload.ui": "Interface utilisateur de rechargement de tâche"
},
@ -1089,6 +1090,7 @@
"187": "Déverrouillage du répertoire de synchronisation cloud",
"188": "Échec du verrouillage du répertoire de synchronisation cloud, veuillez réessayer plus tard",
"189": "Le répertoire de synchronisation cloud est toujours verrouillé par d'autres appareils, veuillez réessayer plus tard",
"190": "Un problème a été trouvé lors de la validation de l'index, qui a été automatiquement corrigé"
"190": "Un problème a été trouvé lors de la validation de l'index, qui a été automatiquement corrigé",
"191": "[%d/%d] Création d'un index de données historiques"
}
}

4
app/appearance/langs/zh_CHT.json
View file

@ -885,6 +885,7 @@
"task.database.index.fix": "執行數據庫索引訂正",
"task.ocr.image": "執行圖片 OCR 提取文本",
"task.history.generateDoc": "執行生成文件歷史",
"task.history.database.index.commit": "執行歷史數據庫索引提交",
"task.database.index.embedBlock": "執行數據庫索引嵌入塊",
"task.reload.ui": "執行重載界面"
},
@ -1089,6 +1090,7 @@
"187": "正在解鎖雲端同步目錄",
"188": "鎖定雲端同步目錄失敗，請稍後再試",
"189": "雲端同步目錄還在被其他設備鎖定，請稍後再試",
"190": "校驗索引時發現一個問題，已經自動修復"
"190": "校驗索引時發現一個問題，已經自動修復",
"191": "[%d/%d] 已经建立条历史数据索引"
}
}

4
app/appearance/langs/zh_CN.json
View file

@ -885,6 +885,7 @@
"task.database.index.fix": "执行数据库索引订正",
"task.ocr.image": "执行图片 OCR 提取文本",
"task.history.generateDoc": "执行生成文件历史",
"task.history.database.index.commit": "执行历史数据库索引提交",
"task.database.index.embedBlock": "执行数据库索引嵌入块",
"task.reload.ui": "执行重载界面"
},
@ -1089,6 +1090,7 @@
"187": "正在解锁云端同步目录",
"188": "锁定云端同步目录失败，请稍后再试",
"189": "云端同步目录还在被其他设备锁定，请稍后再试",
"190": "校验索引时发现一个问题，已经自动修复"
"190": "校验索引时发现一个问题，已经自动修复",
"191": "[%d/%d] 已经建立条历史数据索引"
}
}

2
kernel/go.mod
View file

@ -42,7 +42,7 @@ require (
github.com/shirou/gopsutil/v3 v3.23.1
github.com/siyuan-note/dejavu v0.0.0-20230212031819-32964d704bd2
github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75
github.com/siyuan-note/eventbus v0.0.0-20230203085647-fb624740be03
github.com/siyuan-note/eventbus v0.0.0-20230216103454-41885eac6c2b
github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e
github.com/siyuan-note/httpclient v0.0.0-20230211023949-b9d36c2da3ea
github.com/siyuan-note/logging v0.0.0-20221031125421-9b7234d79d8a

4
kernel/go.sum
View file

@ -272,6 +272,10 @@ github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 h1:Bi7/7f29
github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw=
github.com/siyuan-note/eventbus v0.0.0-20230203085647-fb624740be03 h1:W7nGGluE6sBrFSVkmucGsh2NruleOPsQle7fcAW115o=
github.com/siyuan-note/eventbus v0.0.0-20230203085647-fb624740be03/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI=
github.com/siyuan-note/eventbus v0.0.0-20230216101534-15f8e2f2fb12 h1:DjZa4jP3J+cZK9BuCXXzY4kr37QXXaCX+IHt8JP8UXQ=
github.com/siyuan-note/eventbus v0.0.0-20230216101534-15f8e2f2fb12/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI=
github.com/siyuan-note/eventbus v0.0.0-20230216103454-41885eac6c2b h1:828lTUW2C0uNiolODqoACu7J8sDUzswD4Xo04mUombg=
github.com/siyuan-note/eventbus v0.0.0-20230216103454-41885eac6c2b/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI=
github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e h1:i3RKrdrddr4AuaHJtoWYAEVNuR7Y9wIsEqPmuFFbJC4=
github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e/go.mod h1:NmpSIVtIGy8eNWapjDIiiCw5+5r5wxC76k40oG+WRXQ=
github.com/siyuan-note/httpclient v0.0.0-20230211023949-b9d36c2da3ea h1:Q6cuN3L4zWR+MwQVdMKeUnfusd+W0LLu1KeqSB3vfdQ=

1
kernel/job/cron.go
View file

@ -37,6 +37,7 @@ func StartCron() {
go every(3*time.Second, model.FlushUpdateRefTextRenameDocJob)
go every(50*time.Millisecond, model.FlushTxJob)
go every(util.SQLFlushInterval, sql.FlushTxJob)
go every(util.SQLFlushInterval, sql.FlushHistoryTxJob)
go every(10*time.Minute, model.FixIndexJob)
go every(10*time.Minute, model.IndexEmbedBlockJob)
go every(10*time.Minute, model.CacheVirtualBlockRefJob)

35
kernel/model/history.go
View file

@ -70,15 +70,12 @@ func generateDocHistory() {
clearOutdatedHistoryDir(historyDir)
// 以下部分是老版本的历史数据，不再保留
for _, box := range Conf.GetBoxes() {
historyDir = filepath.Join(util.DataDir, box.ID, ".siyuan", "history")
os.RemoveAll(historyDir)
}
historyDir = filepath.Join(util.DataDir, "assets", ".siyuan", "history")
os.RemoveAll(historyDir)
historyDir = filepath.Join(util.DataDir, ".siyuan", "history")
os.RemoveAll(historyDir)
}
@ -517,23 +514,7 @@ func clearOutdatedHistoryDir(historyDir string) {
//logging.LogInfof("auto removed history dir [%s]", dir)
// 清理历史库
tx, txErr := sql.BeginHistoryTx()
if nil != txErr {
logging.LogErrorf("begin history tx failed: %s", txErr)
return
}
p := strings.TrimPrefix(dir, util.HistoryDir)
p = filepath.ToSlash(p[1:])
if txErr = sql.DeleteHistoriesByPathPrefix(tx, dir); nil != txErr {
logging.LogErrorf("delete history [%s] failed: %s", dir, txErr)
return
}
if txErr = sql.CommitHistoryTx(tx); nil != txErr {
logging.LogErrorf("commit history tx failed: %s", txErr)
return
}
sql.DeleteHistoriesByPathPrefixQueue(dir)
}
}
@ -678,19 +659,7 @@ func indexHistoryDir(name string, luteEngine *lute.Lute) {
})
}
tx, txErr := sql.BeginHistoryTx()
if nil != txErr {
logging.LogErrorf("begin transaction failed: %s", txErr)
return
}
if err := sql.InsertHistories(tx, histories); nil != err {
logging.LogErrorf("insert histories failed: %s", err)
return
}
if err := sql.CommitHistoryTx(tx); nil != err {
logging.LogErrorf("commit transaction failed: %s", err)
return
}
sql.IndexHistoriesQueue(histories)
return
}

12
kernel/model/index.go
View file

@ -286,4 +286,16 @@ func init() {
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
eventbus.Subscribe(eventbus.EvtSQLInsertHistory, func(context map[string]interface{}) {
if util.ContainerAndroid == util.Container || util.ContainerIOS == util.Container {
return
}
current := context["current"].(int) + 1
total := context["total"]
msg := fmt.Sprintf(Conf.Language(191), current, total)
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
}

4
kernel/sql/database.go
View file

@ -1142,7 +1142,7 @@ func beginTx() (tx *sql.Tx, err error) {
return
}
func BeginHistoryTx() (tx *sql.Tx, err error) {
func beginHistoryTx() (tx *sql.Tx, err error) {
if tx, err = historyDB.Begin(); nil != err {
logging.LogErrorf("begin history tx failed: %s\n %s", err, logging.ShortStack())
if strings.Contains(err.Error(), "database is locked") {
@ -1152,7 +1152,7 @@ func BeginHistoryTx() (tx *sql.Tx, err error) {
return
}
func CommitHistoryTx(tx *sql.Tx) (err error) {
func commitHistoryTx(tx *sql.Tx) (err error) {
if nil == tx {
logging.LogErrorf("tx is nil")
return errors.New("tx is nil")

13
kernel/sql/history.go
View file

@ -22,6 +22,7 @@ import (
"fmt"
"strings"
"github.com/siyuan-note/eventbus"
"github.com/siyuan-note/logging"
)
@ -102,7 +103,7 @@ func queryHistory(query string, args ...interface{}) (*sql.Rows, error) {
return historyDB.Query(query, args...)
}
func DeleteHistoriesByPathPrefix(tx *sql.Tx, pathPrefix string) (err error) {
func deleteHistoriesByPathPrefix(tx *sql.Tx, pathPrefix string, context map[string]interface{}) (err error) {
stmt := "DELETE FROM histories_fts_case_insensitive WHERE path LIKE ?"
if err = execStmtTx(tx, stmt, pathPrefix+"%"); nil != err {
return
@ -115,7 +116,7 @@ const (
HistoriesPlaceholder = "(?, ?, ?, ?, ?, ?)"
)
func InsertHistories(tx *sql.Tx, histories []*History) (err error) {
func insertHistories(tx *sql.Tx, histories []*History, context map[string]interface{}) (err error) {
if 1 > len(histories) {
return
}
@ -127,20 +128,20 @@ func InsertHistories(tx *sql.Tx, histories []*History) (err error) {
continue
}
if err = insertHistories0(tx, bulk); nil != err {
if err = insertHistories0(tx, bulk, context); nil != err {
return
}
bulk = []*History{}
}
if 0 < len(bulk) {
if err = insertHistories0(tx, bulk); nil != err {
if err = insertHistories0(tx, bulk, context); nil != err {
return
}
}
return
}
func insertHistories0(tx *sql.Tx, bulk []*History) (err error) {
func insertHistories0(tx *sql.Tx, bulk []*History, context map[string]interface{}) (err error) {
valueStrings := make([]string, 0, len(bulk))
valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(HistoriesPlaceholder, "?"))
for _, b := range bulk {
@ -157,5 +158,7 @@ func insertHistories0(tx *sql.Tx, bulk []*History) (err error) {
if err = prepareExecInsertTx(tx, stmt, valueArgs); nil != err {
return
}
eventbus.Publish(eventbus.EvtSQLInsertHistory, context)
return
}

139
kernel/sql/queue_history.go Normal file
View file

@ -0,0 +1,139 @@
// SiYuan - Build Your Eternal Digital Garden
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package sql
import (
"database/sql"
"errors"
"fmt"
"runtime/debug"
"sync"
"time"
"github.com/siyuan-note/eventbus"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/util"
)
var (
historyOperationQueue []*historyDBQueueOperation
historyDBQueueLock = sync.Mutex{}
historyTxLock = sync.Mutex{}
)
type historyDBQueueOperation struct {
inQueueTime time.Time
action string // index/deletePathPrefix
histories []*History // index
pathPrefix string // deletePathPrefix
}
func FlushHistoryTxJob() {
task.AppendTask(task.HistoryDatabaseIndexCommit, FlushHistoryQueue)
}
func FlushHistoryQueue() {
ops := getHistoryOperations()
if 1 > len(ops) {
return
}
txLock.Lock()
defer txLock.Unlock()
start := time.Now()
context := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar}
total := len(ops)
for i, op := range ops {
if util.IsExiting {
return
}
tx, err := beginHistoryTx()
if nil != err {
return
}
context["current"] = i
context["total"] = total
if err = execHistoryOp(op, tx, context); nil != err {
tx.Rollback()
logging.LogErrorf("queue operation failed: %s", err)
continue
}
if err = commitHistoryTx(tx); nil != err {
logging.LogErrorf("commit tx failed: %s", err)
return
}
if 16 < i && 0 == i%128 {
debug.FreeOSMemory()
}
}
if 128 < len(ops) {
debug.FreeOSMemory()
}
elapsed := time.Now().Sub(start).Milliseconds()
if 7000 < elapsed {
logging.LogInfof("database history op tx [%dms]", elapsed)
}
}
func execHistoryOp(op *historyDBQueueOperation, tx *sql.Tx, context map[string]interface{}) (err error) {
switch op.action {
case "index":
err = insertHistories(tx, op.histories, context)
case "deletePathPrefix":
err = deleteHistoriesByPathPrefix(tx, op.pathPrefix, context)
default:
msg := fmt.Sprintf("unknown history operation [%s]", op.action)
logging.LogErrorf(msg)
err = errors.New(msg)
}
return
}
func DeleteHistoriesByPathPrefixQueue(pathPrefix string) {
historyDBQueueLock.Lock()
defer historyDBQueueLock.Unlock()
newOp := &historyDBQueueOperation{inQueueTime: time.Now(), action: "deletePathPrefix", pathPrefix: pathPrefix}
historyOperationQueue = append(historyOperationQueue, newOp)
}
func IndexHistoriesQueue(histories []*History) {
historyDBQueueLock.Lock()
defer historyDBQueueLock.Unlock()
newOp := &historyDBQueueOperation{inQueueTime: time.Now(), action: "index", histories: histories}
historyOperationQueue = append(historyOperationQueue, newOp)
}
func getHistoryOperations() (ops []*historyDBQueueOperation) {
historyDBQueueLock.Lock()
defer historyDBQueueLock.Unlock()
ops = historyOperationQueue
historyOperationQueue = nil
return
}

22
kernel/task/queue.go
View file

@ -82,16 +82,17 @@ func getCurrentActions() (ret []string) {
}
const (
RepoCheckout = "task.repo.checkout" // 从快照中检出
DatabaseIndexFull = "task.database.index.full" // 重建索引
DatabaseIndex = "task.database.index" // 数据库索引
DatabaseIndexCommit = "task.database.index.commit" // 数据库索引提交
DatabaseIndexRef = "task.database.index.ref" // 数据库索引引用
DatabaseIndexFix = "task.database.index.fix" // 数据库索引订正
OCRImage = "task.ocr.image" // 图片 OCR 提取文本
HistoryGenerateDoc = "task.history.generateDoc" // 生成文件历史
DatabaseIndexEmbedBlock = "task.database.index.embedBlock" // 数据库索引嵌入块
ReloadUI = "task.reload.ui" // 重载 UI
RepoCheckout = "task.repo.checkout" // 从快照中检出
DatabaseIndexFull = "task.database.index.full" // 重建索引
DatabaseIndex = "task.database.index" // 数据库索引
DatabaseIndexCommit = "task.database.index.commit" // 数据库索引提交
DatabaseIndexRef = "task.database.index.ref" // 数据库索引引用
DatabaseIndexFix = "task.database.index.fix" // 数据库索引订正
OCRImage = "task.ocr.image" // 图片 OCR 提取文本
HistoryGenerateDoc = "task.history.generateDoc" // 生成文件历史
HistoryDatabaseIndexCommit = "task.history.database.index.commit" // 历史数据库索引提交
DatabaseIndexEmbedBlock = "task.database.index.embedBlock" // 数据库索引嵌入块
ReloadUI = "task.reload.ui" // 重载 UI
)
// uniqueActions 描述了唯一的任务，即队列中只能存在一个在执行的任务。
@ -101,6 +102,7 @@ var uniqueActions = []string{
DatabaseIndexCommit,
OCRImage,
HistoryGenerateDoc,
HistoryDatabaseIndexCommit,
DatabaseIndexEmbedBlock,
}