0ct0pu5/siyuan
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

✨ Support for searching asset content https://github.com/siyuan-note/siyuan/issues/8874

Browse source
This commit is contained in:
Daniel 2023-08-04 12:05:29 +08:00
parent 1aa4049750
commit 7d992ce175
No known key found for this signature in database
GPG key ID: 86211BA83DF03017
21 changed files with 568 additions and 76 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
app/appearance/langs/en_US.json
View file

@ -1260,6 +1260,8 @@
"212": "There are some defects in the current version of cloud data sync, please upgrade to the latest version. Sorry for the inconvenience",
"213": "Cloud verification failed, please try to upgrade to the latest version and log in again before syncing",
"214": "This function needs to be signed in to use",
"215": "Save failed: The target file is being used by another program"
"215": "Save failed: The target file is being used by another program",
"216": "Rebuilding asset content data index, please wait...",
"217": "[%d/%d] Created asset content data index"
}
}

4
app/appearance/langs/es_ES.json
View file

@ -1260,6 +1260,8 @@
"212": "Hay algunos defectos en la versi\u00f3n actual de sincronizaci\u00f3n de datos en la nube, actualice a la versi\u00f3n m\u00e1s reciente. Disculpe las molestias",
"213": "La verificaci\u00f3n en la nube fall\u00f3, intente actualizar a la versi\u00f3n m\u00e1s reciente e inicie sesi\u00f3n de nuevo antes de sincronizar",
"214": "Esta función requiere iniciar sesión en la cuenta antes de poder usarla",
"215": "Error al guardar: el archivo de destino está siendo utilizado por otro programa"
"215": "Error al guardar: el archivo de destino está siendo utilizado por otro programa",
"216": "Reconstruyendo el índice de datos de contenido de recursos, espere...",
"217": "[%d/%d] Índice de datos de contenido de activos creado"
}
}

4
app/appearance/langs/fr_FR.json
View file

@ -1260,6 +1260,8 @@
"212": "Il y a quelques défauts dans la version actuelle de la synchronisation des données cloud, veuillez mettre à niveau vers la dernière version. Désolé pour le désagrément",
"213": "Échec de la vérification cloud, veuillez essayer de mettre à niveau vers la dernière version et de vous reconnecter avant de synchroniser",
"214": "La fonctionnalité nécessite un numéro de compte de connexion avant de pouvoir être utilisée",
"215": "Échec de l'enregistrement : le fichier de destination est utilisé par un autre programme"
"215": "Échec de l'enregistrement : le fichier de destination est utilisé par un autre programme",
"216": "Reconstruction de l'index des données du contenu des ressources, veuillez patienter...",
"217": "[%d/%d] Création d'un index de données de contenu d'actif"
}
}

4
app/appearance/langs/zh_CHT.json
View file

@ -1260,6 +1260,8 @@
"212": "當前版本雲端數據同步存在一些缺陷，請升級到最新版，帶來不便，敬請諒解",
"213": "雲端校驗失敗，請嘗試升級到最新版並重新登錄後再進行同步",
"214": "該功能需要登錄賬號後才能使用",
"215": "保存失敗：目標文件正在被其他程序佔用"
"215": "保存失敗：目標文件正在被其他程序佔用",
"216": "正在重建資源文件內容數據索引，請稍等...",
"217": "[%d/%d] 已經建立條資源文件內容數據索引"
}
}

4
app/appearance/langs/zh_CN.json
View file

@ -1260,6 +1260,8 @@
"212": "当前版本云端数据同步存在一些缺陷，请升级到最新版，带来不便，敬请谅解",
"213": "云端校验失败，请尝试升级到最新版并重新登录后再进行同步",
"214": "该功能需要登录账号后才能使用",
"215": "保存失败：目标文件并且正在被其他程序占用"
"215": "保存失败：目标文件并且正在被其他程序占用",
"216": "正在重建资源文件内容数据索引，请稍等...",
"217": "[%d/%d] 已经建立条资源文件内容数据索引"
}
}

2
kernel/cache/asset.go vendored
View file

@ -79,7 +79,7 @@ func LoadAssets() {
assetsCache[path] = &Asset{
HName: hName,
Path: path,
Updated: info.ModTime().UnixMilli(),
Updated: info.ModTime().Unix(),
}
return nil
})

2
kernel/go.mod
View file

@ -47,7 +47,7 @@ require (
github.com/shirou/gopsutil/v3 v3.23.6
github.com/siyuan-note/dejavu v0.0.0-20230801123133-2edc24064c33
github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75
github.com/siyuan-note/eventbus v0.0.0-20230702081350-6dde667e7112
github.com/siyuan-note/eventbus v0.0.0-20230804030110-cf250f838c80
github.com/siyuan-note/filelock v0.0.0-20230615140405-d05a21d49524
github.com/siyuan-note/httpclient v0.0.0-20230728124841-53922bac2be2
github.com/siyuan-note/logging v0.0.0-20230327073243-ebe83aec1493

2
kernel/go.sum
View file

@ -296,6 +296,8 @@ github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 h1:Bi7/7f29
github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw=
github.com/siyuan-note/eventbus v0.0.0-20230702081350-6dde667e7112 h1:lb+8C+XEEEn/lcBtoXlrf5mZEoe0y0KlqiIGG93Gozc=
github.com/siyuan-note/eventbus v0.0.0-20230702081350-6dde667e7112/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI=
github.com/siyuan-note/eventbus v0.0.0-20230804030110-cf250f838c80 h1:XghjHKJd+SiL0DkGYFVC+UGUDFtnR4v9gkAbPeh9Eq8=
github.com/siyuan-note/eventbus v0.0.0-20230804030110-cf250f838c80/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI=
github.com/siyuan-note/filelock v0.0.0-20230615140405-d05a21d49524 h1:ZuxN5gwqtUOd1NkOkNhM4OlVWfjujY98zsR+zFi4x9g=
github.com/siyuan-note/filelock v0.0.0-20230615140405-d05a21d49524/go.mod h1:jK5lCYfPbFOrW23/HMeU7kmpLdEd5GkennF+kUpy7Vs=
github.com/siyuan-note/httpclient v0.0.0-20230728124841-53922bac2be2 h1:z6vYbmEOVoytf30Ny6YDjyZTYdCPmazeAl4BN67+308=

1
kernel/job/cron.go
View file

@ -38,6 +38,7 @@ func StartCron() {
go every(50*time.Millisecond, model.FlushTxJob)
go every(util.SQLFlushInterval, sql.FlushTxJob)
go every(util.SQLFlushInterval, sql.FlushHistoryTxJob)
go every(util.SQLFlushInterval, sql.FlushAssetContentTxJob)
go every(10*time.Minute, model.FixIndexJob)
go every(10*time.Minute, model.IndexEmbedBlockJob)
go every(10*time.Minute, model.CacheVirtualBlockRefJob)

1
kernel/main.go
View file

@ -35,6 +35,7 @@ func main() {
model.InitAppearance()
sql.InitDatabase(false)
sql.InitHistoryDatabase(false)
sql.InitAssetContentDatabase(false)
sql.SetCaseSensitive(model.Conf.Search.CaseSensitive)
sql.SetIndexAssetPath(model.Conf.Search.IndexAssetPath)

1
kernel/mobile/kernel.go
View file

@ -49,6 +49,7 @@ func StartKernel(container, appDir, workspaceBaseDir, timezoneID, localIPs, lang
model.InitAppearance()
sql.InitDatabase(false)
sql.InitHistoryDatabase(false)
sql.InitAssetContentDatabase(false)
sql.SetCaseSensitive(model.Conf.Search.CaseSensitive)
sql.SetIndexAssetPath(model.Conf.Search.IndexAssetPath)

155
kernel/model/asset_content.go Normal file
View file

@ -0,0 +1,155 @@
// SiYuan - Refactor your thinking
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package model
import (
"github.com/88250/gulu"
"github.com/88250/lute/ast"
"github.com/siyuan-note/eventbus"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/util"
"io/fs"
"path/filepath"
"strings"
)
func ReindexAssetContent() {
task.AppendTask(task.AssetContentDatabaseIndexFull, fullReindexAssetContent)
return
}
func fullReindexAssetContent() {
util.PushMsg(Conf.Language(216), 7*1000)
sql.InitAssetContentDatabase(true)
assetsSearch := NewAssetsSearcher()
assetsSearch.Index()
return
}
func init() {
subscribeSQLAssetContentEvents()
}
func subscribeSQLAssetContentEvents() {
eventbus.Subscribe(util.EvtSQLAssetContentRebuild, func() {
ReindexAssetContent()
})
}
var (
AssetsSearchEnabled = true
)
type AssetsSearcher struct {
AssetsDir string
Parsers map[string]AssetParser
}
func (searcher *AssetsSearcher) Index() {
assetsDir := searcher.AssetsDir
if !gulu.File.IsDir(assetsDir) {
return
}
var results []*AssetParseResult
filepath.Walk(assetsDir, func(absPath string, info fs.FileInfo, err error) error {
if nil != err {
logging.LogErrorf("walk dir [%s] failed: %s", absPath, err)
return err
}
if info.IsDir() {
return nil
}
ext := strings.ToLower(filepath.Ext(absPath))
parser, found := searcher.Parsers[ext]
if !found {
return nil
}
result := parser.Parse(absPath)
if nil == result {
return nil
}
result.Path = "assets" + filepath.ToSlash(strings.TrimPrefix(absPath, assetsDir))
result.Size = info.Size()
result.Updated = info.ModTime().Unix()
results = append(results, result)
return nil
})
var assetContents []*sql.AssetContent
for _, result := range results {
assetContents = append(assetContents, &sql.AssetContent{
ID: ast.NewNodeID(),
Name: filepath.Base(result.Path),
Ext: filepath.Ext(result.Path),
Path: result.Path,
Size: result.Size,
Updated: result.Updated,
Content: result.Content,
})
}
sql.IndexAssetContentsQueue(assetContents)
}
func NewAssetsSearcher() *AssetsSearcher {
return &AssetsSearcher{
AssetsDir: util.GetDataAssetsAbsPath(),
Parsers: map[string]AssetParser{
".txt": &TxtAssetParser{},
},
}
}
type AssetParseResult struct {
Path string
Size int64
Updated int64
Content string
}
type AssetParser interface {
Parse(absPath string) *AssetParseResult
}
type TxtAssetParser struct {
}
func (parser *TxtAssetParser) Parse(absPath string) (ret *AssetParseResult) {
if !strings.HasSuffix(strings.ToLower(absPath), ".txt") {
return
}
data, err := filelock.ReadFile(absPath)
if nil != err {
logging.LogErrorf("read file [%s] failed: %s", absPath, err)
return
}
ret = &AssetParseResult{
Content: string(data),
}
return
}

12
kernel/model/index.go
View file

@ -303,4 +303,16 @@ func subscribeSQLEvents() {
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
eventbus.Subscribe(eventbus.EvtSQLInsertAssetContent, func(context map[string]interface{}) {
if util.ContainerAndroid == util.Container || util.ContainerIOS == util.Container {
return
}
current := context["current"].(int)
total := context["total"]
msg := fmt.Sprintf(Conf.Language(217), current, total)
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
}

96
kernel/sql/asset_content.go Normal file
View file

@ -0,0 +1,96 @@
// SiYuan - Refactor your thinking
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package sql
import (
"database/sql"
"fmt"
"strings"
"github.com/siyuan-note/eventbus"
)
type AssetContent struct {
ID string
Name string
Ext string
Path string
Size int64
Updated int64
Content string
}
const (
AssetContentsFTSCaseInsensitiveInsert = "INSERT INTO asset_contents_fts_case_insensitive (id, name, ext, path, size, updated, content) VALUES %s"
AssetContentsPlaceholder = "(?, ?, ?, ?, ?, ?, ?)"
)
func insertAssetContents(tx *sql.Tx, assetContents []*AssetContent, context map[string]interface{}) (err error) {
if 1 > len(assetContents) {
return
}
var bulk []*AssetContent
for _, assetContent := range assetContents {
bulk = append(bulk, assetContent)
if 512 > len(bulk) {
continue
}
if err = insertAssetContents0(tx, bulk, context); nil != err {
return
}
bulk = []*AssetContent{}
}
if 0 < len(bulk) {
if err = insertAssetContents0(tx, bulk, context); nil != err {
return
}
}
return
}
func insertAssetContents0(tx *sql.Tx, bulk []*AssetContent, context map[string]interface{}) (err error) {
valueStrings := make([]string, 0, len(bulk))
valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(AssetContentsPlaceholder, "?"))
for _, b := range bulk {
valueStrings = append(valueStrings, AssetContentsPlaceholder)
valueArgs = append(valueArgs, b.ID)
valueArgs = append(valueArgs, b.Name)
valueArgs = append(valueArgs, b.Ext)
valueArgs = append(valueArgs, b.Path)
valueArgs = append(valueArgs, b.Size)
valueArgs = append(valueArgs, b.Updated)
valueArgs = append(valueArgs, b.Content)
}
stmt := fmt.Sprintf(AssetContentsFTSCaseInsensitiveInsert, strings.Join(valueStrings, ","))
if err = prepareExecInsertTx(tx, stmt, valueArgs); nil != err {
return
}
eventbus.Publish(eventbus.EvtSQLInsertAssetContent, context)
return
}
func deleteAssetContentsByPath(tx *sql.Tx, path string, context map[string]interface{}) (err error) {
stmt := "DELETE FROM asset_contents_fts_case_insensitive WHERE path = ?"
if err = execStmtTx(tx, stmt, path); nil != err {
return
}
return
}

109
kernel/sql/database.go
View file

@ -45,8 +45,9 @@ import (
)
var (
db *sql.DB
historyDB *sql.DB
db *sql.DB
historyDB *sql.DB
assetContentDB *sql.DB
)
func init() {
@ -193,7 +194,36 @@ func initDBTables() {
}
}
func initDBConnection() {
if nil != db {
closeDatabase()
}
dsn := util.DBPath + "?_journal_mode=WAL" +
"&_synchronous=OFF" +
"&_mmap_size=2684354560" +
"&_secure_delete=OFF" +
"&_cache_size=-20480" +
"&_page_size=32768" +
"&_busy_timeout=7000" +
"&_ignore_check_constraints=ON" +
"&_temp_store=MEMORY" +
"&_case_sensitive_like=OFF"
var err error
db, err = sql.Open("sqlite3_extended", dsn)
if nil != err {
logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create database failed: %s", err)
}
db.SetMaxIdleConns(20)
db.SetMaxOpenConns(20)
db.SetConnMaxLifetime(365 * 24 * time.Hour)
}
var initHistoryDatabaseLock = sync.Mutex{}
func InitHistoryDatabase(forceRebuild bool) {
initHistoryDatabaseLock.Lock()
defer initHistoryDatabaseLock.Unlock()
initHistoryDBConnection()
if !forceRebuild && gulu.File.IsExist(util.HistoryDBPath) {
@ -228,7 +258,7 @@ func initHistoryDBConnection() {
var err error
historyDB, err = sql.Open("sqlite3_extended", dsn)
if nil != err {
logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create database failed: %s", err)
logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create history database failed: %s", err)
}
historyDB.SetMaxIdleConns(3)
historyDB.SetMaxOpenConns(3)
@ -243,11 +273,34 @@ func initHistoryDBTables() {
}
}
func initDBConnection() {
if nil != db {
closeDatabase()
var initAssetContentDatabaseLock = sync.Mutex{}
func InitAssetContentDatabase(forceRebuild bool) {
initAssetContentDatabaseLock.Lock()
defer initAssetContentDatabaseLock.Unlock()
initAssetContentDBConnection()
if !forceRebuild && gulu.File.IsExist(util.AssetContentDBPath) {
return
}
dsn := util.DBPath + "?_journal_mode=WAL" +
assetContentDB.Close()
if err := os.RemoveAll(util.AssetContentDBPath); nil != err {
logging.LogErrorf("remove assets database file [%s] failed: %s", util.AssetContentDBPath, err)
return
}
initAssetContentDBConnection()
initAssetContentDBTables()
}
func initAssetContentDBConnection() {
if nil != assetContentDB {
assetContentDB.Close()
}
dsn := util.AssetContentDBPath + "?_journal_mode=WAL" +
"&_synchronous=OFF" +
"&_mmap_size=2684354560" +
"&_secure_delete=OFF" +
@ -258,13 +311,21 @@ func initDBConnection() {
"&_temp_store=MEMORY" +
"&_case_sensitive_like=OFF"
var err error
db, err = sql.Open("sqlite3_extended", dsn)
assetContentDB, err = sql.Open("sqlite3_extended", dsn)
if nil != err {
logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create database failed: %s", err)
logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create assets database failed: %s", err)
}
assetContentDB.SetMaxIdleConns(3)
assetContentDB.SetMaxOpenConns(3)
assetContentDB.SetConnMaxLifetime(365 * 24 * time.Hour)
}
func initAssetContentDBTables() {
assetContentDB.Exec("DROP TABLE asset_contents_fts_case_insensitive")
_, err := assetContentDB.Exec("CREATE VIRTUAL TABLE asset_contents_fts_case_insensitive USING fts5(id UNINDEXED, name, ext, path, size UNINDEXED, updated UNINDEXED, content, tokenize=\"siyuan case_insensitive\")")
if nil != err {
logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create table [asset_contents_fts_case_insensitive] failed: %s", err)
}
db.SetMaxIdleConns(20)
db.SetMaxOpenConns(20)
db.SetConnMaxLifetime(365 * 24 * time.Hour)
}
var (
@ -1161,6 +1222,18 @@ func beginTx() (tx *sql.Tx, err error) {
return
}
func commitTx(tx *sql.Tx) (err error) {
if nil == tx {
logging.LogErrorf("tx is nil")
return errors.New("tx is nil")
}
if err = tx.Commit(); nil != err {
logging.LogErrorf("commit tx failed: %s\n %s", err, logging.ShortStack())
}
return
}
func beginHistoryTx() (tx *sql.Tx, err error) {
if tx, err = historyDB.Begin(); nil != err {
logging.LogErrorf("begin history tx failed: %s\n %s", err, logging.ShortStack())
@ -1183,7 +1256,17 @@ func commitHistoryTx(tx *sql.Tx) (err error) {
return
}
func commitTx(tx *sql.Tx) (err error) {
func beginAssetContentTx() (tx *sql.Tx, err error) {
if tx, err = assetContentDB.Begin(); nil != err {
logging.LogErrorf("begin asset content tx failed: %s\n %s", err, logging.ShortStack())
if strings.Contains(err.Error(), "database is locked") {
os.Exit(logging.ExitCodeReadOnlyDatabase)
}
}
return
}
func commitAssetContentTx(tx *sql.Tx) (err error) {
if nil == tx {
logging.LogErrorf("tx is nil")
return errors.New("tx is nil")

147
kernel/sql/queue_asset_content.go Normal file
View file

@ -0,0 +1,147 @@
// SiYuan - Refactor your thinking
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package sql
import (
"database/sql"
"errors"
"fmt"
"runtime/debug"
"sync"
"time"
"github.com/siyuan-note/eventbus"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/util"
)
var (
assetContentOperationQueue []*assetContentDBQueueOperation
assetContentDBQueueLock = sync.Mutex{}
assetContentTxLock = sync.Mutex{}
)
type assetContentDBQueueOperation struct {
inQueueTime time.Time
action string // index/deletePath
assetContents []*AssetContent // index
path string // deletePath
}
func FlushAssetContentTxJob() {
task.AppendTask(task.AssetContentDatabaseIndexCommit, FlushAssetContentQueue)
}
func FlushAssetContentQueue() {
ops := getAssetContentOperations()
if 1 > len(ops) {
return
}
assetContentTxLock.Lock()
defer assetContentTxLock.Unlock()
start := time.Now()
groupOpsTotal := map[string]int{}
for _, op := range ops {
groupOpsTotal[op.action]++
}
context := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar}
groupOpsCurrent := map[string]int{}
for i, op := range ops {
if util.IsExiting {
return
}
tx, err := beginAssetContentTx()
if nil != err {
return
}
groupOpsCurrent[op.action]++
context["current"] = groupOpsCurrent[op.action]
context["total"] = groupOpsTotal[op.action]
if err = execAssetContentOp(op, tx, context); nil != err {
tx.Rollback()
logging.LogErrorf("queue operation failed: %s", err)
eventbus.Publish(util.EvtSQLAssetContentRebuild)
return
}
if err = commitAssetContentTx(tx); nil != err {
logging.LogErrorf("commit tx failed: %s", err)
return
}
if 16 < i && 0 == i%128 {
debug.FreeOSMemory()
}
}
if 128 < len(ops) {
debug.FreeOSMemory()
}
elapsed := time.Now().Sub(start).Milliseconds()
if 7000 < elapsed {
logging.LogInfof("database asset content op tx [%dms]", elapsed)
}
}
func execAssetContentOp(op *assetContentDBQueueOperation, tx *sql.Tx, context map[string]interface{}) (err error) {
switch op.action {
case "index":
err = insertAssetContents(tx, op.assetContents, context)
case "delete":
err = deleteAssetContentsByPath(tx, op.path, context)
default:
msg := fmt.Sprintf("unknown asset content operation [%s]", op.action)
logging.LogErrorf(msg)
err = errors.New(msg)
}
return
}
func DeleteAssetContentsByPathQueue(path string) {
assetContentTxLock.Lock()
defer assetContentTxLock.Unlock()
newOp := &assetContentDBQueueOperation{inQueueTime: time.Now(), action: "deletePath", path: path}
assetContentOperationQueue = append(assetContentOperationQueue, newOp)
}
func IndexAssetContentsQueue(assetContents []*AssetContent) {
assetContentTxLock.Lock()
defer assetContentTxLock.Unlock()
newOp := &assetContentDBQueueOperation{inQueueTime: time.Now(), action: "index", assetContents: assetContents}
assetContentOperationQueue = append(assetContentOperationQueue, newOp)
}
func getAssetContentOperations() (ops []*assetContentDBQueueOperation) {
assetContentTxLock.Lock()
defer assetContentTxLock.Unlock()
ops = assetContentOperationQueue
assetContentOperationQueue = nil
return
}

28
kernel/sql/queue_history.go
View file

@ -55,8 +55,8 @@ func FlushHistoryQueue() {
return
}
txLock.Lock()
defer txLock.Unlock()
historyTxLock.Lock()
defer historyTxLock.Unlock()
start := time.Now()
groupOpsTotal := map[string]int{}
@ -145,27 +145,3 @@ func getHistoryOperations() (ops []*historyDBQueueOperation) {
historyOperationQueue = nil
return
}
func WaitForWritingHistoryDatabase() {
var printLog bool
var lastPrintLog bool
for i := 0; isWritingHistoryDatabase(); i++ {
time.Sleep(50 * time.Millisecond)
if 200 < i && !printLog { // 10s 后打日志
logging.LogWarnf("history database is writing: \n%s", logging.ShortStack())
printLog = true
}
if 1200 < i && !lastPrintLog { // 60s 后打日志
logging.LogWarnf("history database is still writing")
lastPrintLog = true
}
}
}
func isWritingHistoryDatabase() bool {
time.Sleep(util.SQLFlushInterval + 50*time.Millisecond)
if 0 < len(historyOperationQueue) || util.IsMutexLocked(&historyTxLock) {
return true
}
return false
}

30
kernel/task/queue.go
View file

@ -82,19 +82,21 @@ func getCurrentActions() (ret []string) {
}
const (
RepoCheckout = "task.repo.checkout" // 从快照中检出
DatabaseIndexFull = "task.database.index.full" // 重建索引
DatabaseIndex = "task.database.index" // 数据库索引
DatabaseIndexCommit = "task.database.index.commit" // 数据库索引提交
DatabaseIndexRef = "task.database.index.ref" // 数据库索引引用
DatabaseIndexFix = "task.database.index.fix" // 数据库索引订正
OCRImage = "task.ocr.image" // 图片 OCR 提取文本
HistoryGenerateDoc = "task.history.generateDoc" // 生成文件历史
HistoryDatabaseIndexFull = "task.history.database.index.full" // 历史数据库重建索引
HistoryDatabaseIndexCommit = "task.history.database.index.commit" // 历史数据库索引提交
DatabaseIndexEmbedBlock = "task.database.index.embedBlock" // 数据库索引嵌入块
ReloadUI = "task.reload.ui" // 重载 UI
UpgradeUserGuide = "task.upgrade.userGuide" // 升级用户指南文档笔记本
RepoCheckout = "task.repo.checkout" // 从快照中检出
DatabaseIndexFull = "task.database.index.full" // 重建索引
DatabaseIndex = "task.database.index" // 数据库索引
DatabaseIndexCommit = "task.database.index.commit" // 数据库索引提交
DatabaseIndexRef = "task.database.index.ref" // 数据库索引引用
DatabaseIndexFix = "task.database.index.fix" // 数据库索引订正
OCRImage = "task.ocr.image" // 图片 OCR 提取文本
HistoryGenerateDoc = "task.history.generateDoc" // 生成文件历史
HistoryDatabaseIndexFull = "task.history.database.index.full" // 历史数据库重建索引
HistoryDatabaseIndexCommit = "task.history.database.index.commit" // 历史数据库索引提交
DatabaseIndexEmbedBlock = "task.database.index.embedBlock" // 数据库索引嵌入块
ReloadUI = "task.reload.ui" // 重载 UI
UpgradeUserGuide = "task.upgrade.userGuide" // 升级用户指南文档笔记本
AssetContentDatabaseIndexFull = "task.asset.database.index.full" // 资源文件数据库重建索引
AssetContentDatabaseIndexCommit = "task.asset.database.index.commit" // 资源文件数据库索引提交
)
// uniqueActions 描述了唯一的任务，即队列中只能存在一个在执行的任务。
@ -107,6 +109,8 @@ var uniqueActions = []string{
HistoryDatabaseIndexFull,
HistoryDatabaseIndexCommit,
DatabaseIndexEmbedBlock,
AssetContentDatabaseIndexFull,
AssetContentDatabaseIndexCommit,
}
func Contain(action string, moreActions ...string) bool {

3
kernel/util/runtime.go
View file

@ -408,5 +408,6 @@ func existAvailabilityStatus(workspaceAbsPath string) bool {
const (
EvtConfPandocInitialized = "conf.pandoc.initialized"
EvtSQLHistoryRebuild = "sql.history.rebuild"
EvtSQLHistoryRebuild = "sql.history.rebuild"
EvtSQLAssetContentRebuild = "sql.assetContent.rebuild"
)

34
kernel/util/working.go
View file

@ -158,22 +158,23 @@ var (
HomeDir, _ = gulu.OS.Home()
WorkingDir, _ = os.Getwd()
WorkspaceDir string // 工作空间目录路径
WorkspaceLock *flock.Flock // 工作空间锁
ConfDir string // 配置目录路径
DataDir string // 数据目录路径
RepoDir string // 仓库目录路径
HistoryDir string // 数据历史目录路径
TempDir string // 临时目录路径
LogPath string // 配置目录下的日志文件 siyuan.log 路径
DBName = "siyuan.db" // SQLite 数据库文件名
DBPath string // SQLite 数据库文件路径
HistoryDBPath string // SQLite 历史数据库文件路径
BlockTreePath string // 区块树文件路径
AppearancePath string // 配置目录下的外观目录 appearance/ 路径
ThemesPath string // 配置目录下的外观目录下的 themes/ 路径
IconsPath string // 配置目录下的外观目录下的 icons/ 路径
SnippetsPath string // 数据目录下的 snippets/ 路径
WorkspaceDir string // 工作空间目录路径
WorkspaceLock *flock.Flock // 工作空间锁
ConfDir string // 配置目录路径
DataDir string // 数据目录路径
RepoDir string // 仓库目录路径
HistoryDir string // 数据历史目录路径
TempDir string // 临时目录路径
LogPath string // 配置目录下的日志文件 siyuan.log 路径
DBName = "siyuan.db" // SQLite 数据库文件名
DBPath string // SQLite 数据库文件路径
HistoryDBPath string // SQLite 历史数据库文件路径
AssetContentDBPath string // SQLite 资源文件内容数据库文件路径
BlockTreePath string // 区块树文件路径
AppearancePath string // 配置目录下的外观目录 appearance/ 路径
ThemesPath string // 配置目录下的外观目录下的 themes/ 路径
IconsPath string // 配置目录下的外观目录下的 icons/ 路径
SnippetsPath string // 数据目录下的 snippets/ 路径
UIProcessIDs = sync.Map{} // UI 进程 ID
)
@ -247,6 +248,7 @@ func initWorkspaceDir(workspaceArg string) {
os.Setenv("TMP", osTmpDir)
DBPath = filepath.Join(TempDir, DBName)
HistoryDBPath = filepath.Join(TempDir, "history.db")
AssetContentDBPath = filepath.Join(TempDir, "asset_content.db")
BlockTreePath = filepath.Join(TempDir, "blocktree")
SnippetsPath = filepath.Join(DataDir, "snippets")
}

1
kernel/util/working_mobile.go
View file

@ -159,6 +159,7 @@ func initWorkspaceDirMobile(workspaceBaseDir string) {
os.Setenv("TMP", osTmpDir)
DBPath = filepath.Join(TempDir, DBName)
HistoryDBPath = filepath.Join(TempDir, "history.db")
AssetContentDBPath = filepath.Join(TempDir, "asset_content.db")
BlockTreePath = filepath.Join(TempDir, "blocktree")
SnippetsPath = filepath.Join(DataDir, "snippets")