0ct0pu5/siyuan
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

🎨 改进内核任务调度机制提升稳定性 https://github.com/siyuan-note/siyuan/issues/7113

Browse source
This commit is contained in:
Liang Ding 2023-01-23 18:30:52 +08:00
parent a83a08bdb5
commit 72093f7e2e
No known key found for this signature in database
GPG key ID: 136F30F901A2231D
7 changed files with 103 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
kernel/model/backlink.go
View file

@ -40,24 +40,19 @@ import (
func RefreshBacklink(id string) {
WaitForWritingFiles()
tx, err := sql.BeginTx()
if nil != err {
return
}
defer sql.CommitTx(tx)
refs := sql.QueryRefsByDefID(id, false)
trees := map[string]*parse.Tree{}
for _, ref := range refs {
tree := trees[ref.RootID]
if nil == tree {
tree, err = loadTreeByBlockID(ref.RootID)
if nil != err {
logging.LogErrorf("refresh tree refs failed: %s", err)
var loadErr error
tree, loadErr = loadTreeByBlockID(ref.RootID)
if nil != loadErr {
logging.LogErrorf("refresh tree refs failed: %s", loadErr)
continue
}
trees[ref.RootID] = tree
sql.UpsertRefs(tx, tree)
sql.UpdateRefsTreeQueue(tree)
}
}
}

2
kernel/model/box.go
View file

@ -482,6 +482,7 @@ func genTreeID(tree *parse.Tree) {
func FullReindex() {
task.PrependTask(task.DatabaseIndexFull, fullReindex)
task.AppendTask(task.DatabaseIndexRef, IndexRefs)
}
func fullReindex() {
@ -501,7 +502,6 @@ func fullReindex() {
for _, openedBox := range openedBoxes {
index(openedBox.ID)
}
IndexRefs()
treenode.SaveBlockTree(true)
LoadFlashcards()

6
kernel/model/file.go
View file

@ -1459,12 +1459,6 @@ func createDoc(boxID, p, title, dom string) (err error) {
transaction := &Transaction{DoOperations: []*Operation{{Action: "create", Data: tree}}}
err = PerformTransactions(&[]*Transaction{transaction})
if nil != err {
tx, txErr := sql.BeginTx()
if nil != txErr {
logging.LogFatalf("transaction failed: %s", txErr)
return
}
sql.CommitTx(tx)
logging.LogFatalf("transaction failed: %s", err)
return
}

27
kernel/model/index.go
View file

@ -39,18 +39,14 @@ func (box *Box) Unindex() {
}
func unindex(boxID string) {
tx, err := sql.BeginTx()
if nil != err {
return
}
sql.DeleteByBoxTx(tx, boxID)
sql.CommitTx(tx)
ids := treenode.RemoveBlockTreesByBoxID(boxID)
RemoveRecentDoc(ids)
sql.DeleteBoxQueue(boxID)
}
func (box *Box) Index() {
task.PrependTask(task.DatabaseIndex, index, box.ID)
task.AppendTask(task.DatabaseIndexRef, IndexRefs)
}
func index(boxID string) {
@ -94,12 +90,12 @@ func index(boxID string) {
}
cache.PutDocIAL(file.path, docIAL)
treenode.IndexBlockTree(tree)
sql.UpsertTreeQueue(tree)
util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(92), util.ShortPathForBootingDisplay(tree.Path)))
treeSize += file.size
treeCount++
treenode.IndexBlockTree(tree)
if 1 < i && 0 == i%64 {
util.PushEndlessProgress(fmt.Sprintf(Conf.Language(88), i, len(files)-i))
}
@ -128,16 +124,12 @@ func IndexRefs() {
for _, refBlock := range refBlocks {
refTreeIDs.Add(refBlock.RootID)
}
if 0 < refTreeIDs.Size() {
luteEngine := NewLute()
bootProgressPart := 10.0 / float64(refTreeIDs.Size())
for _, box := range Conf.GetOpenedBoxes() {
tx, err := sql.BeginTx()
if nil != err {
return
}
sql.DeleteRefsByBoxTx(tx, box.ID)
sql.CommitTx(tx)
sql.DeleteBoxRefsQueue(box.ID)
files := box.ListFiles("/")
i := 0
@ -163,14 +155,7 @@ func IndexRefs() {
continue
}
tx, err = sql.BeginTx()
if nil != err {
continue
}
sql.InsertRefs(tx, tree)
if err = sql.CommitTx(tx); nil != err {
continue
}
sql.InsertRefsTreeQueue(tree)
if 1 < i && 0 == i%64 {
util.PushEndlessProgress(fmt.Sprintf(Conf.Language(55), i))
}

1
kernel/model/mount.go
View file

@ -196,7 +196,6 @@ func Mount(boxID string) (alreadyMount bool, err error) {
box.SaveConf(boxConf)
box.Index()
IndexRefs()
// 缓存根一级的文档树展开
ListDocTree(box.ID, "/", Conf.FileTree.Sort)
treenode.SaveBlockTree(false)

121
kernel/sql/queue.go
View file

@ -22,7 +22,6 @@ import (
"time"
"github.com/88250/lute/parse"
"github.com/emirpasic/gods/sets/hashset"
"github.com/siyuan-note/eventbus"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/task"
@ -30,19 +29,20 @@ import (
)
var (
operationQueue []*treeQueueOperation
upsertTreeQueueLock = sync.Mutex{}
operationQueue []*dbQueueOperation
dbQueueLock = sync.Mutex{}
txLock = sync.Mutex{}
)
type treeQueueOperation struct {
type dbQueueOperation struct {
inQueueTime time.Time
action string // upsert/delete/delete_id/rename
action string // upsert/delete/delete_id/rename/delete_box/delete_box_refs/insert_refs
upsertTree *parse.Tree // upsert
upsertTree *parse.Tree // upsert/insert_refs
removeTreeBox, removeTreePath string // delete
removeTreeIDBox, removeTreeID string // delete_id
box string // delete_box/delete_box_refs
renameTree *parse.Tree // rename
renameTreeOldHPath string // rename
}
@ -83,8 +83,8 @@ func IsEmptyQueue() bool {
}
func ClearQueue() {
upsertTreeQueueLock.Lock()
defer upsertTreeQueueLock.Unlock()
dbQueueLock.Lock()
defer dbQueueLock.Unlock()
operationQueue = nil
}
@ -103,7 +103,6 @@ func FlushQueue() {
}
context := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar}
boxes := hashset.New()
for _, op := range ops {
switch op.action {
case "upsert":
@ -111,17 +110,21 @@ func FlushQueue() {
if err = upsertTree(tx, tree, context); nil != err {
logging.LogErrorf("upsert tree [%s] into database failed: %s", tree.Box+tree.Path, err)
}
boxes.Add(op.upsertTree.Box)
case "delete":
batchDeleteByPathPrefix(tx, op.removeTreeBox, op.removeTreePath)
boxes.Add(op.removeTreeBox)
case "delete_id":
DeleteByRootID(tx, op.removeTreeID)
boxes.Add(op.removeTreeIDBox)
case "rename":
batchUpdateHPath(tx, op.renameTree.Box, op.renameTree.ID, op.renameTreeOldHPath, op.renameTree.HPath)
updateRootContent(tx, path.Base(op.renameTree.HPath), op.renameTree.Root.IALAttr("updated"), op.renameTree.ID)
boxes.Add(op.renameTree.Box)
case "delete_box":
DeleteByBoxTx(tx, op.box)
case "delete_box_refs":
DeleteRefsByBoxTx(tx, op.box)
case "insert_refs":
InsertRefs(tx, op.upsertTree)
case "update_refs":
UpsertRefs(tx, op.upsertTree)
default:
logging.LogErrorf("unknown operation [%s]", op.action)
}
@ -133,20 +136,76 @@ func FlushQueue() {
}
}
func mergeUpsertTrees() (ops []*treeQueueOperation) {
upsertTreeQueueLock.Lock()
defer upsertTreeQueueLock.Unlock()
func mergeUpsertTrees() (ops []*dbQueueOperation) {
dbQueueLock.Lock()
defer dbQueueLock.Unlock()
ops = operationQueue
operationQueue = nil
return
}
func UpsertTreeQueue(tree *parse.Tree) {
upsertTreeQueueLock.Lock()
defer upsertTreeQueueLock.Unlock()
func UpdateRefsTreeQueue(tree *parse.Tree) {
dbQueueLock.Lock()
defer dbQueueLock.Unlock()
newOp := &treeQueueOperation{upsertTree: tree, inQueueTime: time.Now(), action: "upsert"}
newOp := &dbQueueOperation{upsertTree: tree, inQueueTime: time.Now(), action: "update_refs"}
for i, op := range operationQueue {
if "update_refs" == op.action && op.upsertTree.ID == tree.ID {
operationQueue[i] = newOp
return
}
}
operationQueue = append(operationQueue, newOp)
}
func InsertRefsTreeQueue(tree *parse.Tree) {
dbQueueLock.Lock()
defer dbQueueLock.Unlock()
newOp := &dbQueueOperation{upsertTree: tree, inQueueTime: time.Now(), action: "insert_refs"}
for i, op := range operationQueue {
if "insert_refs" == op.action && op.upsertTree.ID == tree.ID {
operationQueue[i] = newOp
return
}
}
operationQueue = append(operationQueue, newOp)
}
func DeleteBoxRefsQueue(boxID string) {
dbQueueLock.Lock()
defer dbQueueLock.Unlock()
newOp := &dbQueueOperation{box: boxID, inQueueTime: time.Now(), action: "delete_box_refs"}
for i, op := range operationQueue {
if "delete_box_refs" == op.action && op.box == boxID {
operationQueue[i] = newOp
return
}
}
operationQueue = append(operationQueue, newOp)
}
func DeleteBoxQueue(boxID string) {
dbQueueLock.Lock()
defer dbQueueLock.Unlock()
newOp := &dbQueueOperation{box: boxID, inQueueTime: time.Now(), action: "delete_box"}
for i, op := range operationQueue {
if "delete_box" == op.action && op.box == boxID {
operationQueue[i] = newOp
return
}
}
operationQueue = append(operationQueue, newOp)
}
func UpsertTreeQueue(tree *parse.Tree) {
dbQueueLock.Lock()
defer dbQueueLock.Unlock()
newOp := &dbQueueOperation{upsertTree: tree, inQueueTime: time.Now(), action: "upsert"}
for i, op := range operationQueue {
if "upsert" == op.action && op.upsertTree.ID == tree.ID { // 相同树则覆盖
operationQueue[i] = newOp
@ -157,10 +216,10 @@ func UpsertTreeQueue(tree *parse.Tree) {
}
func RenameTreeQueue(tree *parse.Tree, oldHPath string) {
upsertTreeQueueLock.Lock()
defer upsertTreeQueueLock.Unlock()
dbQueueLock.Lock()
defer dbQueueLock.Unlock()
newOp := &treeQueueOperation{
newOp := &dbQueueOperation{
renameTree: tree,
renameTreeOldHPath: oldHPath,
inQueueTime: time.Now(),
@ -175,10 +234,10 @@ func RenameTreeQueue(tree *parse.Tree, oldHPath string) {
}
func RemoveTreeQueue(box, rootID string) {
upsertTreeQueueLock.Lock()
defer upsertTreeQueueLock.Unlock()
dbQueueLock.Lock()
defer dbQueueLock.Unlock()
var tmp []*treeQueueOperation
var tmp []*dbQueueOperation
// 将已有的 upsert 操作去重
for _, op := range operationQueue {
if "upsert" == op.action && op.upsertTree.ID != rootID {
@ -187,15 +246,15 @@ func RemoveTreeQueue(box, rootID string) {
}
operationQueue = tmp
newOp := &treeQueueOperation{removeTreeIDBox: box, removeTreeID: rootID, inQueueTime: time.Now(), action: "delete_id"}
newOp := &dbQueueOperation{removeTreeIDBox: box, removeTreeID: rootID, inQueueTime: time.Now(), action: "delete_id"}
operationQueue = append(operationQueue, newOp)
}
func RemoveTreePathQueue(treeBox, treePathPrefix string) {
upsertTreeQueueLock.Lock()
defer upsertTreeQueueLock.Unlock()
dbQueueLock.Lock()
defer dbQueueLock.Unlock()
var tmp []*treeQueueOperation
var tmp []*dbQueueOperation
// 将已有的 upsert 操作去重
for _, op := range operationQueue {
if "upsert" == op.action && (op.removeTreeBox != treeBox || op.upsertTree.Path != treePathPrefix) {
@ -204,6 +263,6 @@ func RemoveTreePathQueue(treeBox, treePathPrefix string) {
}
operationQueue = tmp
newOp := &treeQueueOperation{removeTreeBox: treeBox, removeTreePath: treePathPrefix, inQueueTime: time.Now(), action: "delete"}
newOp := &dbQueueOperation{removeTreeBox: treeBox, removeTreePath: treePathPrefix, inQueueTime: time.Now(), action: "delete"}
operationQueue = append(operationQueue, newOp)
}

1
kernel/task/queue.go
View file

@ -102,6 +102,7 @@ const (
RepoCheckout = "task.repo.checkout" // 从快照中检出
DatabaseIndexFull = "task.database.index.full" // 重建索引
DatabaseIndex = "task.database.index" // 数据库索引队列
DatabaseIndexRef = "task.database.index.ref" // 数据库索引引用
DatabaseIndexFix = "task.database.index.fix" // 数据库索引订正
OCRImage = "task.ocr.image" // 图片 OCR 提取文本
HistoryGenerateDoc = "task.history.generateDoc" // 生成文件历史