0ct0pu5/siyuan
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

♻️ 重构定时任务实现 https://github.com/siyuan-note/siyuan/issues/7171

Browse source
This commit is contained in:
Liang Ding 2023-01-26 23:30:29 +08:00
parent 78ed38a33a
commit df9122b664
No known key found for this signature in database
GPG key ID: 136F30F901A2231D
16 changed files with 175 additions and 197 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
kernel/go.mod
View file

@ -68,6 +68,7 @@ require (
github.com/dlclark/regexp2 v1.8.0 // indirect
github.com/dsnet/compress v0.0.1 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-co-op/gocron v1.18.0 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.0 // indirect
@ -112,6 +113,7 @@ require (
github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
github.com/qiniu/go-sdk/v7 v7.14.0 // indirect
github.com/restic/chunker v0.4.0 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
github.com/shopspring/decimal v1.3.1 // indirect
github.com/spf13/cast v1.5.0 // indirect

4
kernel/go.sum
View file

@ -108,6 +108,8 @@ github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR
github.com/gin-gonic/gin v1.8.2 h1:UzKToD9/PoFj/V4rvlKqTRKnQYyz8Sc1MJlv4JHPtvY=
github.com/gin-gonic/gin v1.8.2/go.mod h1:qw5AYuDrzRTnhvusDsrov+fDIxp9Dleuu12h8nfB398=
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
github.com/go-co-op/gocron v1.18.0 h1:SxTyJ5xnSN4byCq7b10LmmszFdxQlSQJod8s3gbnXxA=
github.com/go-co-op/gocron v1.18.0/go.mod h1:sD/a0Aadtw5CpflUJ/lpP9Vfdk979Wl1Sg33HPHg0FY=
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
@ -334,6 +336,8 @@ github.com/radovskyb/watcher v1.0.7/go.mod h1:78okwvY5wPdzcb1UYnip1pvrZNIVEIh/Cm
github.com/restic/chunker v0.4.0 h1:YUPYCUn70MYP7VO4yllypp2SjmsRhRJaad3xKu1QFRw=
github.com/restic/chunker v0.4.0/go.mod h1:z0cH2BejpW636LXw0R/BGyv+Ey8+m9QGiOanDHItzyw=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=

49
kernel/job/cron.go Normal file
View file

@ -0,0 +1,49 @@
// SiYuan - Build Your Eternal Digital Garden
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package job
import (
"github.com/siyuan-note/siyuan/kernel/model"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/util"
"time"
"github.com/go-co-op/gocron"
"github.com/siyuan-note/siyuan/kernel/treenode"
)
func StartCron() {
s := gocron.NewScheduler(time.Local)
s.Every(100).Milliseconds().Do(task.ExecTaskJob)
s.Every(5).Seconds().Do(task.StatusJob)
s.Every(1).Second().Do(treenode.SaveBlockTreeJob)
s.Every(5).Seconds().Do(model.SyncDataJob)
s.Every(2).Hours().Do(model.StatJob)
s.Every(2).Hours().Do(model.RefreshCheckJob)
s.Every(3).Seconds().Do(model.FlushUpdateRefTextRenameDocJob)
s.Every(2).Seconds().Do(model.FlushTxJob)
s.Every(util.SQLFlushInterval).Do(sql.FlushTxJob)
s.Every(10).Minutes().Do(model.FixIndexJob)
s.Every(10).Minutes().Do(model.IndexEmbedBlockJob)
s.Every(7).Seconds().Do(model.OCRAssetsJob)
s.Every(7).Seconds().Do(model.FlushAssetsTextsJob)
s.Every(30).Seconds().Do(model.HookDesktopUIProcJob)
s.SingletonModeAll()
s.StartAsync()
}

22
kernel/main.go
View file

@ -20,11 +20,10 @@ package main
import (
"github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/job"
"github.com/siyuan-note/siyuan/kernel/model"
"github.com/siyuan-note/siyuan/kernel/server"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
)
@ -43,24 +42,13 @@ func main() {
model.LoadFlashcards()
model.LoadAssetsTexts()
go task.Loop()
go task.StatusLoop()
go model.AutoGenerateDocHistory()
go model.AutoSync()
go model.AutoStat()
util.SetBooted()
util.PushClearAllMsg()
go model.AutoRefreshCheck()
go model.AutoFlushTx()
go sql.AutoFlushTx()
go treenode.AutoFlushBlockTree()
job.StartCron()
go model.AutoGenerateDocHistory()
go cache.LoadAssets()
go model.AutoFixIndex()
go model.AutoIndexEmbedBlock()
go model.AutoOCRAssets()
go model.AutoFlushAssetsTexts()
go model.HookDesktopUIProc()
model.WatchAssets()
model.HandleSignal()
}

22
kernel/mobile/kernel.go
View file

@ -18,17 +18,16 @@ package mobile
import (
"fmt"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/job"
"os"
"path/filepath"
"strings"
"time"
"github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/model"
"github.com/siyuan-note/siyuan/kernel/server"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
_ "golang.org/x/mobile/bind"
)
@ -57,23 +56,12 @@ func StartKernel(container, appDir, workspaceBaseDir, timezoneID, localIPs, lang
model.LoadFlashcards()
model.LoadAssetsTexts()
go task.Loop()
go task.StatusLoop()
go model.AutoGenerateDocHistory()
go model.AutoSync()
go model.AutoStat()
util.SetBooted()
util.PushClearAllMsg()
go model.AutoRefreshCheck()
go model.AutoFlushTx()
go sql.AutoFlushTx()
go treenode.AutoFlushBlockTree()
job.StartCron()
go model.AutoGenerateDocHistory()
go cache.LoadAssets()
go model.AutoFixIndex()
go model.AutoIndexEmbedBlock()
go model.AutoOCRAssets()
go model.AutoFlushAssetsTexts()
}()
}

10
kernel/model/box.go
View file

@ -56,17 +56,9 @@ type Box struct {
historyGenerated int64 // 最近一次历史生成时间
}
func AutoStat() {
time.Sleep(time.Minute)
autoStat()
for range time.Tick(2 * time.Hour) {
autoStat()
}
}
var statLock = sync.Mutex{}
func autoStat() {
func StatJob() {
statLock.Lock()
defer statLock.Unlock()

11
kernel/model/index.go
View file

@ -187,13 +187,10 @@ func IndexRefs() {
util.PushStatusBar(fmt.Sprintf(Conf.Language(55), i))
}
// AutoIndexEmbedBlock 嵌入块支持搜索 https://github.com/siyuan-note/siyuan/issues/7112
func AutoIndexEmbedBlock() {
for {
embedBlocks := sql.QueryEmptyContentEmbedBlocks()
task.AppendTask(task.DatabaseIndexEmbedBlock, autoIndexEmbedBlock, embedBlocks)
time.Sleep(10 * time.Minute)
}
// IndexEmbedBlockJob 嵌入块支持搜索 https://github.com/siyuan-note/siyuan/issues/7112
func IndexEmbedBlockJob() {
embedBlocks := sql.QueryEmptyContentEmbedBlocks()
task.AppendTask(task.DatabaseIndexEmbedBlock, autoIndexEmbedBlock, embedBlocks)
}
func autoIndexEmbedBlock(embedBlocks []*sql.Block) {

9
kernel/model/index_fix.go
View file

@ -36,12 +36,9 @@ import (
"github.com/siyuan-note/siyuan/kernel/util"
)
// AutoFixIndex 自动校验数据库索引 https://github.com/siyuan-note/siyuan/issues/7016
func AutoFixIndex() {
for {
task.AppendTask(task.DatabaseIndexFix, autoFixIndex)
time.Sleep(10 * time.Minute)
}
// FixIndexJob 自动校验数据库索引 https://github.com/siyuan-note/siyuan/issues/7016
func FixIndexJob() {
task.AppendTask(task.DatabaseIndexFix, autoFixIndex)
}
var autoFixLock = sync.Mutex{}

15
kernel/model/liandi.go
View file

@ -144,19 +144,14 @@ func LoadUploadToken() (err error) {
}
var (
refreshCheckTicker = time.NewTicker(2 * time.Hour)
subscriptionExpirationReminded bool
)
func AutoRefreshCheck() {
for {
go refreshSubscriptionExpirationRemind()
go refreshUser()
go refreshAnnouncement()
go refreshCheckDownloadInstallPkg()
<-refreshCheckTicker.C
}
func RefreshCheckJob() {
go refreshSubscriptionExpirationRemind()
go refreshUser()
go refreshAnnouncement()
go refreshCheckDownloadInstallPkg()
}
func refreshSubscriptionExpirationRemind() {

14
kernel/model/ocr.go
View file

@ -19,15 +19,12 @@ import (
"github.com/siyuan-note/siyuan/kernel/util"
)
func AutoOCRAssets() {
func OCRAssetsJob() {
if !util.TesseractEnabled {
return
}
for {
task.AppendTask(task.OCRImage, autoOCRAssets)
time.Sleep(7 * time.Second)
}
task.AppendTask(task.OCRImage, autoOCRAssets)
}
func autoOCRAssets() {
@ -111,11 +108,8 @@ func getUnOCRAssetsAbsPaths() (ret []string) {
return
}
func AutoFlushAssetsTexts() {
for {
SaveAssetsTexts()
time.Sleep(7 * time.Second)
}
func FlushAssetsTextsJob() {
SaveAssetsTexts()
}
func LoadAssetsTexts() {

70
kernel/model/process.go
View file

@ -37,45 +37,51 @@ func HandleSignal() {
Close(false, 1)
}
func HookDesktopUIProc() {
var firstRunHookDesktopUIProcJob = true
func HookDesktopUIProcJob() {
if util.ContainerStd != util.Container || "dev" == util.Mode {
return
}
time.Sleep(30 * time.Second)
if firstRunHookDesktopUIProcJob {
// 等待启动结束再
time.Sleep(30 * time.Second)
firstRunHookDesktopUIProcJob = false
return
}
uiProcNames := []string{"siyuan", "electron"}
existUIProc := false
for range time.Tick(7 * time.Second) {
util.UIProcessIDs.Range(func(uiProcIDArg, _ interface{}) bool {
uiProcID, err := strconv.Atoi(uiProcIDArg.(string))
if nil != err {
logging.LogErrorf("invalid UI proc ID [%s]: %s", uiProcIDArg, err)
return true
}
proc, err := goPS.FindProcess(uiProcID)
if nil != err {
logging.LogErrorf("find UI proc [%d] failed: %s", uiProcID, err)
return true
}
if nil == proc {
return true
}
procName := strings.ToLower(proc.Executable())
for _, name := range uiProcNames {
if strings.Contains(procName, name) {
existUIProc = true
return false
}
}
util.UIProcessIDs.Range(func(uiProcIDArg, _ interface{}) bool {
uiProcID, err := strconv.Atoi(uiProcIDArg.(string))
if nil != err {
logging.LogErrorf("invalid UI proc ID [%s]: %s", uiProcIDArg, err)
return true
})
if !existUIProc {
logging.LogInfof("no active UI proc, exit kernel process now")
Close(false, 1)
}
proc, err := goPS.FindProcess(uiProcID)
if nil != err {
logging.LogErrorf("find UI proc [%d] failed: %s", uiProcID, err)
return true
}
if nil == proc {
return true
}
procName := strings.ToLower(proc.Executable())
for _, name := range uiProcNames {
if strings.Contains(procName, name) {
existUIProc = true
return false
}
}
return true
})
if !existUIProc {
logging.LogInfof("no active UI proc, exit kernel process now")
Close(false, 1)
}
}

9
kernel/model/sync.go
View file

@ -46,12 +46,9 @@ var (
ExitSyncSucc = -1
)
func AutoSync() {
for {
time.Sleep(5 * time.Second)
if time.Now().After(syncPlanTime) {
SyncData(false, false, false)
}
func SyncDataJob() {
if time.Now().After(syncPlanTime) {
SyncData(false, false, false)
}
}

33
kernel/model/transaction.go
View file

@ -66,7 +66,6 @@ const txFixDelay = 10
var (
txQueue []*Transaction
txQueueLock = sync.Mutex{}
txDelay = txFixDelay
currentTx *Transaction
)
@ -88,19 +87,15 @@ func WaitForWritingFiles() {
}
func isWritingFiles() bool {
time.Sleep(time.Duration(txDelay+5) * time.Millisecond)
time.Sleep(time.Duration(txFixDelay+10) * time.Millisecond)
if 0 < len(txQueue) || util.IsMutexLocked(&txQueueLock) {
return true
}
return nil != currentTx
}
func AutoFlushTx() {
go autoFlushUpdateRefTextRenameDoc()
for {
flushTx()
time.Sleep(time.Duration(txDelay) * time.Millisecond)
}
func FlushTxJob() {
flushTx()
}
func flushTx() {
@ -123,7 +118,7 @@ func flushTx() {
elapsed := time.Now().Sub(start).Milliseconds()
if 0 < len(currentTx.DoOperations) {
if 2000 < elapsed {
logging.LogWarnf("tx [%dms]", elapsed)
logging.LogWarnf("op tx [%dms]", elapsed)
}
}
currentTx = nil
@ -178,12 +173,6 @@ type TxErr struct {
func performTx(tx *Transaction) (ret *TxErr) {
if 1 > len(tx.DoOperations) {
txDelay -= 1000
if 100*txFixDelay < txDelay {
txDelay = txDelay / 2
} else if 0 > txDelay {
txDelay = txFixDelay
}
return
}
@ -202,7 +191,6 @@ func performTx(tx *Transaction) (ret *TxErr) {
return
}
start := time.Now()
for _, op := range tx.DoOperations {
switch op.Action {
case "create":
@ -243,11 +231,6 @@ func performTx(tx *Transaction) (ret *TxErr) {
logging.LogErrorf("commit tx failed: %s", cr)
return &TxErr{msg: cr.Error()}
}
elapsed := int(time.Now().Sub(start).Milliseconds())
txDelay = 10 + elapsed
if 1000*10 < txDelay {
txDelay = 1000 * 10
}
return
}
@ -1186,11 +1169,9 @@ func updateRefTextRenameDoc(renamedTree *parse.Tree) {
updateRefTextRenameDocLock.Unlock()
}
func autoFlushUpdateRefTextRenameDoc() {
for {
sql.WaitForWritingDatabase()
flushUpdateRefTextRenameDoc()
}
func FlushUpdateRefTextRenameDocJob() {
sql.WaitForWritingDatabase()
flushUpdateRefTextRenameDoc()
}
func flushUpdateRefTextRenameDoc() {

7
kernel/sql/queue.go
View file

@ -53,11 +53,8 @@ type dbQueueOperation struct {
renameTreeOldHPath string // rename
}
func AutoFlushTx() {
for {
time.Sleep(util.SQLFlushInterval)
task.AppendTask(task.DatabaseIndexCommit, FlushQueue)
}
func FlushTxJob() {
task.AppendTask(task.DatabaseIndexCommit, FlushQueue)
}
func WaitForWritingDatabase() {

84
kernel/task/queue.go
View file

@ -121,56 +121,50 @@ func ContainIndexTask() bool {
return false
}
func StatusLoop() {
for {
time.Sleep(5 * time.Second)
tasks := taskQueue
data := map[string]interface{}{}
var items []map[string]interface{}
for _, task := range tasks {
if OCRImage == task.Action || DatabaseIndexEmbedBlock == task.Action {
continue
}
actionLangs := util.TaskActionLangs[util.Lang]
action := task.Action
if nil != actionLangs {
if label := actionLangs[task.Action]; nil != label {
action = label.(string)
}
}
item := map[string]interface{}{
"action": action,
}
items = append(items, item)
}
if 1 > len(items) {
items = []map[string]interface{}{}
}
data["tasks"] = items
util.PushBackgroundTask(data)
}
}
func Loop() {
for {
time.Sleep(100 * time.Millisecond)
if QueueStatusClosing == taskQueueStatus {
clearQueue()
break
}
task := popTask()
if nil == task {
func StatusJob() {
tasks := taskQueue
data := map[string]interface{}{}
var items []map[string]interface{}
for _, task := range tasks {
if OCRImage == task.Action || DatabaseIndexEmbedBlock == task.Action {
continue
}
if util.IsExiting {
break
actionLangs := util.TaskActionLangs[util.Lang]
action := task.Action
if nil != actionLangs {
if label := actionLangs[task.Action]; nil != label {
action = label.(string)
}
}
execTask(task)
item := map[string]interface{}{
"action": action,
}
items = append(items, item)
}
if 1 > len(items) {
items = []map[string]interface{}{}
}
data["tasks"] = items
util.PushBackgroundTask(data)
}
func ExecTaskJob() {
if QueueStatusClosing == taskQueueStatus {
clearQueue()
return
}
task := popTask()
if nil == task {
return
}
if util.IsExiting {
return
}
execTask(task)
}
func clearQueue() {

11
kernel/treenode/blocktree.go
View file

@ -400,13 +400,6 @@ func IndexBlockTree(tree *parse.Tree) {
})
}
func AutoFlushBlockTree() {
for {
SaveBlockTree(false)
time.Sleep(1 * time.Second)
}
}
func InitBlockTree(force bool) {
start := time.Now()
@ -483,6 +476,10 @@ func InitBlockTree(force bool) {
return
}
func SaveBlockTreeJob() {
SaveBlockTree(false)
}
func SaveBlockTree(force bool) {
start := time.Now()
os.MkdirAll(util.BlockTreePath, 0755)