0ct0pu5/siyuan
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/dev' into dev

Browse source
This commit is contained in:
Vanessa 2023-01-19 00:03:31 +08:00
commit 3a93650d9d
14 changed files with 216 additions and 112 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
kernel/api/repo.go
View file

@ -113,11 +113,7 @@ func checkoutRepo(c *gin.Context) {
}
id := arg["id"].(string)
if err := model.CheckoutRepo(id); nil != err {
ret.Code = -1
ret.Msg = model.Conf.Language(141)
return
}
model.CheckoutRepo(id)
}
func downloadCloudSnapshot(c *gin.Context) {

5
kernel/main.go
View file

@ -23,6 +23,7 @@ import (
"github.com/siyuan-note/siyuan/kernel/model"
"github.com/siyuan-note/siyuan/kernel/server"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
)
@ -42,6 +43,8 @@ func main() {
model.LoadFlashcards()
model.LoadAssetsTexts()
go task.Loop()
go model.AutoGenerateDocHistory()
go model.AutoSync()
go model.AutoStat()
@ -49,7 +52,7 @@ func main() {
util.PushClearAllMsg()
go model.AutoRefreshCheck()
go model.AutoFlushTx()
go sql.AutoFlushTreeQueue()
go sql.AutoFlushTx()
go treenode.AutoFlushBlockTree()
go cache.LoadAssets()
go model.AutoFixIndex()

5
kernel/mobile/kernel.go
View file

@ -27,6 +27,7 @@ import (
"github.com/siyuan-note/siyuan/kernel/model"
"github.com/siyuan-note/siyuan/kernel/server"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
_ "golang.org/x/mobile/bind"
@ -56,6 +57,8 @@ func StartKernel(container, appDir, workspaceBaseDir, timezoneID, localIPs, lang
model.LoadFlashcards()
model.LoadAssetsTexts()
go task.Loop()
go model.AutoGenerateDocHistory()
go model.AutoSync()
go model.AutoStat()
@ -63,7 +66,7 @@ func StartKernel(container, appDir, workspaceBaseDir, timezoneID, localIPs, lang
util.PushClearAllMsg()
go model.AutoRefreshCheck()
go model.AutoFlushTx()
go sql.AutoFlushTreeQueue()
go sql.AutoFlushTx()
go treenode.AutoFlushBlockTree()
go cache.LoadAssets()
go model.AutoFixIndex()

9
kernel/model/box.go
View file

@ -39,6 +39,7 @@ import (
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/conf"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
)
@ -504,10 +505,11 @@ func genTreeID(tree *parse.Tree) {
return
}
var isFullReindexing = false
func FullReindex() {
isFullReindexing = true
task.PrependTask(task.DatabaseIndexFull, fullReindex)
}
func fullReindex() {
util.PushEndlessProgress(Conf.Language(35))
WaitForWritingFiles()
@ -526,7 +528,6 @@ func FullReindex() {
LoadFlashcards()
util.PushEndlessProgress(Conf.Language(58))
isFullReindexing = false
go func() {
time.Sleep(1 * time.Second)
util.ReloadUI()

2
kernel/model/conf.go
View file

@ -37,6 +37,7 @@ import (
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/conf"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
"golang.org/x/text/language"
@ -425,6 +426,7 @@ func Close(force bool, execInstallPkg int) (exitCode int) {
}
}
task.CloseWait()
Conf.Close()
sql.CloseDatabase()
treenode.SaveBlockTree(false)

3
kernel/model/history.go
View file

@ -38,6 +38,7 @@ import (
"github.com/siyuan-note/siyuan/kernel/conf"
"github.com/siyuan-note/siyuan/kernel/search"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
)
@ -48,7 +49,7 @@ func AutoGenerateDocHistory() {
ChangeHistoryTick(Conf.Editor.GenerateHistoryInterval)
for {
<-historyTicker.C
generateDocHistory()
task.PrependTask(task.HistoryGenerateDoc, generateDocHistory)
}
}

15
kernel/model/ocr.go
View file

@ -1,7 +1,6 @@
package model
import (
"github.com/dustin/go-humanize"
"io"
"os"
"path/filepath"
@ -12,9 +11,11 @@ import (
"time"
"github.com/88250/gulu"
"github.com/dustin/go-humanize"
"github.com/panjf2000/ants/v2"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/util"
)
@ -24,7 +25,7 @@ func AutoOCRAssets() {
}
for {
autoOCRAssets()
task.AppendTask(task.OCRImage, autoOCRAssets)
time.Sleep(7 * time.Second)
}
}
@ -52,17 +53,21 @@ func autoOCRAssets() {
util.AssetsTextsLock.Unlock()
util.AssetsTextsChanged = true
})
for _, assetAbsPath := range assets {
for i, assetAbsPath := range assets {
waitGroup.Add(1)
p.Invoke(assetAbsPath)
if 63 <= i { // 一次最多处理 64 张图片，防止卡顿
break
}
}
waitGroup.Wait()
p.Release()
cleanNotFoundAssetsTexts()
cleanNotExistAssetsTexts()
}
func cleanNotFoundAssetsTexts() {
func cleanNotExistAssetsTexts() {
tmp := util.AssetsTexts
assetsPath := util.GetDataAssetsAbsPath()

25
kernel/model/repository.go
View file

@ -49,6 +49,7 @@ import (
"github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/conf"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
"github.com/studio-b12/gowebdav"
@ -501,16 +502,21 @@ func InitRepoKey() (err error) {
return
}
var isCheckoutRepo bool
func CheckoutRepo(id string) {
task.PrependTask(task.RepoCheckout, checkoutRepo, id)
}
func CheckoutRepo(id string) (err error) {
func checkoutRepo(id string) {
var err error
if 1 > len(Conf.Repo.Key) {
err = errors.New(Conf.Language(26))
util.PushErrMsg(Conf.Language(26), 7000)
return
}
repo, err := newRepository()
if nil != err {
logging.LogErrorf("new repository failed: %s", err)
util.PushErrMsg(Conf.Language(141), 7000)
return
}
@ -525,23 +531,16 @@ func CheckoutRepo(id string) (err error) {
Conf.Sync.Enabled = false
Conf.Save()
if util.IsMutexLocked(&syncLock) {
err = errors.New("sync is running, please try again later")
return
}
isCheckoutRepo = true
defer func() {
isCheckoutRepo = false
}()
_, _, err = repo.Checkout(id, map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress})
if nil != err {
logging.LogErrorf("checkout repository failed: %s", err)
util.PushClearProgress()
util.PushErrMsg(Conf.Language(141), 7000)
return
}
FullReindex()
if syncEnabled {
func() {
time.Sleep(5 * time.Second)

32
kernel/model/sync.go
View file

@ -32,6 +32,7 @@ import (
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/conf"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
)
@ -106,6 +107,10 @@ func BootSyncData() {
}
func SyncData(boot, exit, byHand bool) {
task.PrependTask(task.CloudSync, syncData, boot, exit, byHand)
}
func syncData(boot, exit, byHand bool) {
defer logging.Recover()
if !boot && !exit && 2 == Conf.Sync.Mode && !byHand {
@ -267,53 +272,35 @@ func SetCloudSyncDir(name string) {
return
}
syncLock.Lock()
defer syncLock.Unlock()
Conf.Sync.CloudName = name
Conf.Save()
}
func SetSyncGenerateConflictDoc(b bool) {
syncLock.Lock()
defer syncLock.Unlock()
Conf.Sync.GenerateConflictDoc = b
Conf.Save()
return
}
func SetSyncEnable(b bool) (err error) {
syncLock.Lock()
defer syncLock.Unlock()
Conf.Sync.Enabled = b
Conf.Save()
return
}
func SetSyncMode(mode int) (err error) {
syncLock.Lock()
defer syncLock.Unlock()
Conf.Sync.Mode = mode
Conf.Save()
return
}
func SetSyncProvider(provider int) (err error) {
syncLock.Lock()
defer syncLock.Unlock()
Conf.Sync.Provider = provider
Conf.Save()
return
}
func SetSyncProviderS3(s3 *conf.S3) (err error) {
syncLock.Lock()
defer syncLock.Unlock()
s3.Endpoint = strings.TrimSpace(s3.Endpoint)
s3.Endpoint = util.NormalizeEndpoint(s3.Endpoint)
s3.AccessKey = strings.TrimSpace(s3.AccessKey)
@ -328,9 +315,6 @@ func SetSyncProviderS3(s3 *conf.S3) (err error) {
}
func SetSyncProviderWebDAV(webdav *conf.WebDAV) (err error) {
syncLock.Lock()
defer syncLock.Unlock()
webdav.Endpoint = strings.TrimSpace(webdav.Endpoint)
webdav.Endpoint = util.NormalizeEndpoint(webdav.Endpoint)
webdav.Username = strings.TrimSpace(webdav.Username)
@ -350,9 +334,6 @@ func CreateCloudSyncDir(name string) (err error) {
return
}
syncLock.Lock()
defer syncLock.Unlock()
name = strings.TrimSpace(name)
name = gulu.Str.RemoveInvisible(name)
if !cloud.IsValidCloudDirName(name) {
@ -380,9 +361,6 @@ func RemoveCloudSyncDir(name string) (err error) {
msgId := util.PushMsg(Conf.Language(116), 15000)
syncLock.Lock()
defer syncLock.Unlock()
if "" == name {
return
}

52
kernel/model/transaction.go
View file

@ -20,7 +20,6 @@ import (
"bytes"
"errors"
"fmt"
"io"
"os"
"path"
"path/filepath"
@ -38,6 +37,7 @@ import (
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
)
@ -106,12 +106,7 @@ func AutoFlushTx() {
}
}
var txLock = sync.Mutex{}
func flushTx() {
txLock.Lock()
defer txLock.Unlock()
defer logging.Recover()
currentTx = mergeTx()
@ -1224,7 +1219,7 @@ func updateRefText(refNode *ast.Node, changedDefNodes map[string]*ast.Node) (cha
// AutoFixIndex 自动校验数据库索引 https://github.com/siyuan-note/siyuan/issues/7016
func AutoFixIndex() {
for {
autoFixIndex()
task.AppendTask(task.DatabaseIndexFix, autoFixIndex)
time.Sleep(10 * time.Minute)
}
}
@ -1234,21 +1229,6 @@ var autoFixLock = sync.Mutex{}
func autoFixIndex() {
defer logging.Recover()
if isFullReindexing {
logging.LogInfof("skip check index caused by full reindexing")
return
}
if util.IsMutexLocked(&syncLock) {
logging.LogInfof("skip check index caused by sync lock")
return
}
if isCheckoutRepo {
logging.LogInfof("skip check index caused by checkout repo")
return
}
if util.IsMutexLocked(&autoFixLock) {
return
}
@ -1262,10 +1242,6 @@ func autoFixIndex() {
boxPath := filepath.Join(util.DataDir, box.ID)
var paths []string
filepath.Walk(boxPath, func(path string, info os.FileInfo, err error) error {
if isFullReindexing {
return io.EOF
}
if !info.IsDir() && filepath.Ext(path) == ".sy" {
p := path[len(boxPath):]
p = filepath.ToSlash(p)
@ -1278,19 +1254,11 @@ func autoFixIndex() {
redundantPaths := treenode.GetRedundantPaths(box.ID, paths)
for _, p := range redundantPaths {
if isFullReindexing {
break
}
treenode.RemoveBlockTreesByPath(p)
}
missingPaths := treenode.GetNotExistPaths(box.ID, paths)
for i, p := range missingPaths {
if isFullReindexing {
break
}
id := path.Base(p)
id = strings.TrimSuffix(id, ".sy")
if !ast.IsNodeIDPattern(id) {
@ -1308,10 +1276,6 @@ func autoFixIndex() {
i := -1
size := len(rootUpdatedMap)
for rootID, updated := range rootUpdatedMap {
if isFullReindexing {
break
}
i++
rootUpdated := dbRootUpdatedMap[rootID]
@ -1341,10 +1305,6 @@ func autoFixIndex() {
duplicatedRootIDs := sql.GetDuplicatedRootIDs()
size := len(duplicatedRootIDs)
for i, rootID := range duplicatedRootIDs {
if isFullReindexing {
break
}
root := sql.GetBlock(rootID)
if nil == root {
continue
@ -1359,10 +1319,6 @@ func autoFixIndex() {
}
func reindexTreeByPath(box, p string, i, size int) {
if isFullReindexing {
return
}
tree, err := LoadTree(box, p)
if nil != err {
return
@ -1372,10 +1328,6 @@ func reindexTreeByPath(box, p string, i, size int) {
}
func reindexTree(rootID string, i, size int) {
if isFullReindexing {
return
}
root := treenode.GetBlockTree(rootID)
if nil == root {
logging.LogWarnf("root block not found", rootID)

2
kernel/sql/database.go
View file

@ -361,7 +361,7 @@ func buildEmbedRef(tree *parse.Tree, embedNode *ast.Node) *Ref {
RootID: tree.ID,
Box: tree.Box,
Path: tree.Path,
Content: "",
Content: "", // 通过嵌入块构建引用时定义块可能还没有入库，所以这里统一不填充内容
Markdown: "",
Type: treenode.TypeAbbr(embedNode.Type.String()),
}

7
kernel/sql/queue.go
View file

@ -29,6 +29,7 @@ import (
"github.com/emirpasic/gods/sets/hashset"
"github.com/siyuan-note/eventbus"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/util"
)
@ -50,10 +51,10 @@ type treeQueueOperation struct {
renameTreeOldHPath string // rename
}
func AutoFlushTreeQueue() {
func AutoFlushTx() {
for {
flushTreeQueue()
time.Sleep(util.SQLFlushInterval)
task.PrependTask(task.DatabaseIndex, FlushQueue)
}
}
@ -91,7 +92,7 @@ func ClearQueue() {
operationQueue = nil
}
func flushTreeQueue() {
func FlushQueue() {
ops := mergeUpsertTrees()
if 1 > len(ops) {
return

163
kernel/task/queue.go Normal file
View file

@ -0,0 +1,163 @@
// SiYuan - Build Your Eternal Digital Garden
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package task
import (
"reflect"
"sync"
"time"
"github.com/siyuan-note/logging"
)
var (
taskQueue []*Task
taskQueueStatus int
queueLock = sync.Mutex{}
taskLock = sync.Mutex{}
)
const (
QueueStatusRunning = iota
QueueStatusClosing
)
type Task struct {
Action string
Handler reflect.Value
Args []interface{}
Created time.Time
}
func PrependTask(action string, handler interface{}, args ...interface{}) {
queueLock.Lock()
defer queueLock.Unlock()
if QueueStatusRunning != taskQueueStatus {
logging.LogWarnf("task queue is paused, action [%s] will be ignored", action)
return
}
cancelTask(action)
taskQueue = append([]*Task{newTask(action, handler, args...)}, taskQueue...)
}
func AppendTask(action string, handler interface{}, args ...interface{}) {
queueLock.Lock()
defer queueLock.Unlock()
if QueueStatusRunning != taskQueueStatus {
logging.LogWarnf("task queue is paused, action [%s] will be ignored", action)
return
}
cancelTask(action)
taskQueue = append(taskQueue, newTask(action, handler, args...))
}
func CancelTask(actions ...string) {
queueLock.Lock()
defer queueLock.Unlock()
cancelTask(actions...)
}
func cancelTask(actions ...string) {
for i := len(taskQueue) - 1; i >= 0; i-- {
task := taskQueue[i]
for _, action := range actions {
if action == task.Action {
taskQueue = append(taskQueue[:i], taskQueue[i+1:]...)
break
}
}
}
}
func newTask(action string, handler interface{}, args ...interface{}) *Task {
return &Task{
Action: action,
Handler: reflect.ValueOf(handler),
Args: args,
Created: time.Now(),
}
}
const (
CloudSync = "task.cloud.sync" // 数据同步
RepoCheckout = "task.repo.checkout" // 从快照中检出
DatabaseIndexFull = "task.database.index.full" // 重建索引
DatabaseIndex = "task.database.index" // 数据库所以队列
DatabaseIndexFix = "task.database.index.fix" // 数据库索引订正
OCRImage = "task.ocr.image" // 图片 OCR 提取文本
HistoryGenerateDoc = "task.history.generateDoc" // 生成文件历史
DatabaseIndexEmbedBlock = "task.database.index.embedBlock" // 数据库索引嵌入块
)
func Loop() {
for {
time.Sleep(10 * time.Millisecond)
task := popTask()
if nil == task {
continue
}
execTask(task)
}
}
func CloseWait() {
queueLock.Lock()
defer queueLock.Unlock()
taskQueueStatus = QueueStatusClosing
for {
time.Sleep(10 * time.Millisecond)
if 1 > len(taskQueue) {
break
}
}
}
func popTask() (ret *Task) {
queueLock.Lock()
defer queueLock.Unlock()
if 0 == len(taskQueue) {
return
}
ret = taskQueue[0]
taskQueue = taskQueue[1:]
return
}
func execTask(task *Task) {
taskLock.Lock()
defer taskLock.Unlock()
defer logging.Recover()
args := make([]reflect.Value, len(task.Args))
for i, v := range task.Args {
if nil == v {
args[i] = reflect.New(task.Handler.Type().In(i)).Elem()
} else {
args[i] = reflect.ValueOf(v)
}
}
task.Handler.Call(args)
}

2
kernel/treenode/node.go
View file

@ -186,7 +186,7 @@ func NodeStaticContent(node *ast.Node, excludeTypes []string) string {
buf.WriteByte(' ')
}
if nil != linkDest {
buf.Write(n.Tokens)
buf.Write(linkDest.Tokens)
buf.WriteByte(' ')
}