Merge remote-tracking branch 'origin/dev' into dev

This commit is contained in:
Vanessa 2022-08-23 11:55:51 +08:00
commit d31f23e2da
13 changed files with 253 additions and 25 deletions

View file

@ -29,7 +29,7 @@ require (
github.com/gin-contrib/gzip v0.0.6
github.com/gin-contrib/sessions v0.0.5
github.com/gin-gonic/gin v1.8.1
github.com/imroc/req/v3 v3.19.0
github.com/imroc/req/v3 v3.19.1
github.com/jinzhu/copier v0.3.5
github.com/mattn/go-sqlite3 v2.0.3+incompatible
github.com/mitchellh/go-ps v1.0.0
@ -38,7 +38,7 @@ require (
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/qiniu/go-sdk/v7 v7.13.0
github.com/radovskyb/watcher v1.0.7
github.com/siyuan-note/dejavu v0.0.0-20220821052517-f8edbabd0423
github.com/siyuan-note/dejavu v0.0.0-20220823012437-83b9401aeea3
github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75
github.com/siyuan-note/eventbus v0.0.0-20220624162334-ca7c06dc771f
github.com/siyuan-note/filelock v0.0.0-20220720144616-011221f7e128
@ -113,7 +113,7 @@ require (
go.uber.org/multierr v1.8.0 // indirect
golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8 // indirect
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
golang.org/x/net v0.0.0-20220812174116-3211cb980234 // indirect
golang.org/x/net v0.0.0-20220822230855-b0a4917ee28c // indirect
golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde // indirect
golang.org/x/sys v0.0.0-20220818161305-2296e01440c6 // indirect
golang.org/x/tools v0.1.12 // indirect

View file

@ -200,8 +200,8 @@ github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq
github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
github.com/imroc/req/v3 v3.19.0 h1:CgUA54GlgeivADx8nodAFwlgifctXZSxf+V8XibzdDs=
github.com/imroc/req/v3 v3.19.0/go.mod h1:EluRnkfh8A39BmrCARYhcUrfGyR8qPw+O0BZyTy4j9k=
github.com/imroc/req/v3 v3.19.1 h1:djAwxGYmQcnvnE5rQViUSll0GrmxlexYhRkL9Chuat4=
github.com/imroc/req/v3 v3.19.1/go.mod h1:EluRnkfh8A39BmrCARYhcUrfGyR8qPw+O0BZyTy4j9k=
github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg=
github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
@ -349,8 +349,8 @@ github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1l
github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
github.com/siyuan-note/dejavu v0.0.0-20220821052517-f8edbabd0423 h1:isIyTU/P59uEY9Teol3F55Va0NWrcMF1cbu+LqyEkY8=
github.com/siyuan-note/dejavu v0.0.0-20220821052517-f8edbabd0423/go.mod h1:/7pAviNPlpJiwZkEg2eyLTEq2/8sfW/AU4eHBvyrHFk=
github.com/siyuan-note/dejavu v0.0.0-20220823012437-83b9401aeea3 h1:Jo6xIoRVswwki+TQK5t8fefd/wGTEAdmQiETKTh6YSo=
github.com/siyuan-note/dejavu v0.0.0-20220823012437-83b9401aeea3/go.mod h1:/7pAviNPlpJiwZkEg2eyLTEq2/8sfW/AU4eHBvyrHFk=
github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 h1:Bi7/7f29LW+Fm0cHc0J1NO1cZqyJwljSWVmfOqVZgaE=
github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw=
github.com/siyuan-note/eventbus v0.0.0-20220624162334-ca7c06dc771f h1:JMobMNZ7AqaKKyEK+WeWFhix/2TDQXgPZDajU00IybU=
@ -465,8 +465,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220802222814-0bcc04d9c69b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.0.0-20220812174116-3211cb980234 h1:RDqmgfe7SvlMWoqC3xwQ2blLO3fcWcxMa3eBLRdRW7E=
golang.org/x/net v0.0.0-20220812174116-3211cb980234/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.0.0-20220822230855-b0a4917ee28c h1:JVAXQ10yGGVbSyoer5VILysz6YKjdNT2bsvlayjqhes=
golang.org/x/net v0.0.0-20220822230855-b0a4917ee28c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=

View file

@ -34,6 +34,7 @@ func main() {
go server.Serve(false)
model.InitAppearance()
sql.InitDatabase(false)
sql.InitHistoryDatabase(false)
sql.SetCaseSensitive(model.Conf.Search.CaseSensitive)
model.SyncData(true, false, false)

View file

@ -48,6 +48,7 @@ func StartKernel(container, appDir, workspaceDir, nativeLibDir, privateDataDir,
go func() {
model.InitAppearance()
sql.InitDatabase(false)
sql.InitHistoryDatabase(false)
sql.SetCaseSensitive(model.Conf.Search.CaseSensitive)
model.SyncData(true, false, false)

View file

@ -393,7 +393,7 @@ func RemoveUnusedAssets() (ret []string) {
ret = []string{}
unusedAssets := UnusedAssets()
historyDir, err := util.GetHistoryDir("delete")
historyDir, err := GetHistoryDir(HistoryOpClean)
if nil != err {
logging.LogErrorf("get history dir failed: %s", err)
return
@ -428,7 +428,7 @@ func RemoveUnusedAsset(p string) (ret string) {
return p
}
historyDir, err := util.GetHistoryDir("delete")
historyDir, err := GetHistoryDir(HistoryOpClean)
if nil != err {
logging.LogErrorf("get history dir failed: %s", err)
return

View file

@ -1185,7 +1185,7 @@ func RemoveDoc(boxID, p string) (err error) {
return
}
historyDir, err := util.GetHistoryDir("delete")
historyDir, err := GetHistoryDir(HistoryOpDelete)
if nil != err {
logging.LogErrorf("get history dir failed: %s", err)
return

View file

@ -79,7 +79,7 @@ func AutoSpace(rootID string) (err error) {
}
func generateFormatHistory(tree *parse.Tree) {
historyDir, err := util.GetHistoryDir("format")
historyDir, err := GetHistoryDir(HistoryOpFormat)
if nil != err {
logging.LogErrorf("get history dir failed: %s", err)
return

View file

@ -18,6 +18,7 @@ package model
import (
"encoding/json"
"fmt"
"io"
"io/fs"
"os"
@ -31,6 +32,7 @@ import (
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/conf"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
)
@ -476,7 +478,7 @@ func (box *Box) generateDocHistory0() {
return
}
historyDir, err := util.GetHistoryDir("update")
historyDir, err := GetHistoryDir(HistoryOpUpdate)
if nil != err {
logging.LogErrorf("get history dir failed: %s", err)
return
@ -562,3 +564,101 @@ func (box *Box) recentModifiedDocs() (ret []string) {
box.UpdateHistoryGenerated()
return
}
const (
HistoryOpClean = "clean"
HistoryOpUpdate = "update"
HistoryOpDelete = "delete"
HistoryOpFormat = "format"
)
func GetHistoryDir(suffix string) (ret string, err error) {
ret = filepath.Join(util.HistoryDir, time.Now().Format("2006-01-02-150405")+"-"+suffix)
if err = os.MkdirAll(ret, 0755); nil != err {
logging.LogErrorf("make history dir failed: %s", err)
return
}
return
}
func indexHistory() {
historyDirs, err := os.ReadDir(util.HistoryDir)
if nil != err {
logging.LogErrorf("read history dir [%s] failed: %s", util.HistoryDir, err)
return
}
validOps := []string{HistoryOpClean, HistoryOpUpdate, HistoryOpDelete, HistoryOpFormat}
lutEngine := NewLute()
for _, historyDir := range historyDirs {
if !historyDir.IsDir() {
continue
}
name := historyDir.Name()
op := name[strings.LastIndex(name, "-")+1:]
if !gulu.Str.Contains(op, validOps) {
logging.LogWarnf("invalid history op [%s]", op)
continue
}
t := name[:strings.LastIndex(name, "-")]
tt, parseErr := time.Parse("2006-01-02-150405", t)
if nil != parseErr {
logging.LogWarnf("parse time [%s] failed: %s", t, parseErr)
continue
}
created := fmt.Sprintf("%d", tt.Unix())
entryPath := filepath.Join(util.HistoryDir, name)
var docs, assets []string
filepath.Walk(entryPath, func(path string, info os.FileInfo, err error) error {
if strings.HasSuffix(info.Name(), ".sy") {
docs = append(docs, path)
} else if strings.Contains(path, "assets"+string(os.PathSeparator)) {
assets = append(assets, path)
}
return nil
})
var histories []*sql.History
for _, doc := range docs {
tree, loadErr := loadTree(doc, lutEngine)
if nil != err {
logging.LogErrorf("load tree [%s] failed: %s", doc, loadErr)
continue
}
title := tree.Root.IALAttr("title")
content := tree.Root.Content()
histories = append(histories, &sql.History{
Type: 0,
Op: op,
Title: title,
Content: content,
Path: doc,
Created: created,
})
}
for _, asset := range assets {
histories = append(histories, &sql.History{
Type: 1,
Op: op,
Title: filepath.Base(asset),
Path: asset,
Created: created,
})
}
tx, txErr := sql.BeginHistoryTx()
if nil != txErr {
logging.LogErrorf("begin transaction failed: %s", txErr)
return
}
if err = sql.InsertHistories(tx, histories); nil != err {
logging.LogErrorf("insert histories failed: %s", err)
sql.RollbackTx(tx)
return
}
}
}

View file

@ -91,7 +91,7 @@ func RemoveBox(boxID string) (err error) {
filelock.ReleaseFileLocks(localPath)
if !IsUserGuide(boxID) {
var historyDir string
historyDir, err = util.GetHistoryDir("delete")
historyDir, err = GetHistoryDir(HistoryOpDelete)
if nil != err {
logging.LogErrorf("get history dir failed: %s", err)
return

View file

@ -38,7 +38,10 @@ import (
"github.com/siyuan-note/siyuan/kernel/util"
)
var db *sql.DB
var (
db *sql.DB
historyDB *sql.DB
)
func init() {
regex := func(re, s string) (bool, error) {
@ -143,6 +146,40 @@ func initDBTables() {
}
}
func InitHistoryDatabase(forceRebuild bool) {
if !forceRebuild && gulu.File.IsExist(util.HistoryDBPath) {
return
}
if nil != historyDB {
historyDB.Close()
}
dsn := util.HistoryDBPath + "?_journal_mode=OFF" +
"&_synchronous=OFF" +
"&_secure_delete=OFF" +
"&_cache_size=-20480" +
"&_page_size=8192" +
"&_busy_timeout=7000" +
"&_ignore_check_constraints=ON" +
"&_temp_store=MEMORY" +
"&_case_sensitive_like=OFF" +
"&_locking_mode=EXCLUSIVE"
var err error
historyDB, err = sql.Open("sqlite3_extended", dsn)
if nil != err {
logging.LogFatalf("create database failed: %s", err)
}
historyDB.SetMaxIdleConns(1)
historyDB.SetMaxOpenConns(1)
historyDB.SetConnMaxLifetime(365 * 24 * time.Hour)
historyDB.Exec("DROP TABLE histories_fts_case_insensitive")
_, err = historyDB.Exec("CREATE VIRTUAL TABLE histories_fts_case_insensitive USING fts5(type UNINDEXED, op UNINDEXED, title, content, path UNINDEXED, created UNINDEXED, tokenize=\"siyuan case_insensitive\")")
if nil != err {
logging.LogFatalf("create table [histories_fts_case_insensitive] failed: %s", err)
}
}
func IndexMode() {
if nil != db {
db.Close()
@ -1029,6 +1066,9 @@ func CloseDatabase() {
if err := db.Close(); nil != err {
logging.LogErrorf("close database failed: %s", err)
}
if err := historyDB.Close(); nil != err {
logging.LogErrorf("close history database failed: %s", err)
}
}
func queryRow(query string, args ...interface{}) *sql.Row {
@ -1058,6 +1098,16 @@ func BeginTx() (tx *sql.Tx, err error) {
return
}
func BeginHistoryTx() (tx *sql.Tx, err error) {
if tx, err = historyDB.Begin(); nil != err {
logging.LogErrorf("begin history tx failed: %s\n %s", err, logging.ShortStack())
if strings.Contains(err.Error(), "database is locked") {
os.Exit(util.ExitCodeReadOnlyDatabase)
}
}
return
}
func CommitTx(tx *sql.Tx) (err error) {
if nil == tx {
logging.LogErrorf("tx is nil")

82
kernel/sql/history.go Normal file
View file

@ -0,0 +1,82 @@
// SiYuan - Build Your Eternal Digital Garden
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package sql
import (
"database/sql"
"fmt"
"strings"
)
type History struct {
Type int
Op string
Title string
Content string
Created string
Path string
}
const (
HistoriesFTSCaseInsensitiveInsert = "INSERT INTO histories_fts_case_insensitive (type, op, title, content, path, created) VALUES %s"
HistoriesPlaceholder = "(?, ?, ?, ?, ?, ?)"
)
func InsertHistories(tx *sql.Tx, histories []*History) (err error) {
if 1 > len(histories) {
return
}
var bulk []*History
for _, history := range histories {
bulk = append(bulk, history)
if 512 > len(bulk) {
continue
}
if err = insertHistories0(tx, bulk); nil != err {
return
}
bulk = []*History{}
}
if 0 < len(bulk) {
if err = insertHistories0(tx, bulk); nil != err {
return
}
}
return
}
func insertHistories0(tx *sql.Tx, bulk []*History) (err error) {
valueStrings := make([]string, 0, len(bulk))
valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(HistoriesPlaceholder, "?"))
for _, b := range bulk {
valueStrings = append(valueStrings, HistoriesPlaceholder)
valueArgs = append(valueArgs, b.Type)
valueArgs = append(valueArgs, b.Op)
valueArgs = append(valueArgs, b.Title)
valueArgs = append(valueArgs, b.Content)
valueArgs = append(valueArgs, b.Path)
valueArgs = append(valueArgs, b.Created)
}
stmt := fmt.Sprintf(HistoriesFTSCaseInsensitiveInsert, strings.Join(valueStrings, ","))
if err = prepareExecInsertTx(tx, stmt, valueArgs); nil != err {
return
}
return
}

View file

@ -146,15 +146,6 @@ func SetBooted() {
logging.LogInfof("kernel booted")
}
func GetHistoryDir(suffix string) (ret string, err error) {
ret = filepath.Join(HistoryDir, time.Now().Format("2006-01-02-150405")+"-"+suffix)
if err = os.MkdirAll(ret, 0755); nil != err {
logging.LogErrorf("make history dir failed: %s", err)
return
}
return
}
var (
HomeDir, _ = gulu.OS.Home()
WorkingDir, _ = os.Getwd()
@ -168,6 +159,7 @@ var (
LogPath string // 配置目录下的日志文件 siyuan.log 路径
DBName = "siyuan.db" // SQLite 数据库文件名
DBPath string // SQLite 数据库文件路径
HistoryDBPath string // SQLite 历史数据库文件路径
BlockTreePath string // 区块树文件路径
AppearancePath string // 配置目录下的外观目录 appearance/ 路径
ThemesPath string // 配置目录下的外观目录下的 themes/ 路径
@ -271,6 +263,7 @@ func initWorkspaceDir(workspaceArg string) {
os.Setenv("TEMP", osTmpDir)
os.Setenv("TMP", osTmpDir)
DBPath = filepath.Join(TempDir, DBName)
HistoryDBPath = filepath.Join(TempDir, "history.db")
BlockTreePath = filepath.Join(TempDir, "blocktree.msgpack")
}

View file

@ -51,6 +51,7 @@ func BootMobile(container, appDir, workspaceDir, nativeLibDir, privateDataDir, l
os.RemoveAll(filepath.Join(TempDir, "repo"))
os.Setenv("TMPDIR", osTmpDir)
DBPath = filepath.Join(TempDir, DBName)
HistoryDBPath = filepath.Join(TempDir, "history.db")
BlockTreePath = filepath.Join(TempDir, "blocktree.msgpack")
AndroidNativeLibDir = nativeLibDir
AndroidPrivateDataDir = privateDataDir