0ct0pu5/siyuan
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/dev' into dev

Browse source
This commit is contained in:
Vanessa 2023-11-23 11:35:38 +08:00
commit aeb7ace58d
12 changed files with 276 additions and 81 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
API.md
View file

@ -1032,6 +1032,7 @@ View API token in <kbd>Settings - About</kbd>, request header: `Authorization: T
* `code`: non-zero for exceptions
* `-1`: Parameter parsing error
* `403`: Permission denied (file is not in the workspace)
* `404`: Not Found (file doesn't exist)
* `405`: Method Not Allowed (it's a directory)
* `500`: Server Error (stat file failed / read file failed)

1
API_zh_CN.md
View file

@ -1025,6 +1025,7 @@
* `code`: 非零的异常值
* `-1`: 参数解析错误
* `403`: 无访问权限 (文件不在工作空间下)
* `404`: 未找到 (文件不存在)
* `405`: 方法不被允许 (这是一个目录)
* `500`: 服务器错误 (文件查询失败 / 文件读取失败)

48
kernel/api/archive.go
View file

@ -34,20 +34,34 @@ func zip(c *gin.Context) {
return
}
path := arg["path"].(string)
zipPath := arg["zipPath"].(string)
zipFile, err := gulu.Zip.Create(zipPath)
entryPath := arg["path"].(string)
entryAbsPath, err := util.GetAbsPathInWorkspace(entryPath)
if nil != err {
ret.Code = -1
ret.Msg = err.Error()
return
}
base := filepath.Base(path)
if gulu.File.IsDir(path) {
err = zipFile.AddDirectory(base, path)
zipFilePath := arg["zipPath"].(string)
zipAbsFilePath, err := util.GetAbsPathInWorkspace(zipFilePath)
if nil != err {
ret.Code = -1
ret.Msg = err.Error()
return
}
zipFile, err := gulu.Zip.Create(zipAbsFilePath)
if nil != err {
ret.Code = -1
ret.Msg = err.Error()
return
}
base := filepath.Base(entryAbsPath)
if gulu.File.IsDir(entryAbsPath) {
err = zipFile.AddDirectory(base, entryAbsPath)
} else {
err = zipFile.AddEntry(base, path)
err = zipFile.AddEntry(base, entryAbsPath)
}
if nil != err {
ret.Code = -1
@ -71,9 +85,23 @@ func unzip(c *gin.Context) {
return
}
zipPath := arg["zipPath"].(string)
path := arg["path"].(string)
if err := gulu.Zip.Unzip(zipPath, path); nil != err {
zipFilePath := arg["zipPath"].(string)
zipAbsFilePath, err := util.GetAbsPathInWorkspace(zipFilePath)
if nil != err {
ret.Code = -1
ret.Msg = err.Error()
return
}
entryPath := arg["path"].(string)
entryAbsPath, err := util.GetAbsPathInWorkspace(entryPath)
if nil != err {
ret.Code = -1
ret.Msg = err.Error()
return
}
if err := gulu.Zip.Unzip(zipAbsFilePath, entryAbsPath); nil != err {
ret.Code = -1
ret.Msg = err.Error()
return

40
kernel/api/av.go
View file

@ -26,6 +26,46 @@ import (
"github.com/siyuan-note/siyuan/kernel/util"
)
func renderHistoryAttributeView(c *gin.Context) {
ret := gulu.Ret.NewResult()
defer c.JSON(http.StatusOK, ret)
arg, ok := util.JsonArg(c, ret)
if !ok {
return
}
id := arg["id"].(string)
created := arg["created"].(string)
view, attrView, err := model.RenderHistoryAttributeView(id, created)
if nil != err {
ret.Code = -1
ret.Msg = err.Error()
return
}
var views []map[string]interface{}
for _, v := range attrView.Views {
view := map[string]interface{}{
"id": v.ID,
"name": v.Name,
"type": v.LayoutType,
}
views = append(views, view)
}
ret.Data = map[string]interface{}{
"name": attrView.Name,
"id": attrView.ID,
"viewType": view.GetType(),
"viewID": view.GetID(),
"views": views,
"view": view,
"isMirror": av.IsMirror(attrView.ID),
}
}
func renderAttributeView(c *gin.Context) {
ret := gulu.Ret.NewResult()
defer c.JSON(http.StatusOK, ret)

145
kernel/api/file.go
View file

@ -90,38 +90,45 @@ func getFile(c *gin.Context) {
}
filePath := arg["path"].(string)
filePath = filepath.Join(util.WorkspaceDir, filePath)
info, err := os.Stat(filePath)
fileAbsPath, err := util.GetAbsPathInWorkspace(filePath)
if nil != err {
ret.Code = http.StatusForbidden
ret.Msg = err.Error()
c.JSON(http.StatusAccepted, ret)
return
}
info, err := os.Stat(fileAbsPath)
if os.IsNotExist(err) {
ret.Code = 404
ret.Code = http.StatusNotFound
ret.Msg = err.Error()
c.JSON(http.StatusAccepted, ret)
return
}
if nil != err {
logging.LogErrorf("stat [%s] failed: %s", filePath, err)
ret.Code = 500
logging.LogErrorf("stat [%s] failed: %s", fileAbsPath, err)
ret.Code = http.StatusInternalServerError
ret.Msg = err.Error()
c.JSON(http.StatusAccepted, ret)
return
}
if info.IsDir() {
logging.LogErrorf("file [%s] is a directory", filePath)
ret.Code = 405
logging.LogErrorf("file [%s] is a directory", fileAbsPath)
ret.Code = http.StatusMethodNotAllowed
ret.Msg = "file is a directory"
c.JSON(http.StatusAccepted, ret)
return
}
data, err := filelock.ReadFile(filePath)
data, err := filelock.ReadFile(fileAbsPath)
if nil != err {
logging.LogErrorf("read file [%s] failed: %s", filePath, err)
ret.Code = 500
logging.LogErrorf("read file [%s] failed: %s", fileAbsPath, err)
ret.Code = http.StatusInternalServerError
ret.Msg = err.Error()
c.JSON(http.StatusAccepted, ret)
return
}
contentType := mime.TypeByExtension(filepath.Ext(filePath))
contentType := mime.TypeByExtension(filepath.Ext(fileAbsPath))
if "" == contentType {
if m := mimetype.Detect(data); nil != m {
contentType = m.String()
@ -144,40 +151,46 @@ func readDir(c *gin.Context) {
}
dirPath := arg["path"].(string)
dirPath = filepath.Join(util.WorkspaceDir, dirPath)
info, err := os.Stat(dirPath)
dirAbsPath, err := util.GetAbsPathInWorkspace(dirPath)
if nil != err {
ret.Code = http.StatusForbidden
ret.Msg = err.Error()
return
}
info, err := os.Stat(dirAbsPath)
if os.IsNotExist(err) {
ret.Code = 404
ret.Code = http.StatusNotFound
ret.Msg = err.Error()
return
}
if nil != err {
logging.LogErrorf("stat [%s] failed: %s", dirPath, err)
ret.Code = 500
logging.LogErrorf("stat [%s] failed: %s", dirAbsPath, err)
ret.Code = http.StatusInternalServerError
ret.Msg = err.Error()
return
}
if !info.IsDir() {
logging.LogErrorf("file [%s] is not a directory", dirPath)
ret.Code = 405
logging.LogErrorf("file [%s] is not a directory", dirAbsPath)
ret.Code = http.StatusMethodNotAllowed
ret.Msg = "file is not a directory"
return
}
entries, err := os.ReadDir(dirPath)
entries, err := os.ReadDir(dirAbsPath)
if nil != err {
logging.LogErrorf("read dir [%s] failed: %s", dirPath, err)
ret.Code = 500
logging.LogErrorf("read dir [%s] failed: %s", dirAbsPath, err)
ret.Code = http.StatusInternalServerError
ret.Msg = err.Error()
return
}
files := []map[string]interface{}{}
for _, entry := range entries {
path := filepath.Join(dirPath, entry.Name())
path := filepath.Join(dirAbsPath, entry.Name())
info, err = os.Stat(path)
if nil != err {
logging.LogErrorf("stat [%s] failed: %s", path, err)
ret.Code = 500
ret.Code = http.StatusInternalServerError
ret.Msg = err.Error()
return
}
@ -202,25 +215,36 @@ func renameFile(c *gin.Context) {
return
}
filePath := arg["path"].(string)
filePath = filepath.Join(util.WorkspaceDir, filePath)
if !filelock.IsExist(filePath) {
ret.Code = 404
srcPath := arg["path"].(string)
srcAbsPath, err := util.GetAbsPathInWorkspace(srcPath)
if nil != err {
ret.Code = http.StatusForbidden
ret.Msg = err.Error()
return
}
if !filelock.IsExist(srcAbsPath) {
ret.Code = http.StatusNotFound
ret.Msg = "the [path] file or directory does not exist"
return
}
newPath := arg["newPath"].(string)
newPath = filepath.Join(util.WorkspaceDir, newPath)
if filelock.IsExist(newPath) {
ret.Code = 409
destPath := arg["newPath"].(string)
destAbsPath, err := util.GetAbsPathInWorkspace(destPath)
if nil != err {
ret.Code = http.StatusForbidden
ret.Msg = err.Error()
c.JSON(http.StatusAccepted, ret)
return
}
if filelock.IsExist(destAbsPath) {
ret.Code = http.StatusConflict
ret.Msg = "the [newPath] file or directory already exists"
return
}
if err := filelock.Rename(filePath, newPath); nil != err {
logging.LogErrorf("rename file [%s] to [%s] failed: %s", filePath, newPath, err)
ret.Code = 500
if err := filelock.Rename(srcAbsPath, destAbsPath); nil != err {
logging.LogErrorf("rename file [%s] to [%s] failed: %s", srcAbsPath, destAbsPath, err)
ret.Code = http.StatusInternalServerError
ret.Msg = err.Error()
return
}
@ -237,22 +261,27 @@ func removeFile(c *gin.Context) {
}
filePath := arg["path"].(string)
filePath = filepath.Join(util.WorkspaceDir, filePath)
_, err := os.Stat(filePath)
fileAbsPath, err := util.GetAbsPathInWorkspace(filePath)
if nil != err {
ret.Code = http.StatusForbidden
ret.Msg = err.Error()
return
}
_, err = os.Stat(fileAbsPath)
if os.IsNotExist(err) {
ret.Code = 404
ret.Code = http.StatusNotFound
return
}
if nil != err {
logging.LogErrorf("stat [%s] failed: %s", filePath, err)
ret.Code = 500
logging.LogErrorf("stat [%s] failed: %s", fileAbsPath, err)
ret.Code = http.StatusInternalServerError
ret.Msg = err.Error()
return
}
if err = filelock.Remove(filePath); nil != err {
logging.LogErrorf("remove [%s] failed: %s", filePath, err)
ret.Code = 500
if err = filelock.Remove(fileAbsPath); nil != err {
logging.LogErrorf("remove [%s] failed: %s", fileAbsPath, err)
ret.Code = http.StatusInternalServerError
ret.Msg = err.Error()
return
}
@ -262,30 +291,36 @@ func putFile(c *gin.Context) {
ret := gulu.Ret.NewResult()
defer c.JSON(http.StatusOK, ret)
var err error
filePath := c.PostForm("path")
filePath = filepath.Join(util.WorkspaceDir, filePath)
fileAbsPath, err := util.GetAbsPathInWorkspace(filePath)
if nil != err {
ret.Code = http.StatusForbidden
ret.Msg = err.Error()
return
}
isDirStr := c.PostForm("isDir")
isDir, _ := strconv.ParseBool(isDirStr)
var err error
if isDir {
err = os.MkdirAll(filePath, 0755)
err = os.MkdirAll(fileAbsPath, 0755)
if nil != err {
logging.LogErrorf("make a dir [%s] failed: %s", filePath, err)
logging.LogErrorf("make dir [%s] failed: %s", fileAbsPath, err)
}
} else {
fileHeader, _ := c.FormFile("file")
if nil == fileHeader {
logging.LogErrorf("form file is nil [path=%s]", filePath)
ret.Code = 400
logging.LogErrorf("form file is nil [path=%s]", fileAbsPath)
ret.Code = http.StatusBadRequest
ret.Msg = "form file is nil"
return
}
for {
dir := filepath.Dir(filePath)
dir := filepath.Dir(fileAbsPath)
if err = os.MkdirAll(dir, 0755); nil != err {
logging.LogErrorf("put a file [%s] make dir [%s] failed: %s", filePath, dir, err)
logging.LogErrorf("put file [%s] make dir [%s] failed: %s", fileAbsPath, dir, err)
break
}
@ -303,9 +338,9 @@ func putFile(c *gin.Context) {
break
}
err = filelock.WriteFile(filePath, data)
err = filelock.WriteFile(fileAbsPath, data)
if nil != err {
logging.LogErrorf("put a file [%s] failed: %s", filePath, err)
logging.LogErrorf("write file [%s] failed: %s", fileAbsPath, err)
break
}
break
@ -323,15 +358,15 @@ func putFile(c *gin.Context) {
modTimeInt, parseErr := strconv.ParseInt(modTimeStr, 10, 64)
if nil != parseErr {
logging.LogErrorf("parse mod time [%s] failed: %s", modTimeStr, parseErr)
ret.Code = 500
ret.Code = http.StatusInternalServerError
ret.Msg = parseErr.Error()
return
}
modTime = millisecond2Time(modTimeInt)
}
if err = os.Chtimes(filePath, modTime, modTime); nil != err {
if err = os.Chtimes(fileAbsPath, modTime, modTime); nil != err {
logging.LogErrorf("change time failed: %s", err)
ret.Code = 500
ret.Code = http.StatusInternalServerError
ret.Msg = err.Error()
return
}

1
kernel/api/router.go
View file

@ -375,6 +375,7 @@ func ServeAPI(ginServer *gin.Engine) {
ginServer.Handle("POST", "/api/snippet/removeSnippet", model.CheckAuth, model.CheckReadonly, removeSnippet)
ginServer.Handle("POST", "/api/av/renderAttributeView", model.CheckAuth, renderAttributeView)
ginServer.Handle("POST", "/api/av/renderHistoryAttributeView", model.CheckAuth, renderHistoryAttributeView)
ginServer.Handle("POST", "/api/av/getAttributeViewKeys", model.CheckAuth, getAttributeViewKeys)
ginServer.Handle("POST", "/api/av/setAttributeViewBlockAttr", model.CheckAuth, model.CheckReadonly, setAttributeViewBlockAttr)

50
kernel/model/attribute_view.go
View file

@ -18,7 +18,10 @@ package model
import (
"bytes"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"text/template"
"time"
@ -181,6 +184,46 @@ func GetBlockAttributeViewKeys(blockID string) (ret []*BlockAttributeViewKeys) {
return
}
func RenderHistoryAttributeView(avID, created string) (viewable av.Viewable, attrView *av.AttributeView, err error) {
createdUnix, parseErr := strconv.ParseInt(created, 10, 64)
if nil != parseErr {
logging.LogErrorf("parse created [%s] failed: %s", created, parseErr)
return
}
dirPrefix := time.Unix(createdUnix, 0).Format("2006-01-02-150405")
globPath := filepath.Join(util.HistoryDir, dirPrefix+"*")
matches, err := filepath.Glob(globPath)
if nil != err {
logging.LogErrorf("glob [%s] failed: %s", globPath, err)
return
}
if 1 > len(matches) {
return
}
historyDir := matches[0]
avJSONPath := filepath.Join(historyDir, "storage", "av", avID+".json")
if !gulu.File.IsExist(avJSONPath) {
return
}
data, readErr := os.ReadFile(avJSONPath)
if nil != readErr {
logging.LogErrorf("read attribute view [%s] failed: %s", avID, readErr)
return
}
attrView = &av.AttributeView{}
if err = gulu.JSON.UnmarshalJSON(data, attrView); nil != err {
logging.LogErrorf("unmarshal attribute view [%s] failed: %s", avID, err)
return
}
viewable, err = renderAttributeView(attrView)
return
}
func RenderAttributeView(avID string) (viewable av.Viewable, attrView *av.AttributeView, err error) {
waitForSyncingStorages()
@ -198,12 +241,17 @@ func RenderAttributeView(avID string) (viewable av.Viewable, attrView *av.Attrib
return
}
viewable, err = renderAttributeView(attrView)
return
}
func renderAttributeView(attrView *av.AttributeView) (viewable av.Viewable, err error) {
if 1 > len(attrView.Views) {
view := av.NewView()
attrView.Views = append(attrView.Views, view)
attrView.ViewID = view.ID
if err = av.SaveAttributeView(attrView); nil != err {
logging.LogErrorf("save attribute view [%s] failed: %s", avID, err)
logging.LogErrorf("save attribute view [%s] failed: %s", attrView.ID, err)
return
}
}

10
kernel/model/file.go
View file

@ -1409,6 +1409,16 @@ func removeDoc(box *Box, p string, luteEngine *lute.Lute) {
return
}
// 关联的属性视图也要复制到历史中 https://github.com/siyuan-note/siyuan/issues/9567
avNodes := tree.Root.ChildrenByType(ast.NodeAttributeView)
for _, avNode := range avNodes {
srcAvPath := filepath.Join(util.DataDir, "storage", "av", avNode.AttributeViewID+".json")
destAvPath := filepath.Join(historyDir, "storage", "av", avNode.AttributeViewID+".json")
if copyErr := filelock.Copy(srcAvPath, destAvPath); nil != copyErr {
logging.LogErrorf("copy av [%s] failed: %s", srcAvPath, copyErr)
}
}
copyDocAssetsToDataAssets(box.ID, p)
removeIDs := treenode.RootChildIDs(tree.ID)

32
kernel/model/history.go
View file

@ -394,6 +394,9 @@ func buildSearchHistoryQueryFilter(query, op, box, table string, typ int) (stmt
} else if HistoryTypeAsset == typ {
stmt += " AND path LIKE '%/assets/%'"
}
ago := time.Now().Add(-24 * time.Hour * time.Duration(Conf.Editor.HistoryRetentionDays))
stmt += " AND created > '" + fmt.Sprintf("%d", ago.Unix()) + "'"
return
}
@ -461,6 +464,7 @@ func (box *Box) generateDocHistory0() {
return
}
luteEngine := util.NewLute()
for _, file := range files {
historyPath := filepath.Join(historyDir, box.ID, strings.TrimPrefix(file, filepath.Join(util.DataDir, box.ID)))
if err = os.MkdirAll(filepath.Dir(historyPath), 0755); nil != err {
@ -478,6 +482,23 @@ func (box *Box) generateDocHistory0() {
logging.LogErrorf("generate history failed: %s", err)
return
}
if strings.HasSuffix(file, ".sy") {
tree, loadErr := loadTree(file, luteEngine)
if nil != loadErr {
logging.LogErrorf("load tree [%s] failed: %s", file, loadErr)
} else {
// 关联的属性视图也要复制到历史中 https://github.com/siyuan-note/siyuan/issues/9567
avNodes := tree.Root.ChildrenByType(ast.NodeAttributeView)
for _, avNode := range avNodes {
srcAvPath := filepath.Join(util.DataDir, "storage", "av", avNode.AttributeViewID+".json")
destAvPath := filepath.Join(historyDir, "storage", "av", avNode.AttributeViewID+".json")
if copyErr := filelock.Copy(srcAvPath, destAvPath); nil != copyErr {
logging.LogErrorf("copy av [%s] failed: %s", srcAvPath, copyErr)
}
}
}
}
}
indexHistoryDir(filepath.Base(historyDir), util.NewLute())
@ -497,6 +518,7 @@ func clearOutdatedHistoryDir(historyDir string) {
}
now := time.Now()
ago := now.Add(-24 * time.Hour * time.Duration(Conf.Editor.HistoryRetentionDays)).Unix()
var removes []string
for _, dir := range dirs {
dirInfo, err := dir.Info()
@ -504,7 +526,7 @@ func clearOutdatedHistoryDir(historyDir string) {
logging.LogErrorf("read history dir [%s] failed: %s", dir.Name(), err)
continue
}
if Conf.Editor.HistoryRetentionDays < int(now.Sub(dirInfo.ModTime()).Hours()/24) {
if dirInfo.ModTime().Unix() < ago {
removes = append(removes, filepath.Join(historyDir, dir.Name()))
}
}
@ -514,10 +536,10 @@ func clearOutdatedHistoryDir(historyDir string) {
continue
}
//logging.LogInfof("auto removed history dir [%s]", dir)
// 清理历史库
sql.DeleteHistoriesByPathPrefixQueue(dir)
}
// 清理历史库
sql.DeleteOutdatedHistories(fmt.Sprintf("%d", ago))
}
var boxLatestHistoryTime = map[string]time.Time{}
@ -540,7 +562,7 @@ func (box *Box) recentModifiedDocs() (ret []string) {
}
if info.ModTime().After(latestHistoryTime) {
ret = append(ret, filepath.Join(path))
ret = append(ret, path)
}
return nil
})

6
kernel/sql/history.go
View file

@ -104,9 +104,9 @@ func queryHistory(query string, args ...interface{}) (*sql.Rows, error) {
return historyDB.Query(query, args...)
}
func deleteHistoriesByPathPrefix(tx *sql.Tx, pathPrefix string, context map[string]interface{}) (err error) {
stmt := "DELETE FROM histories_fts_case_insensitive WHERE path LIKE ?"
if err = execStmtTx(tx, stmt, pathPrefix+"%"); nil != err {
func deleteOutdatedHistories(tx *sql.Tx, before string, context map[string]interface{}) (err error) {
stmt := "DELETE FROM histories_fts_case_insensitive WHERE created < ?"
if err = execStmtTx(tx, stmt, before); nil != err {
return
}
return

14
kernel/sql/queue_history.go
View file

@ -39,10 +39,10 @@ var (
type historyDBQueueOperation struct {
inQueueTime time.Time
action string // index/deletePathPrefix
action string // index/deleteOutdated
histories []*History // index
pathPrefix string // deletePathPrefix
histories []*History // index
before string // deleteOutdated
}
func FlushHistoryTxJob() {
@ -111,8 +111,8 @@ func execHistoryOp(op *historyDBQueueOperation, tx *sql.Tx, context map[string]i
switch op.action {
case "index":
err = insertHistories(tx, op.histories, context)
case "deletePathPrefix":
err = deleteHistoriesByPathPrefix(tx, op.pathPrefix, context)
case "deleteOutdated":
err = deleteOutdatedHistories(tx, op.before, context)
default:
msg := fmt.Sprintf("unknown history operation [%s]", op.action)
logging.LogErrorf(msg)
@ -121,11 +121,11 @@ func execHistoryOp(op *historyDBQueueOperation, tx *sql.Tx, context map[string]i
return
}
func DeleteHistoriesByPathPrefixQueue(pathPrefix string) {
func DeleteOutdatedHistories(before string) {
historyDBQueueLock.Lock()
defer historyDBQueueLock.Unlock()
newOp := &historyDBQueueOperation{inQueueTime: time.Now(), action: "deletePathPrefix", pathPrefix: pathPrefix}
newOp := &historyDBQueueOperation{inQueueTime: time.Now(), action: "deleteOutdated", before: before}
historyOperationQueue = append(historyOperationQueue, newOp)
}

9
kernel/util/path.go
View file

@ -265,3 +265,12 @@ func IsDisplayableAsset(p string) bool {
}
return false
}
func GetAbsPathInWorkspace(relPath string) (string, error) {
absPath := filepath.Join(WorkspaceDir, relPath)
if IsSubPath(WorkspaceDir, absPath) {
return absPath, nil
} else {
return "", os.ErrPermission
}
}