0ct0pu5/siyuan
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

⚡ Improve data sync performance for booting https://github.com/siyuan-note/siyuan/issues/13216

Browse source
This commit is contained in:
Daniel 2024-11-21 10:59:29 +08:00
parent eba4dfa0da
commit 89f1887c3b
No known key found for this signature in database
GPG key ID: 86211BA83DF03017
16 changed files with 153 additions and 113 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
kernel/cache/asset.go vendored
View file

@ -72,21 +72,27 @@ func LoadAssets() {
assetsCache = map[string]*Asset{}
assets := util.GetDataAssetsAbsPath()
filelock.Walk(assets, func(path string, info fs.FileInfo, err error) error {
if nil == info {
filelock.Walk(assets, func(path string, d fs.DirEntry, err error) error {
if nil != err || nil == d {
return err
}
if info.IsDir() {
if strings.HasPrefix(info.Name(), ".") {
if d.IsDir() {
if strings.HasPrefix(d.Name(), ".") {
return filepath.SkipDir
}
return nil
}
if strings.HasSuffix(info.Name(), ".sya") || strings.HasPrefix(info.Name(), ".") || filelock.IsHidden(path) {
if strings.HasSuffix(d.Name(), ".sya") || strings.HasPrefix(d.Name(), ".") || filelock.IsHidden(path) {
return nil
}
hName := util.RemoveID(info.Name())
info, err := d.Info()
if nil != err {
logging.LogErrorf("load assets failed: %s", err)
return nil
}
hName := util.RemoveID(d.Name())
path = "assets" + filepath.ToSlash(strings.TrimPrefix(path, assets))
assetsCache[path] = &Asset{
HName: hName,

6
kernel/go.mod
View file

@ -56,10 +56,10 @@ require (
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
github.com/sashabaranov/go-openai v1.29.1
github.com/shirou/gopsutil/v3 v3.24.5
github.com/siyuan-note/dejavu v0.0.0-20241120170449-12bce3fc2a58
github.com/siyuan-note/dejavu v0.0.0-20241121022337-e418aabe08a7
github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4
github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97
github.com/siyuan-note/filelock v0.0.0-20240724034355-d1ed7bf21d04
github.com/siyuan-note/filelock v0.0.0-20241121021809-1cf0626a7d64
github.com/siyuan-note/httpclient v0.0.0-20241113084556-839baaab03f6
github.com/siyuan-note/logging v0.0.0-20240505035402-6430d57006a2
github.com/siyuan-note/riff v0.0.0-20240912073907-1e89a5f8e7dc
@ -137,7 +137,7 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/onsi/ginkgo/v2 v2.21.0 // indirect
github.com/onsi/ginkgo/v2 v2.22.0 // indirect
github.com/otiai10/gosseract/v2 v2.4.1 // indirect
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
github.com/pkg/errors v0.9.1 // indirect

12
kernel/go.sum
View file

@ -267,8 +267,8 @@ github.com/olahol/melody v1.2.1/go.mod h1:GgkTl6Y7yWj/HtfD48Q5vLKPVoZOH+Qqgfa7Cv
github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM=
github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg=
github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8=
github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc=
github.com/open-spaced-repetition/go-fsrs/v3 v3.2.0 h1:lDY1dURLg5xYGbCcz9bxB7c+1v36+wZRYnSHOdQloPA=
@ -340,14 +340,14 @@ github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+D
github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d h1:lvCTyBbr36+tqMccdGMwuEU+hjux/zL6xSmf5S9ITaA=
github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8=
github.com/siyuan-note/dejavu v0.0.0-20241120170449-12bce3fc2a58 h1:P7dolAfpWlMW5LTqY/4uG+g2q2WG12/+EhbZcIVBFiE=
github.com/siyuan-note/dejavu v0.0.0-20241120170449-12bce3fc2a58/go.mod h1:sVINGoilQS1l5ZQJJBHzUwKmyHhx8qdQps7gKqZVbgU=
github.com/siyuan-note/dejavu v0.0.0-20241121022337-e418aabe08a7 h1:hbd6Txaysk2B27vybCQEeGgVDcVMu6Z+yYoWalHJOz4=
github.com/siyuan-note/dejavu v0.0.0-20241121022337-e418aabe08a7/go.mod h1:KBNaIdV5TSJmWrdof7C7A46RhB0IVm8hofDox0zgMqA=
github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4 h1:kJaw5L/evyW6LcB9IQT8PR4ppx8JVqOFP9Ix3rfwSrc=
github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4/go.mod h1:UYcCCY+0wh+GmUoDOaO63j1sV5lgy7laLAk1XhEiUis=
github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97 h1:lM5v8BfNtbOL5jYwhCdMYBcYtr06IYBKjjSLAPMKTM8=
github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97/go.mod h1:1/nGgthl89FPA7GzAcEWKl6zRRnfgyTjzLZj9bW7kuw=
github.com/siyuan-note/filelock v0.0.0-20240724034355-d1ed7bf21d04 h1:aoXvEO6BMqm6L0EnTjRhB4ynQIyJvHpqz+Ue3g0D3a0=
github.com/siyuan-note/filelock v0.0.0-20240724034355-d1ed7bf21d04/go.mod h1:iqFhf4EDKy4MjQgT6RQJ6Z6NSW662NS0PR40K1qdSpE=
github.com/siyuan-note/filelock v0.0.0-20241121021809-1cf0626a7d64 h1:w4i1AumKqsUgMcuNKw5FkJjAWK8jXVKpvW1BkAJvB+0=
github.com/siyuan-note/filelock v0.0.0-20241121021809-1cf0626a7d64/go.mod h1:QUxfb/zE/lMrpiGBV9MBT5c5NKQanGvpdBXtMwWjTD0=
github.com/siyuan-note/httpclient v0.0.0-20241113084556-839baaab03f6 h1:IzvOUPUc6YbtLgQuj3rVVIdbn1HzDOGMsTLLb9yN0Wo=
github.com/siyuan-note/httpclient v0.0.0-20241113084556-839baaab03f6/go.mod h1:cMXV74/pjFOvH3zIwQW/SyH3gZw8YzTBbqt7dxof69k=
github.com/siyuan-note/logging v0.0.0-20240505035402-6430d57006a2 h1:/2+tlOThVB86RxSLeW0JFw2ISUrH2ZFRg15ULGAUGAE=

10
kernel/model/asset_content.go
View file

@ -387,13 +387,13 @@ func (searcher *AssetsSearcher) FullIndex() {
}
var results []*AssetParseResult
filelock.Walk(assetsDir, func(absPath string, info fs.FileInfo, err error) error {
filelock.Walk(assetsDir, func(absPath string, d fs.DirEntry, err error) error {
if err != nil {
logging.LogErrorf("walk dir [%s] failed: %s", absPath, err)
return err
}
if info.IsDir() {
if d.IsDir() {
return nil
}
@ -410,6 +410,12 @@ func (searcher *AssetsSearcher) FullIndex() {
return nil
}
info, err := d.Info()
if err != nil {
logging.LogErrorf("stat file [%s] failed: %s", absPath, err)
return nil
}
result.Path = "assets" + filepath.ToSlash(strings.TrimPrefix(absPath, assetsDir))
result.Size = info.Size()
result.Updated = info.ModTime().Unix()

38
kernel/model/assets.go
View file

@ -311,9 +311,9 @@ func GetAssetAbsPath(relativePath string) (ret string, err error) {
// 在笔记本下搜索
for _, notebook := range notebooks {
notebookAbsPath := filepath.Join(util.DataDir, notebook.ID)
filelock.Walk(notebookAbsPath, func(path string, info fs.FileInfo, _ error) error {
if isSkipFile(info.Name()) {
if info.IsDir() {
filelock.Walk(notebookAbsPath, func(path string, d fs.DirEntry, err error) error {
if isSkipFile(d.Name()) {
if d.IsDir() {
return filepath.SkipDir
}
return nil
@ -1270,12 +1270,12 @@ func allAssetAbsPaths() (assetsAbsPathMap map[string]string, err error) {
// 笔记本 assets
for _, notebook := range notebooks {
notebookAbsPath := filepath.Join(util.DataDir, notebook.ID)
filelock.Walk(notebookAbsPath, func(path string, info fs.FileInfo, err error) error {
filelock.Walk(notebookAbsPath, func(path string, d fs.DirEntry, err error) error {
if notebookAbsPath == path {
return nil
}
if isSkipFile(info.Name()) {
if info.IsDir() {
if isSkipFile(d.Name()) {
if d.IsDir() {
return filepath.SkipDir
}
return nil
@ -1286,20 +1286,20 @@ func allAssetAbsPaths() (assetsAbsPathMap map[string]string, err error) {
return nil
}
if info.IsDir() && "assets" == info.Name() {
filelock.Walk(path, func(assetPath string, info fs.FileInfo, err error) error {
if d.IsDir() && "assets" == d.Name() {
filelock.Walk(path, func(assetPath string, d fs.DirEntry, err error) error {
if path == assetPath {
return nil
}
if isSkipFile(info.Name()) {
if info.IsDir() {
if isSkipFile(d.Name()) {
if d.IsDir() {
return filepath.SkipDir
}
return nil
}
relPath := filepath.ToSlash(assetPath)
relPath = relPath[strings.Index(relPath, "assets/"):]
if info.IsDir() {
if d.IsDir() {
relPath += "/"
}
assetsAbsPathMap[relPath] = assetPath
@ -1313,13 +1313,13 @@ func allAssetAbsPaths() (assetsAbsPathMap map[string]string, err error) {
// 全局 assets
dataAssetsAbsPath := util.GetDataAssetsAbsPath()
filelock.Walk(dataAssetsAbsPath, func(assetPath string, info fs.FileInfo, err error) error {
filelock.Walk(dataAssetsAbsPath, func(assetPath string, d fs.DirEntry, err error) error {
if dataAssetsAbsPath == assetPath {
return nil
}
if isSkipFile(info.Name()) {
if info.IsDir() {
if isSkipFile(d.Name()) {
if d.IsDir() {
return filepath.SkipDir
}
return nil
@ -1332,7 +1332,7 @@ func allAssetAbsPaths() (assetsAbsPathMap map[string]string, err error) {
relPath := filepath.ToSlash(assetPath)
relPath = relPath[strings.Index(relPath, "assets/"):]
if info.IsDir() {
if d.IsDir() {
relPath += "/"
}
assetsAbsPathMap[relPath] = assetPath
@ -1356,14 +1356,12 @@ func copyDocAssetsToDataAssets(boxID, parentDocPath string) {
func copyAssetsToDataAssets(rootPath string) {
var assetsDirPaths []string
filelock.Walk(rootPath, func(path string, info fs.FileInfo, err error) error {
if rootPath == path || nil == info {
filelock.Walk(rootPath, func(path string, d fs.DirEntry, err error) error {
if nil != err || rootPath == path || nil == d {
return nil
}
isDir := info.IsDir()
name := info.Name()
isDir, name := d.IsDir(), d.Name()
if isSkipFile(name) {
if isDir {
return filepath.SkipDir

5
kernel/model/file.go
View file

@ -20,6 +20,7 @@ import (
"bytes"
"errors"
"fmt"
"io/fs"
"os"
"path"
"path/filepath"
@ -1461,11 +1462,11 @@ func MoveDocs(fromPaths []string, toBoxID, toPath string, callback interface{})
func countSubDocs(box, p string) (ret int) {
p = strings.TrimSuffix(p, ".sy")
_ = filepath.Walk(filepath.Join(util.DataDir, box, p), func(path string, info os.FileInfo, err error) error {
_ = filelock.Walk(filepath.Join(util.DataDir, box, p), func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if info.IsDir() {
if d.IsDir() {
return nil
}
if strings.HasSuffix(path, ".sy") {

19
kernel/model/history.go
View file

@ -673,21 +673,26 @@ var boxLatestHistoryTime = map[string]time.Time{}
func (box *Box) recentModifiedDocs() (ret []string) {
latestHistoryTime := boxLatestHistoryTime[box.ID]
filelock.Walk(filepath.Join(util.DataDir, box.ID), func(path string, info fs.FileInfo, err error) error {
if nil == info {
filelock.Walk(filepath.Join(util.DataDir, box.ID), func(path string, d fs.DirEntry, err error) error {
if nil != err || nil == d {
return nil
}
if isSkipFile(info.Name()) {
if info.IsDir() {
if isSkipFile(d.Name()) {
if d.IsDir() {
return filepath.SkipDir
}
return nil
}
if info.IsDir() {
if d.IsDir() {
return nil
}
info, err := d.Info()
if nil != err {
return err
}
if info.ModTime().After(latestHistoryTime) {
ret = append(ret, path)
}
@ -817,8 +822,8 @@ func indexHistoryDir(name string, luteEngine *lute.Lute) {
entryPath := filepath.Join(util.HistoryDir, name)
var docs, assets []string
filelock.Walk(entryPath, func(path string, info os.FileInfo, err error) error {
if strings.HasSuffix(info.Name(), ".sy") {
filelock.Walk(entryPath, func(path string, d fs.DirEntry, err error) error {
if strings.HasSuffix(d.Name(), ".sy") {
docs = append(docs, path)
} else if strings.Contains(path, "assets"+string(os.PathSeparator)) {
assets = append(assets, path)

52
kernel/model/import.go
View file

@ -118,12 +118,12 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
defer os.RemoveAll(unzipPath)
var syPaths []string
filelock.Walk(unzipPath, func(path string, info fs.FileInfo, err error) error {
filelock.Walk(unzipPath, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if !info.IsDir() && strings.HasSuffix(info.Name(), ".sy") {
if !d.IsDir() && strings.HasSuffix(d.Name(), ".sy") {
syPaths = append(syPaths, path)
}
return nil
@ -226,14 +226,14 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
renameAvPaths := map[string]string{}
if gulu.File.IsExist(storageAvDir) {
// 重新生成数据库数据
filelock.Walk(storageAvDir, func(path string, info fs.FileInfo, err error) error {
if !strings.HasSuffix(path, ".json") || !ast.IsNodeIDPattern(strings.TrimSuffix(info.Name(), ".json")) {
filelock.Walk(storageAvDir, func(path string, d fs.DirEntry, err error) error {
if !strings.HasSuffix(path, ".json") || !ast.IsNodeIDPattern(strings.TrimSuffix(d.Name(), ".json")) {
return nil
}
// 重命名数据库
newAvID := ast.NewNodeID()
oldAvID := strings.TrimSuffix(info.Name(), ".json")
oldAvID := strings.TrimSuffix(d.Name(), ".json")
newPath := filepath.Join(filepath.Dir(path), newAvID+".json")
renameAvPaths[path] = newPath
avIDs[oldAvID] = newAvID
@ -460,12 +460,12 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
// 重命名文件路径
renamePaths := map[string]string{}
filelock.Walk(unzipRootPath, func(path string, info fs.FileInfo, err error) error {
filelock.Walk(unzipRootPath, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if info.IsDir() && ast.IsNodeIDPattern(info.Name()) {
if d.IsDir() && ast.IsNodeIDPattern(d.Name()) {
renamePaths[path] = path
}
return nil
@ -535,8 +535,8 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
// 将包含的资源文件统一移动到 data/assets/ 下
var assetsDirs []string
filelock.Walk(unzipRootPath, func(path string, info fs.FileInfo, err error) error {
if strings.Contains(path, "assets") && info.IsDir() {
filelock.Walk(unzipRootPath, func(path string, d fs.DirEntry, err error) error {
if strings.Contains(path, "assets") && d.IsDir() {
assetsDirs = append(assetsDirs, path)
}
return nil
@ -570,15 +570,15 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
}
var treePaths []string
filelock.Walk(unzipRootPath, func(path string, info fs.FileInfo, err error) error {
if info.IsDir() {
if strings.HasPrefix(info.Name(), ".") {
filelock.Walk(unzipRootPath, func(path string, d fs.DirEntry, err error) error {
if d.IsDir() {
if strings.HasPrefix(d.Name(), ".") {
return filepath.SkipDir
}
return nil
}
if !strings.HasSuffix(info.Name(), ".sy") {
if !strings.HasSuffix(d.Name(), ".sy") {
return nil
}
@ -699,18 +699,18 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) {
if gulu.File.IsDir(localPath) { // 导入文件夹
// 收集所有资源文件
assets := map[string]string{}
filelock.Walk(localPath, func(currentPath string, info os.FileInfo, walkErr error) error {
filelock.Walk(localPath, func(currentPath string, d fs.DirEntry, err error) error {
if localPath == currentPath {
return nil
}
if strings.HasPrefix(info.Name(), ".") {
if info.IsDir() {
if strings.HasPrefix(d.Name(), ".") {
if d.IsDir() {
return filepath.SkipDir
}
return nil
}
if !strings.HasSuffix(info.Name(), ".md") && !strings.HasSuffix(info.Name(), ".markdown") {
if !strings.HasSuffix(d.Name(), ".md") && !strings.HasSuffix(d.Name(), ".markdown") {
assets[currentPath] = currentPath
return nil
}
@ -721,9 +721,9 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) {
assetsDone := map[string]string{}
// md 转换 sy
filelock.Walk(localPath, func(currentPath string, info os.FileInfo, walkErr error) error {
if strings.HasPrefix(info.Name(), ".") {
if info.IsDir() {
filelock.Walk(localPath, func(currentPath string, d fs.DirEntry, err error) error {
if strings.HasPrefix(d.Name(), ".") {
if d.IsDir() {
return filepath.SkipDir
}
return nil
@ -731,10 +731,10 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) {
var tree *parse.Tree
var ext string
title := info.Name()
if !info.IsDir() {
ext = path.Ext(info.Name())
title = strings.TrimSuffix(info.Name(), ext)
title := d.Name()
if !d.IsDir() {
ext = path.Ext(d.Name())
title = strings.TrimSuffix(d.Name(), ext)
}
id := ast.NewNodeID()
@ -759,7 +759,7 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) {
id = strings.TrimSuffix(path.Base(targetPath), ".sy")
}
if info.IsDir() {
if d.IsDir() {
if subMdFiles := util.GetFilePathsByExts(currentPath, []string{".md", ".markdown"}); 1 > len(subMdFiles) {
// 如果该文件夹中不包含 Markdown 文件则不处理 https://github.com/siyuan-note/siyuan/issues/11567
return nil
@ -776,7 +776,7 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) {
return nil
}
if !strings.HasSuffix(info.Name(), ".md") && !strings.HasSuffix(info.Name(), ".markdown") {
if !strings.HasSuffix(d.Name(), ".md") && !strings.HasSuffix(d.Name(), ".markdown") {
return nil
}

2
kernel/model/index.go
View file

@ -82,7 +82,7 @@ func RemoveIndexes(paths []string) {
func listSyFiles(dir string) (ret []string) {
dirPath := filepath.Join(util.DataDir, dir)
err := filelock.Walk(dirPath, func(path string, d fs.FileInfo, err error) error {
err := filelock.Walk(dirPath, func(path string, d fs.DirEntry, err error) error {
if err != nil {
logging.LogWarnf("walk dir [%s] failed: %s", dirPath, err)
return err

27
kernel/model/index_fix.go
View file

@ -18,6 +18,7 @@ package model
import (
"fmt"
"io/fs"
"os"
"path"
"path/filepath"
@ -151,22 +152,22 @@ func resetDuplicateBlocksOnFileSys() {
boxPath := filepath.Join(util.DataDir, box.ID)
var duplicatedTrees []*parse.Tree
filelock.Walk(boxPath, func(path string, info os.FileInfo, err error) error {
if nil == info {
filelock.Walk(boxPath, func(path string, d fs.DirEntry, err error) error {
if err != nil || nil == d {
return nil
}
if info.IsDir() {
if d.IsDir() {
if boxPath == path {
// 跳过笔记本文件夹
return nil
}
if strings.HasPrefix(info.Name(), ".") {
if strings.HasPrefix(d.Name(), ".") {
return filepath.SkipDir
}
if !ast.IsNodeIDPattern(info.Name()) {
if !ast.IsNodeIDPattern(d.Name()) {
return nil
}
return nil
@ -176,7 +177,7 @@ func resetDuplicateBlocksOnFileSys() {
return nil
}
if !ast.IsNodeIDPattern(strings.TrimSuffix(info.Name(), ".sy")) {
if !ast.IsNodeIDPattern(strings.TrimSuffix(d.Name(), ".sy")) {
logging.LogWarnf("invalid .sy file name [%s]", path)
box.moveCorruptedData(path)
return nil
@ -285,18 +286,18 @@ func fixBlockTreeByFileSys() {
for _, box := range boxes {
boxPath := filepath.Join(util.DataDir, box.ID)
var paths []string
filelock.Walk(boxPath, func(path string, info os.FileInfo, err error) error {
filelock.Walk(boxPath, func(path string, d fs.DirEntry, err error) error {
if nil != err || nil == d {
return nil
}
if boxPath == path {
// 跳过根路径（笔记本文件夹）
return nil
}
if nil == info {
return nil
}
if info.IsDir() {
if strings.HasPrefix(info.Name(), ".") {
if d.IsDir() {
if strings.HasPrefix(d.Name(), ".") {
return filepath.SkipDir
}
return nil

6
kernel/model/template.go
View file

@ -95,10 +95,10 @@ func SearchTemplate(keyword string) (ret []*Block) {
if group.IsDir() {
var templateBlocks []*Block
templateDir := filepath.Join(templates, group.Name())
filelock.Walk(templateDir, func(path string, info fs.FileInfo, err error) error {
name := strings.ToLower(info.Name())
filelock.Walk(templateDir, func(path string, d fs.DirEntry, err error) error {
name := strings.ToLower(d.Name())
if strings.HasPrefix(name, ".") {
if info.IsDir() {
if d.IsDir() {
return filepath.SkipDir
}
return nil

20
kernel/model/tree.go
View file

@ -130,15 +130,19 @@ func resetTree(tree *parse.Tree, titleSuffix string, removeAvBinding bool) {
func pagedPaths(localPath string, pageSize int) (ret map[int][]string) {
ret = map[int][]string{}
page := 1
filelock.Walk(localPath, func(path string, info fs.FileInfo, err error) error {
if info.IsDir() {
if strings.HasPrefix(info.Name(), ".") {
filelock.Walk(localPath, func(path string, d fs.DirEntry, err error) error {
if nil != err || nil == d {
return nil
}
if d.IsDir() {
if strings.HasPrefix(d.Name(), ".") {
return filepath.SkipDir
}
return nil
}
if !strings.HasSuffix(info.Name(), ".sy") {
if !strings.HasSuffix(d.Name(), ".sy") {
return nil
}
@ -248,15 +252,15 @@ func searchTreeInFilesystem(rootID string) {
logging.LogWarnf("searching tree on filesystem [rootID=%s]", rootID)
var treePath string
filepath.Walk(util.DataDir, func(path string, info fs.FileInfo, err error) error {
if info.IsDir() {
if strings.HasPrefix(info.Name(), ".") {
filelock.Walk(util.DataDir, func(path string, d fs.DirEntry, err error) error {
if d.IsDir() {
if strings.HasPrefix(d.Name(), ".") {
return filepath.SkipDir
}
return nil
}
if !strings.HasSuffix(info.Name(), ".sy") {
if !strings.HasSuffix(d.Name(), ".sy") {
return nil
}

2
kernel/treenode/tree.go
View file

@ -96,7 +96,7 @@ func RootChildIDs(rootID string) (ret []string) {
if !gulu.File.IsDir(subFolder) {
return
}
filelock.Walk(subFolder, func(path string, info fs.FileInfo, err error) error {
filelock.Walk(subFolder, func(path string, d fs.DirEntry, err error) error {
if strings.HasSuffix(path, ".sy") {
name := filepath.Base(path)
id := strings.TrimSuffix(name, ".sy")

41
kernel/util/file.go
View file

@ -36,13 +36,13 @@ import (
)
func GetFilePathsByExts(dirPath string, exts []string) (ret []string) {
filelock.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
filelock.Walk(dirPath, func(path string, d fs.DirEntry, err error) error {
if err != nil {
logging.LogErrorf("get file paths by ext failed: %s", err)
return err
}
if info.IsDir() {
if d.IsDir() {
return nil
}
@ -275,13 +275,19 @@ func IsSubPath(absPath, toCheckPath string) bool {
}
func SizeOfDirectory(path string) (size int64, err error) {
err = filelock.Walk(path, func(_ string, info os.FileInfo, err error) error {
err = filelock.Walk(path, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
info, err := d.Info()
if err != nil {
logging.LogErrorf("size of dir [%s] failed: %s", path, err)
return err
}
if !info.IsDir() {
s := info.Size()
size += s
size += info.Size()
} else {
size += 4096
}
@ -294,7 +300,7 @@ func SizeOfDirectory(path string) (size int64, err error) {
}
func DataSize() (dataSize, assetsSize int64) {
filelock.Walk(DataDir, func(path string, info os.FileInfo, err error) error {
filelock.Walk(DataDir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
if os.IsNotExist(err) {
return nil
@ -302,6 +308,13 @@ func DataSize() (dataSize, assetsSize int64) {
logging.LogErrorf("size of data failed: %s", err)
return io.EOF
}
info, err := d.Info()
if err != nil {
logging.LogErrorf("size of data failed: %s", err)
return nil
}
if !info.IsDir() {
s := info.Size()
dataSize += s
@ -334,7 +347,7 @@ func IsReservedFilename(baseName string) bool {
return "assets" == baseName || "templates" == baseName || "widgets" == baseName || "emojis" == baseName || ".siyuan" == baseName || strings.HasPrefix(baseName, ".")
}
func WalkWithSymlinks(root string, fn filepath.WalkFunc) error {
func WalkWithSymlinks(root string, fn fs.WalkDirFunc) error {
// 感谢 https://github.com/edwardrf/symwalk/blob/main/symwalk.go
rr, err := filepath.EvalSymlinks(root) // Find real base if there is any symlinks in the path
@ -346,23 +359,27 @@ func WalkWithSymlinks(root string, fn filepath.WalkFunc) error {
return filelock.Walk(rr, getWalkFn(visitedDirs, fn))
}
func getWalkFn(visitedDirs map[string]struct{}, fn filepath.WalkFunc) filepath.WalkFunc {
return func(path string, info os.FileInfo, err error) error {
func getWalkFn(visitedDirs map[string]struct{}, fn fs.WalkDirFunc) fs.WalkDirFunc {
return func(path string, d fs.DirEntry, err error) error {
if err != nil {
return fn(path, info, err)
return fn(path, d, err)
}
if info.IsDir() {
if d.IsDir() {
if _, ok := visitedDirs[path]; ok {
return filepath.SkipDir
}
visitedDirs[path] = struct{}{}
}
if err := fn(path, info, err); err != nil {
if err := fn(path, d, err); err != nil {
return err
}
info, err := d.Info()
if nil != err {
return err
}
if info.Mode()&os.ModeSymlink == 0 {
return nil
}

3
kernel/util/path.go
View file

@ -18,6 +18,7 @@ package util
import (
"bytes"
"io/fs"
"net"
"os"
"path"
@ -167,7 +168,7 @@ func GetChildDocDepth(treeAbsPath string) (ret int) {
baseDepth := strings.Count(filepath.ToSlash(treeAbsPath), "/")
depth := 1
filelock.Walk(dir, func(path string, info os.FileInfo, err error) error {
filelock.Walk(dir, func(path string, d fs.DirEntry, err error) error {
p := filepath.ToSlash(path)
currentDepth := strings.Count(p, "/")
if depth < currentDepth {

5
kernel/util/runtime.go
View file

@ -20,6 +20,7 @@ import (
"bytes"
"fmt"
"io"
"io/fs"
"math/rand"
"os"
"path/filepath"
@ -316,8 +317,8 @@ func isICloudPath(workspaceAbsPath string) (ret bool) {
// macOS 端对工作空间放置在 iCloud 路径下做检查 https://github.com/siyuan-note/siyuan/issues/7747
iCloudRoot := filepath.Join(HomeDir, "Library", "Mobile Documents")
WalkWithSymlinks(iCloudRoot, func(path string, info os.FileInfo, err error) error {
if !info.IsDir() {
WalkWithSymlinks(iCloudRoot, func(path string, d fs.DirEntry, err error) error {
if !d.IsDir() {
return nil
}