Merge remote-tracking branch 'origin/dev' into dev

.github/CONTRIBUTING.md

@@ -7,7 +7,7 @@
 
 ## User Interface
 
-Install pnpm: `npm install -g pnpm@9.1.1`
+Install pnpm: `npm install -g pnpm@9.12.1`
 
 <details>
 <summary>For China mainland</summary>

.github/CONTRIBUTING_zh_CN.md

@@ -7,7 +7,7 @@
 
 ## NPM 依赖
 
-安装 pnpm：`npm install -g pnpm@9.1.1`
+安装 pnpm：`npm install -g pnpm@9.12.1`
 
 <details>
 <summary>适用于中国大陆</summary>

app/package.json

@@ -4,7 +4,7 @@
   "description": "Refactor your thinking",
   "homepage": "https://b3log.org/siyuan",
   "main": "./electron/main.js",
-  "packageManager": "pnpm@9.1.1",
+  "packageManager": "pnpm@9.12.1",
   "scripts": {
     "lint": "eslint . --fix --cache",
     "dev": "webpack --mode development",
@@ -71,7 +71,7 @@
     "sass": "^1.53.0",
     "sass-loader": "^12.6.0",
     "typescript": "^4.7.4",
-    "webpack": "^5.76.0",
+    "webpack": "^5.94.0",
     "webpack-bundle-analyzer": "^4.5.0",
     "webpack-cli": "^4.10.0"
   },

app/src/config/repos.ts

@@ -99,6 +99,11 @@ const renderProvider = (provider: number) => {
         <option ${window.siyuan.config.sync.s3.skipTlsVerify ? "selected" : ""} value="true">Skip</option>
     </select>
 </div>
+<div class="b3-label b3-label--inner fn__flex">
+    <div class="fn__flex-center fn__size200">Concurrent Reqs</div>
+    <div class="fn__space"></div>
+    <input id="s3ConcurrentReqs" class="b3-text-field fn__block" type="number" min="1" max="16" value="${window.siyuan.config.sync.s3.concurrentReqs}">
+</div>
 <div class="b3-label b3-label--inner fn__flex">
     <div class="fn__flex-1"></div>
     <button class="b3-button b3-button--outline fn__size200" data-action="purgeData">
@@ -153,6 +158,11 @@ const renderProvider = (provider: number) => {
         <option ${window.siyuan.config.sync.webdav.skipTlsVerify ? "selected" : ""} value="true">Skip</option>
     </select>
 </div>
+<div class="b3-label b3-label--inner fn__flex">
+    <div class="fn__flex-center fn__size200">Concurrent Reqs</div>
+    <div class="fn__space"></div>
+    <input id="webdavConcurrentReqs" class="b3-text-field fn__block" type="number" min="1" max="16" value="${window.siyuan.config.sync.webdav.concurrentReqs}">
+</div>
 <div class="b3-label b3-label--inner fn__flex">
     <div class="fn__flex-1"></div>
     <button class="b3-button b3-button--outline fn__size200" data-action="purgeData">
@@ -267,6 +277,13 @@ const bindProviderEvent = () => {
                 if (300 < timeout) {
                     timeout = 300;
                 }
+                let concurrentReqs = parseInt((providerPanelElement.querySelector("#s3ConcurrentReqs") as HTMLInputElement).value, 10);
+                if (1 > concurrentReqs) {
+                    concurrentReqs = 1;
+                }
+                if (16 < concurrentReqs) {
+                    concurrentReqs = 16;
+                }
                 (providerPanelElement.querySelector("#timeout") as HTMLInputElement).value = timeout.toString();
                 const s3 = {
                     endpoint: (providerPanelElement.querySelector("#endpoint") as HTMLInputElement).value,
@@ -277,6 +294,7 @@ const bindProviderEvent = () => {
                     region: (providerPanelElement.querySelector("#region") as HTMLInputElement).value,
                     skipTlsVerify: (providerPanelElement.querySelector("#s3SkipTlsVerify") as HTMLInputElement).value === "true",
                     timeout: timeout,
+                    concurrentReqs: concurrentReqs,
                 };
                 fetchPost("/api/sync/setSyncProviderS3", {s3}, () => {
                     window.siyuan.config.sync.s3 = s3;
@@ -289,6 +307,13 @@ const bindProviderEvent = () => {
                 if (300 < timeout) {
                     timeout = 300;
                 }
+                let concurrentReqs = parseInt((providerPanelElement.querySelector("#webdavConcurrentReqs") as HTMLInputElement).value, 10);
+                if (1 > concurrentReqs) {
+                    concurrentReqs = 1;
+                }
+                if (16 < concurrentReqs) {
+                    concurrentReqs = 16;
+                }
                 (providerPanelElement.querySelector("#timeout") as HTMLInputElement).value = timeout.toString();
                 const webdav = {
                     endpoint: (providerPanelElement.querySelector("#endpoint") as HTMLInputElement).value,
@@ -296,6 +321,7 @@ const bindProviderEvent = () => {
                     password: (providerPanelElement.querySelector("#password") as HTMLInputElement).value,
                     skipTlsVerify: (providerPanelElement.querySelector("#webdavSkipTlsVerify") as HTMLInputElement).value === "true",
                     timeout: timeout,
+                    concurrentReqs: concurrentReqs,
                 };
                 fetchPost("/api/sync/setSyncProviderWebDAV", {webdav}, () => {
                     window.siyuan.config.sync.webdav = webdav;

app/src/types/config.d.ts

@@ -1381,6 +1381,10 @@ declare namespace Config {
          * Timeout (unit: seconds)
          */
         timeout: number;
+        /**
+         * Concurrent requests.
+         */
+        concurrentReqs: number;
     }
 
     /**
@@ -1403,6 +1407,10 @@ declare namespace Config {
          * Timeout (unit: seconds)
          */
         timeout: number;
+        /**
+         * Concurrent requests.
+         */
+        concurrentReqs: number;
         /**
          * Username
          */

kernel/conf/sync.go

@@ -41,22 +41,24 @@ func NewSync() *Sync {
 }
 
 type S3 struct {
-	Endpoint      string `json:"endpoint"`      // 服务端点
-	AccessKey     string `json:"accessKey"`     // Access Key
-	SecretKey     string `json:"secretKey"`     // Secret Key
-	Bucket        string `json:"bucket"`        // 存储空间
-	Region        string `json:"region"`        // 存储区域
-	PathStyle     bool   `json:"pathStyle"`     // 是否使用路径风格
-	SkipTlsVerify bool   `json:"skipTlsVerify"` // 是否跳过 TLS 验证
-	Timeout       int    `json:"timeout"`       // 超时时间，单位：秒
+	Endpoint       string `json:"endpoint"`       // 服务端点
+	AccessKey      string `json:"accessKey"`      // Access Key
+	SecretKey      string `json:"secretKey"`      // Secret Key
+	Bucket         string `json:"bucket"`         // 存储空间
+	Region         string `json:"region"`         // 存储区域
+	PathStyle      bool   `json:"pathStyle"`      // 是否使用路径风格
+	SkipTlsVerify  bool   `json:"skipTlsVerify"`  // 是否跳过 TLS 验证
+	Timeout        int    `json:"timeout"`        // 超时时间，单位：秒
+	ConcurrentReqs int    `json:"concurrentReqs"` // 并发请求数
 }
 
 type WebDAV struct {
-	Endpoint      string `json:"endpoint"`      // 服务端点
-	Username      string `json:"username"`      // 用户名
-	Password      string `json:"password"`      // 密码
-	SkipTlsVerify bool   `json:"skipTlsVerify"` // 是否跳过 TLS 验证
-	Timeout       int    `json:"timeout"`       // 超时时间，单位：秒
+	Endpoint       string `json:"endpoint"`       // 服务端点
+	Username       string `json:"username"`       // 用户名
+	Password       string `json:"password"`       // 密码
+	SkipTlsVerify  bool   `json:"skipTlsVerify"`  // 是否跳过 TLS 验证
+	Timeout        int    `json:"timeout"`        // 超时时间，单位：秒
+	ConcurrentReqs int    `json:"concurrentReqs"` // 并发请求数
 }
 
 const (

kernel/filesys/tree.go

@@ -43,19 +43,17 @@ func LoadTrees(ids []string) (ret map[string]*parse.Tree) {
 	luteEngine := util.NewLute()
 	var boxIDs []string
 	var paths []string
-	blockIDs := map[string]string{}
-	seen := map[string]bool{}
+	blockIDs := map[string][]string{}
 	for _, bt := range bts {
-		key := bt.BoxID + bt.Path
-		if !seen[key] {
-			seen[key] = true
-			boxIDs = append(boxIDs, bt.BoxID)
-			paths = append(paths, bt.Path)
-			blockIDs[bt.RootID] = bt.ID
+		boxIDs = append(boxIDs, bt.BoxID)
+		paths = append(paths, bt.Path)
+		if _, ok := blockIDs[bt.RootID]; !ok {
+			blockIDs[bt.RootID] = []string{}
 		}
+		blockIDs[bt.RootID] = append(blockIDs[bt.RootID], bt.ID)
 	}
 
-	trees, errs := batchLoadTrees(boxIDs, paths, luteEngine)
+	trees, errs := BatchLoadTrees(boxIDs, paths, luteEngine)
 	for i := range trees {
 		tree := trees[i]
 		err := errs[i]
@@ -64,8 +62,10 @@ func LoadTrees(ids []string) (ret map[string]*parse.Tree) {
 			continue
 		}
 
-		id := blockIDs[tree.Root.ID]
-		ret[id] = tree
+		bIDs := blockIDs[tree.Root.ID]
+		for _, bID := range bIDs {
+			ret[bID] = tree
+		}
 	}
 	return
 }
@@ -82,11 +82,18 @@ func LoadTree(boxID, p string, luteEngine *lute.Lute) (ret *parse.Tree, err erro
 	return
 }
 
-func batchLoadTrees(boxIDs, paths []string, luteEngine *lute.Lute) (ret []*parse.Tree, errs []error) {
+func BatchLoadTrees(boxIDs, paths []string, luteEngine *lute.Lute) (ret []*parse.Tree, errs []error) {
 	var wg sync.WaitGroup
 	lock := sync.Mutex{}
+	loaded := map[string]bool{}
 	for i := range paths {
+		if loaded[boxIDs[i]+paths[i]] {
+			continue
+		}
+
+		loaded[boxIDs[i]+paths[i]] = true
 		wg.Add(1)
+
 		go func(i int) {
 			defer wg.Done()
 

kernel/go.mod

@@ -54,7 +54,7 @@ require (
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
 	github.com/sashabaranov/go-openai v1.29.1
 	github.com/shirou/gopsutil/v3 v3.24.5
-	github.com/siyuan-note/dejavu v0.0.0-20241014034432-7d61fd4d1fe0
+	github.com/siyuan-note/dejavu v0.0.0-20241016112457-5fc03573be87
 	github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4
 	github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97
 	github.com/siyuan-note/filelock v0.0.0-20240724034355-d1ed7bf21d04

kernel/go.sum

@@ -332,8 +332,8 @@ github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+D
 github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d h1:lvCTyBbr36+tqMccdGMwuEU+hjux/zL6xSmf5S9ITaA=
 github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
 github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8=
-github.com/siyuan-note/dejavu v0.0.0-20241014034432-7d61fd4d1fe0 h1:r2XSfIkA6rF0Cj4RqFlJzw9Am7x5xOlA6WcClV7vgm8=
-github.com/siyuan-note/dejavu v0.0.0-20241014034432-7d61fd4d1fe0/go.mod h1:7E+jOWm75Me1ss7Sc5UM6O5xMiATAyCbN3g7MP6Cgxw=
+github.com/siyuan-note/dejavu v0.0.0-20241016112457-5fc03573be87 h1:EG8h35jYH8/iOukQlhB2OE7S35PwOBBn+XMOK1rtgtg=
+github.com/siyuan-note/dejavu v0.0.0-20241016112457-5fc03573be87/go.mod h1:7E+jOWm75Me1ss7Sc5UM6O5xMiATAyCbN3g7MP6Cgxw=
 github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4 h1:kJaw5L/evyW6LcB9IQT8PR4ppx8JVqOFP9Ix3rfwSrc=
 github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4/go.mod h1:UYcCCY+0wh+GmUoDOaO63j1sV5lgy7laLAk1XhEiUis=
 github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97 h1:lM5v8BfNtbOL5jYwhCdMYBcYtr06IYBKjjSLAPMKTM8=

kernel/model/conf.go

@@ -339,15 +339,17 @@ func InitConf() {
 		Conf.Sync.Mode = 1
 	}
 	if nil == Conf.Sync.S3 {
-		Conf.Sync.S3 = &conf.S3{}
+		Conf.Sync.S3 = &conf.S3{SkipTlsVerify: true}
 	}
 	Conf.Sync.S3.Endpoint = util.NormalizeEndpoint(Conf.Sync.S3.Endpoint)
 	Conf.Sync.S3.Timeout = util.NormalizeTimeout(Conf.Sync.S3.Timeout)
+	Conf.Sync.S3.ConcurrentReqs = util.NormalizeConcurrentReqs(Conf.Sync.S3.ConcurrentReqs, conf.ProviderS3)
 	if nil == Conf.Sync.WebDAV {
-		Conf.Sync.WebDAV = &conf.WebDAV{}
+		Conf.Sync.WebDAV = &conf.WebDAV{SkipTlsVerify: true}
 	}
 	Conf.Sync.WebDAV.Endpoint = util.NormalizeEndpoint(Conf.Sync.WebDAV.Endpoint)
 	Conf.Sync.WebDAV.Timeout = util.NormalizeTimeout(Conf.Sync.WebDAV.Timeout)
+	Conf.Sync.WebDAV.ConcurrentReqs = util.NormalizeConcurrentReqs(Conf.Sync.WebDAV.ConcurrentReqs, conf.ProviderWebDAV)
 	if util.ContainerDocker == util.Container {
 		Conf.Sync.Perception = false
 	}

kernel/model/repository.go

@@ -1959,6 +1959,7 @@ func buildCloudConf() (ret *cloud.Conf, err error) {
 			PathStyle:     Conf.Sync.S3.PathStyle,
 			SkipTlsVerify: Conf.Sync.S3.SkipTlsVerify,
 			Timeout:       Conf.Sync.S3.Timeout,
+			ConcurrentReqs: Conf.Sync.S3.ConcurrentReqs,
 		}
 	case conf.ProviderWebDAV:
 		ret.WebDAV = &cloud.ConfWebDAV{
@@ -1967,6 +1968,7 @@ func buildCloudConf() (ret *cloud.Conf, err error) {
 			Password:      Conf.Sync.WebDAV.Password,
 			SkipTlsVerify: Conf.Sync.WebDAV.SkipTlsVerify,
 			Timeout:       Conf.Sync.WebDAV.Timeout,
+			ConcurrentReqs: Conf.Sync.WebDAV.ConcurrentReqs,
 		}
 	default:
 		err = fmt.Errorf("invalid provider [%d]", Conf.Sync.Provider)

kernel/model/sync.go

@@ -426,6 +426,7 @@ func SetSyncProviderS3(s3 *conf.S3) (err error) {
 	s3.Bucket = strings.TrimSpace(s3.Bucket)
 	s3.Region = strings.TrimSpace(s3.Region)
 	s3.Timeout = util.NormalizeTimeout(s3.Timeout)
+	s3.ConcurrentReqs = util.NormalizeConcurrentReqs(s3.ConcurrentReqs, conf.ProviderS3)
 
 	if !cloud.IsValidCloudDirName(s3.Bucket) {
 		util.PushErrMsg(Conf.Language(37), 5000)
@@ -450,6 +451,7 @@ func SetSyncProviderWebDAV(webdav *conf.WebDAV) (err error) {
 	webdav.Username = strings.TrimSpace(webdav.Username)
 	webdav.Password = strings.TrimSpace(webdav.Password)
 	webdav.Timeout = util.NormalizeTimeout(webdav.Timeout)
+	webdav.ConcurrentReqs = util.NormalizeConcurrentReqs(webdav.ConcurrentReqs, conf.ProviderWebDAV)
 
 	Conf.Sync.WebDAV = webdav
 	Conf.Save()

kernel/util/path.go

@@ -179,6 +179,21 @@ func GetChildDocDepth(treeAbsPath string) (ret int) {
 	return
 }
 
+func NormalizeConcurrentReqs(concurrentReqs int, provider int) int {
+	if 1 > concurrentReqs {
+		if 2 == provider { // S3
+			return 8
+		} else if 3 == provider { // WebDAV
+			return 1
+		}
+		return 8
+	}
+	if 16 < concurrentReqs {
+		return 16
+	}
+	return concurrentReqs
+}
+
 func NormalizeTimeout(timeout int) int {
 	if 7 > timeout {
 		if 1 > timeout {