0ct0pu5/siyuan
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/dev' into dev

Browse source
This commit is contained in:
Vanessa 2024-10-16 21:43:25 +08:00
commit 3169afbee8
14 changed files with 405 additions and 271 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/CONTRIBUTING.md vendored
View file

@ -7,7 +7,7 @@
## User Interface
Install pnpm: `npm install -g pnpm@9.1.1`
Install pnpm: `npm install -g pnpm@9.12.1`
<details>
<summary>For China mainland</summary>

2
.github/CONTRIBUTING_zh_CN.md vendored
View file

@ -7,7 +7,7 @@
## NPM 依赖
安装 pnpm：`npm install -g pnpm@9.1.1`
安装 pnpm：`npm install -g pnpm@9.12.1`
<details>
<summary>适用于中国大陆</summary>

4
app/package.json
View file

@ -4,7 +4,7 @@
"description": "Refactor your thinking",
"homepage": "https://b3log.org/siyuan",
"main": "./electron/main.js",
"packageManager": "pnpm@9.1.1",
"packageManager": "pnpm@9.12.1",
"scripts": {
"lint": "eslint . --fix --cache",
"dev": "webpack --mode development",
@ -71,7 +71,7 @@
"sass": "^1.53.0",
"sass-loader": "^12.6.0",
"typescript": "^4.7.4",
"webpack": "^5.76.0",
"webpack": "^5.94.0",
"webpack-bundle-analyzer": "^4.5.0",
"webpack-cli": "^4.10.0"
},

544
app/pnpm-lock.yaml generated
View file

File diff suppressed because it is too large Load diff

26
app/src/config/repos.ts
View file

@ -99,6 +99,11 @@ const renderProvider = (provider: number) => {
<option ${window.siyuan.config.sync.s3.skipTlsVerify ? "selected" : ""} value="true">Skip</option>
</select>
</div>
<div class="b3-label b3-label--inner fn__flex">
<div class="fn__flex-center fn__size200">Concurrent Reqs</div>
<div class="fn__space"></div>
<input id="s3ConcurrentReqs" class="b3-text-field fn__block" type="number" min="1" max="16" value="${window.siyuan.config.sync.s3.concurrentReqs}">
</div>
<div class="b3-label b3-label--inner fn__flex">
<div class="fn__flex-1"></div>
<button class="b3-button b3-button--outline fn__size200" data-action="purgeData">
@ -153,6 +158,11 @@ const renderProvider = (provider: number) => {
<option ${window.siyuan.config.sync.webdav.skipTlsVerify ? "selected" : ""} value="true">Skip</option>
</select>
</div>
<div class="b3-label b3-label--inner fn__flex">
<div class="fn__flex-center fn__size200">Concurrent Reqs</div>
<div class="fn__space"></div>
<input id="webdavConcurrentReqs" class="b3-text-field fn__block" type="number" min="1" max="16" value="${window.siyuan.config.sync.webdav.concurrentReqs}">
</div>
<div class="b3-label b3-label--inner fn__flex">
<div class="fn__flex-1"></div>
<button class="b3-button b3-button--outline fn__size200" data-action="purgeData">
@ -267,6 +277,13 @@ const bindProviderEvent = () => {
if (300 < timeout) {
timeout = 300;
}
let concurrentReqs = parseInt((providerPanelElement.querySelector("#s3ConcurrentReqs") as HTMLInputElement).value, 10);
if (1 > concurrentReqs) {
concurrentReqs = 1;
}
if (16 < concurrentReqs) {
concurrentReqs = 16;
}
(providerPanelElement.querySelector("#timeout") as HTMLInputElement).value = timeout.toString();
const s3 = {
endpoint: (providerPanelElement.querySelector("#endpoint") as HTMLInputElement).value,
@ -277,6 +294,7 @@ const bindProviderEvent = () => {
region: (providerPanelElement.querySelector("#region") as HTMLInputElement).value,
skipTlsVerify: (providerPanelElement.querySelector("#s3SkipTlsVerify") as HTMLInputElement).value === "true",
timeout: timeout,
concurrentReqs: concurrentReqs,
};
fetchPost("/api/sync/setSyncProviderS3", {s3}, () => {
window.siyuan.config.sync.s3 = s3;
@ -289,6 +307,13 @@ const bindProviderEvent = () => {
if (300 < timeout) {
timeout = 300;
}
let concurrentReqs = parseInt((providerPanelElement.querySelector("#webdavConcurrentReqs") as HTMLInputElement).value, 10);
if (1 > concurrentReqs) {
concurrentReqs = 1;
}
if (16 < concurrentReqs) {
concurrentReqs = 16;
}
(providerPanelElement.querySelector("#timeout") as HTMLInputElement).value = timeout.toString();
const webdav = {
endpoint: (providerPanelElement.querySelector("#endpoint") as HTMLInputElement).value,
@ -296,6 +321,7 @@ const bindProviderEvent = () => {
password: (providerPanelElement.querySelector("#password") as HTMLInputElement).value,
skipTlsVerify: (providerPanelElement.querySelector("#webdavSkipTlsVerify") as HTMLInputElement).value === "true",
timeout: timeout,
concurrentReqs: concurrentReqs,
};
fetchPost("/api/sync/setSyncProviderWebDAV", {webdav}, () => {
window.siyuan.config.sync.webdav = webdav;

8
app/src/types/config.d.ts vendored
View file

@ -1381,6 +1381,10 @@ declare namespace Config {
* Timeout (unit: seconds)
*/
timeout: number;
/**
* Concurrent requests.
*/
concurrentReqs: number;
}
/**
@ -1403,6 +1407,10 @@ declare namespace Config {
* Timeout (unit: seconds)
*/
timeout: number;
/**
* Concurrent requests.
*/
concurrentReqs: number;
/**
* Username
*/

28
kernel/conf/sync.go
View file

@ -41,22 +41,24 @@ func NewSync() *Sync {
}
type S3 struct {
Endpoint string `json:"endpoint"` // 服务端点
AccessKey string `json:"accessKey"` // Access Key
SecretKey string `json:"secretKey"` // Secret Key
Bucket string `json:"bucket"` // 存储空间
Region string `json:"region"` // 存储区域
PathStyle bool `json:"pathStyle"` // 是否使用路径风格
SkipTlsVerify bool `json:"skipTlsVerify"` // 是否跳过 TLS 验证
Timeout int `json:"timeout"` // 超时时间，单位：秒
Endpoint string `json:"endpoint"` // 服务端点
AccessKey string `json:"accessKey"` // Access Key
SecretKey string `json:"secretKey"` // Secret Key
Bucket string `json:"bucket"` // 存储空间
Region string `json:"region"` // 存储区域
PathStyle bool `json:"pathStyle"` // 是否使用路径风格
SkipTlsVerify bool `json:"skipTlsVerify"` // 是否跳过 TLS 验证
Timeout int `json:"timeout"` // 超时时间，单位：秒
ConcurrentReqs int `json:"concurrentReqs"` // 并发请求数
}
type WebDAV struct {
Endpoint string `json:"endpoint"` // 服务端点
Username string `json:"username"` // 用户名
Password string `json:"password"` // 密码
SkipTlsVerify bool `json:"skipTlsVerify"` // 是否跳过 TLS 验证
Timeout int `json:"timeout"` // 超时时间，单位：秒
Endpoint string `json:"endpoint"` // 服务端点
Username string `json:"username"` // 用户名
Password string `json:"password"` // 密码
SkipTlsVerify bool `json:"skipTlsVerify"` // 是否跳过 TLS 验证
Timeout int `json:"timeout"` // 超时时间，单位：秒
ConcurrentReqs int `json:"concurrentReqs"` // 并发请求数
}
const (

31
kernel/filesys/tree.go
View file

@ -43,19 +43,17 @@ func LoadTrees(ids []string) (ret map[string]*parse.Tree) {
luteEngine := util.NewLute()
var boxIDs []string
var paths []string
blockIDs := map[string]string{}
seen := map[string]bool{}
blockIDs := map[string][]string{}
for _, bt := range bts {
key := bt.BoxID + bt.Path
if !seen[key] {
seen[key] = true
boxIDs = append(boxIDs, bt.BoxID)
paths = append(paths, bt.Path)
blockIDs[bt.RootID] = bt.ID
boxIDs = append(boxIDs, bt.BoxID)
paths = append(paths, bt.Path)
if _, ok := blockIDs[bt.RootID]; !ok {
blockIDs[bt.RootID] = []string{}
}
blockIDs[bt.RootID] = append(blockIDs[bt.RootID], bt.ID)
}
trees, errs := batchLoadTrees(boxIDs, paths, luteEngine)
trees, errs := BatchLoadTrees(boxIDs, paths, luteEngine)
for i := range trees {
tree := trees[i]
err := errs[i]
@ -64,8 +62,10 @@ func LoadTrees(ids []string) (ret map[string]*parse.Tree) {
continue
}
id := blockIDs[tree.Root.ID]
ret[id] = tree
bIDs := blockIDs[tree.Root.ID]
for _, bID := range bIDs {
ret[bID] = tree
}
}
return
}
@ -82,11 +82,18 @@ func LoadTree(boxID, p string, luteEngine *lute.Lute) (ret *parse.Tree, err erro
return
}
func batchLoadTrees(boxIDs, paths []string, luteEngine *lute.Lute) (ret []*parse.Tree, errs []error) {
func BatchLoadTrees(boxIDs, paths []string, luteEngine *lute.Lute) (ret []*parse.Tree, errs []error) {
var wg sync.WaitGroup
lock := sync.Mutex{}
loaded := map[string]bool{}
for i := range paths {
if loaded[boxIDs[i]+paths[i]] {
continue
}
loaded[boxIDs[i]+paths[i]] = true
wg.Add(1)
go func(i int) {
defer wg.Done()

2
kernel/go.mod
View file

@ -54,7 +54,7 @@ require (
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
github.com/sashabaranov/go-openai v1.29.1
github.com/shirou/gopsutil/v3 v3.24.5
github.com/siyuan-note/dejavu v0.0.0-20241014034432-7d61fd4d1fe0
github.com/siyuan-note/dejavu v0.0.0-20241016112457-5fc03573be87
github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4
github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97
github.com/siyuan-note/filelock v0.0.0-20240724034355-d1ed7bf21d04

4
kernel/go.sum
View file

@ -332,8 +332,8 @@ github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+D
github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d h1:lvCTyBbr36+tqMccdGMwuEU+hjux/zL6xSmf5S9ITaA=
github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8=
github.com/siyuan-note/dejavu v0.0.0-20241014034432-7d61fd4d1fe0 h1:r2XSfIkA6rF0Cj4RqFlJzw9Am7x5xOlA6WcClV7vgm8=
github.com/siyuan-note/dejavu v0.0.0-20241014034432-7d61fd4d1fe0/go.mod h1:7E+jOWm75Me1ss7Sc5UM6O5xMiATAyCbN3g7MP6Cgxw=
github.com/siyuan-note/dejavu v0.0.0-20241016112457-5fc03573be87 h1:EG8h35jYH8/iOukQlhB2OE7S35PwOBBn+XMOK1rtgtg=
github.com/siyuan-note/dejavu v0.0.0-20241016112457-5fc03573be87/go.mod h1:7E+jOWm75Me1ss7Sc5UM6O5xMiATAyCbN3g7MP6Cgxw=
github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4 h1:kJaw5L/evyW6LcB9IQT8PR4ppx8JVqOFP9Ix3rfwSrc=
github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4/go.mod h1:UYcCCY+0wh+GmUoDOaO63j1sV5lgy7laLAk1XhEiUis=
github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97 h1:lM5v8BfNtbOL5jYwhCdMYBcYtr06IYBKjjSLAPMKTM8=

6
kernel/model/conf.go
View file

@ -339,15 +339,17 @@ func InitConf() {
Conf.Sync.Mode = 1
}
if nil == Conf.Sync.S3 {
Conf.Sync.S3 = &conf.S3{}
Conf.Sync.S3 = &conf.S3{SkipTlsVerify: true}
}
Conf.Sync.S3.Endpoint = util.NormalizeEndpoint(Conf.Sync.S3.Endpoint)
Conf.Sync.S3.Timeout = util.NormalizeTimeout(Conf.Sync.S3.Timeout)
Conf.Sync.S3.ConcurrentReqs = util.NormalizeConcurrentReqs(Conf.Sync.S3.ConcurrentReqs, conf.ProviderS3)
if nil == Conf.Sync.WebDAV {
Conf.Sync.WebDAV = &conf.WebDAV{}
Conf.Sync.WebDAV = &conf.WebDAV{SkipTlsVerify: true}
}
Conf.Sync.WebDAV.Endpoint = util.NormalizeEndpoint(Conf.Sync.WebDAV.Endpoint)
Conf.Sync.WebDAV.Timeout = util.NormalizeTimeout(Conf.Sync.WebDAV.Timeout)
Conf.Sync.WebDAV.ConcurrentReqs = util.NormalizeConcurrentReqs(Conf.Sync.WebDAV.ConcurrentReqs, conf.ProviderWebDAV)
if util.ContainerDocker == util.Container {
Conf.Sync.Perception = false
}

2
kernel/model/repository.go
View file

@ -1959,6 +1959,7 @@ func buildCloudConf() (ret *cloud.Conf, err error) {
PathStyle: Conf.Sync.S3.PathStyle,
SkipTlsVerify: Conf.Sync.S3.SkipTlsVerify,
Timeout: Conf.Sync.S3.Timeout,
ConcurrentReqs: Conf.Sync.S3.ConcurrentReqs,
}
case conf.ProviderWebDAV:
ret.WebDAV = &cloud.ConfWebDAV{
@ -1967,6 +1968,7 @@ func buildCloudConf() (ret *cloud.Conf, err error) {
Password: Conf.Sync.WebDAV.Password,
SkipTlsVerify: Conf.Sync.WebDAV.SkipTlsVerify,
Timeout: Conf.Sync.WebDAV.Timeout,
ConcurrentReqs: Conf.Sync.WebDAV.ConcurrentReqs,
}
default:
err = fmt.Errorf("invalid provider [%d]", Conf.Sync.Provider)

2
kernel/model/sync.go
View file

@ -426,6 +426,7 @@ func SetSyncProviderS3(s3 *conf.S3) (err error) {
s3.Bucket = strings.TrimSpace(s3.Bucket)
s3.Region = strings.TrimSpace(s3.Region)
s3.Timeout = util.NormalizeTimeout(s3.Timeout)
s3.ConcurrentReqs = util.NormalizeConcurrentReqs(s3.ConcurrentReqs, conf.ProviderS3)
if !cloud.IsValidCloudDirName(s3.Bucket) {
util.PushErrMsg(Conf.Language(37), 5000)
@ -450,6 +451,7 @@ func SetSyncProviderWebDAV(webdav *conf.WebDAV) (err error) {
webdav.Username = strings.TrimSpace(webdav.Username)
webdav.Password = strings.TrimSpace(webdav.Password)
webdav.Timeout = util.NormalizeTimeout(webdav.Timeout)
webdav.ConcurrentReqs = util.NormalizeConcurrentReqs(webdav.ConcurrentReqs, conf.ProviderWebDAV)
Conf.Sync.WebDAV = webdav
Conf.Save()

15
kernel/util/path.go
View file

@ -179,6 +179,21 @@ func GetChildDocDepth(treeAbsPath string) (ret int) {
return
}
func NormalizeConcurrentReqs(concurrentReqs int, provider int) int {
if 1 > concurrentReqs {
if 2 == provider { // S3
return 8
} else if 3 == provider { // WebDAV
return 1
}
return 8
}
if 16 < concurrentReqs {
return 16
}
return concurrentReqs
}
func NormalizeTimeout(timeout int) int {
if 7 > timeout {
if 1 > timeout {