Prechádzať zdrojové kódy

Merge remote-tracking branch 'origin/dev' into dev

Vanessa 2 rokov pred
rodič
commit
beb5005125
5 zmenil súbory, kde vykonal 59 pridanie a 68 odobranie
  1. 6 6
      kernel/go.mod
  2. 12 0
      kernel/go.sum
  3. 9 6
      kernel/model/assets.go
  4. 26 47
      kernel/model/repository.go
  5. 6 9
      kernel/model/sync.go

+ 6 - 6
kernel/go.mod

@@ -33,16 +33,16 @@ require (
 	github.com/mitchellh/go-ps v1.0.0
 	github.com/mssola/user_agent v0.5.3
 	github.com/olahol/melody v1.1.1
-	github.com/panjf2000/ants/v2 v2.5.0
+	github.com/panjf2000/ants/v2 v2.6.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/qiniu/go-sdk/v7 v7.13.0
 	github.com/radovskyb/watcher v1.0.7
-	github.com/siyuan-note/dejavu v0.0.0-20221023024107-17bf52da95a1
+	github.com/siyuan-note/dejavu v0.0.0-20221031125623-fe9358b6cc16
 	github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75
 	github.com/siyuan-note/eventbus v0.0.0-20220916025349-3ac6e75522da
 	github.com/siyuan-note/filelock v0.0.0-20221007163134-7e64809023ef
 	github.com/siyuan-note/httpclient v0.0.0-20221019094331-c904ac9be571
-	github.com/siyuan-note/logging v0.0.0-20220717040626-f796b05ee520
+	github.com/siyuan-note/logging v0.0.0-20221031125421-9b7234d79d8a
 	github.com/steambap/captcha v1.4.1
 	github.com/vmihailenco/msgpack/v5 v5.3.5
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673
@@ -86,7 +86,7 @@ require (
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/juju/errors v1.0.0 // indirect
-	github.com/klauspost/compress v1.15.11 // indirect
+	github.com/klauspost/compress v1.15.12 // indirect
 	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/lucas-clemente/quic-go v0.30.0 // indirect
 	github.com/marten-seemann/qpack v0.3.0 // indirect
@@ -99,7 +99,7 @@ require (
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/onsi/ginkgo/v2 v2.3.1 // indirect
+	github.com/onsi/ginkgo/v2 v2.4.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/restic/chunker v0.4.0 // indirect
@@ -111,7 +111,7 @@ require (
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
 	golang.org/x/crypto v0.1.0 // indirect
-	golang.org/x/exp v0.0.0-20221019170559-20944726eadf // indirect
+	golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f // indirect
 	golang.org/x/mod v0.6.0 // indirect
 	golang.org/x/net v0.1.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect

+ 12 - 0
kernel/go.sum

@@ -216,6 +216,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c=
 github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
+github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM=
+github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -285,6 +287,8 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/ginkgo/v2 v2.3.1 h1:8SbseP7qM32WcvE6VaN6vfXxv698izmsJ1UQX9ve7T8=
 github.com/onsi/ginkgo/v2 v2.3.1/go.mod h1:Sv4yQXwG5VmF7tm3Q5Z+RWUpPo24LF1mpnz2crUb8Ys=
+github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs=
+github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY=
@@ -292,6 +296,8 @@ github.com/onsi/gomega v1.22.0 h1:AIg2/OntwkBiCg5Tt1ayyiF1ArFrWFoCSMtMi/wdApk=
 github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
 github.com/panjf2000/ants/v2 v2.5.0 h1:1rWGWSnxCsQBga+nQbA4/iY6VMeNoOIAM0ZWh9u3q2Q=
 github.com/panjf2000/ants/v2 v2.5.0/go.mod h1:cU93usDlihJZ5CfRGNDYsiBYvoilLvBF5Qp/BT2GNRE=
+github.com/panjf2000/ants/v2 v2.6.0 h1:xOSpw42m+BMiJ2I33we7h6fYzG4DAlpE1xyI7VS2gxU=
+github.com/panjf2000/ants/v2 v2.6.0/go.mod h1:cU93usDlihJZ5CfRGNDYsiBYvoilLvBF5Qp/BT2GNRE=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
@@ -354,6 +360,8 @@ github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYED
 github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
 github.com/siyuan-note/dejavu v0.0.0-20221023024107-17bf52da95a1 h1:sk6pyVp22MfFXK8C5YOKD5eqWtvcMnkiEh1nyZlw4aU=
 github.com/siyuan-note/dejavu v0.0.0-20221023024107-17bf52da95a1/go.mod h1:f+6y7KcTLlk2N2dLl5HeV/YzkyCTEqWNoMiWT0gkMi8=
+github.com/siyuan-note/dejavu v0.0.0-20221031125623-fe9358b6cc16 h1:05p9tr+rJrKurRQG3NF9i/IPgGdBMPu5tKi1Ux427lY=
+github.com/siyuan-note/dejavu v0.0.0-20221031125623-fe9358b6cc16/go.mod h1:+U86jfsvpacZBThE3Ouf/ZQ4EsB4jGPJsMO2iuRv0LQ=
 github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 h1:Bi7/7f29LW+Fm0cHc0J1NO1cZqyJwljSWVmfOqVZgaE=
 github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw=
 github.com/siyuan-note/eventbus v0.0.0-20220916025349-3ac6e75522da h1:/jNhl7LC+9BhkWvNxuJDdsNfA/2wvfuj9mqWx4CbV90=
@@ -364,6 +372,8 @@ github.com/siyuan-note/httpclient v0.0.0-20221019094331-c904ac9be571 h1:AclwUOWg
 github.com/siyuan-note/httpclient v0.0.0-20221019094331-c904ac9be571/go.mod h1:FTPEFG3q0LAn33zON47HeRbLOWNk7HM+OPafmQgq4hU=
 github.com/siyuan-note/logging v0.0.0-20220717040626-f796b05ee520 h1:kscYjMt7jXYdd7Qj2OSUoBnoHc5B0U/E6OSx86VRLr4=
 github.com/siyuan-note/logging v0.0.0-20220717040626-f796b05ee520/go.mod h1:t1zRGxK13L/9ZFoGyTD39IbFCbee3CsypDj4b5dt4qM=
+github.com/siyuan-note/logging v0.0.0-20221031125421-9b7234d79d8a h1:b9VJCE8IccYjsadwNBI11he+Wn25hI9lCma4uYoIYEM=
+github.com/siyuan-note/logging v0.0.0-20221031125421-9b7234d79d8a/go.mod h1:t1zRGxK13L/9ZFoGyTD39IbFCbee3CsypDj4b5dt4qM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.7 h1:I6tZjLXD2Q1kjvNbIzB1wvQBsXmKXiVrhpRE8ZjP5jY=
@@ -430,6 +440,8 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
 golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4=
 golang.org/x/exp v0.0.0-20221019170559-20944726eadf h1:nFVjjKDgNY37+ZSYCJmtYf7tOlfQswHqplG2eosjOMg=
 golang.org/x/exp v0.0.0-20221019170559-20944726eadf/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
+golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f h1:Al51T6tzvuh3oiwX11vex3QgJ2XTedFPGmbEVh8cdoc=
+golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/image v0.0.0-20190823064033-3a9bac650e44/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=

+ 9 - 6
kernel/model/assets.go

@@ -95,15 +95,18 @@ func NetImg2LocalAssets(rootID string) (err error) {
 			if !sql.IsAssetLinkDest(dest) && (bytes.HasPrefix(bytes.ToLower(dest), []byte("https://")) || bytes.HasPrefix(bytes.ToLower(dest), []byte("http://"))) {
 				u := string(dest)
 				if strings.Contains(u, "qpic.cn") {
-					// 微信图片拉取改进 https://github.com/siyuan-note/siyuan/issues/5052
+					// 改进 `网络图片转换为本地图片` 微信图片拉取 https://github.com/siyuan-note/siyuan/issues/5052
 					if strings.Contains(u, "http://") {
 						u = strings.Replace(u, "http://", "https://", 1)
 					}
-					if strings.HasSuffix(u, "/0") {
-						u = strings.Replace(u, "/0", "/640", 1)
-					} else if strings.Contains(u, "/0?") {
-						u = strings.Replace(u, "/0?", "/640?", 1)
-					}
+
+					// 改进 `网络图片转换为本地图片` 微信图片拉取 https://github.com/siyuan-note/siyuan/issues/6431
+					// 下面这部分需要注释掉,否则会导致响应 400
+					//if strings.HasSuffix(u, "/0") {
+					//	u = strings.Replace(u, "/0", "/640", 1)
+					//} else if strings.Contains(u, "/0?") {
+					//	u = strings.Replace(u, "/0?", "/640?", 1)
+					//}
 				}
 				util.PushUpdateMsg(msgId, fmt.Sprintf(Conf.Language(119), u), 15000)
 				request := httpclient.NewBrowserRequest()

+ 26 - 47
kernel/model/repository.go

@@ -33,9 +33,9 @@ import (
 
 	"github.com/88250/gulu"
 	"github.com/dustin/go-humanize"
-	"github.com/qiniu/go-sdk/v7/storage"
 	"github.com/siyuan-note/dejavu"
 	"github.com/siyuan-note/dejavu/entity"
+	"github.com/siyuan-note/dejavu/transport"
 	"github.com/siyuan-note/encryption"
 	"github.com/siyuan-note/eventbus"
 	"github.com/siyuan-note/httpclient"
@@ -259,13 +259,8 @@ func DownloadCloudSnapshot(tag, id string) (err error) {
 		return
 	}
 
-	cloudInfo, err := buildCloudInfo()
-	if nil != err {
-		return
-	}
-
 	defer util.PushClearProgress()
-	downloadFileCount, downloadChunkCount, downloadBytes, err := repo.DownloadTagIndex(tag, id, cloudInfo, map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress})
+	downloadFileCount, downloadChunkCount, downloadBytes, err := repo.DownloadTagIndex(tag, id, map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress})
 	if nil != err {
 		return
 	}
@@ -286,14 +281,9 @@ func UploadCloudSnapshot(tag, id string) (err error) {
 		return
 	}
 
-	cloudInfo, err := buildCloudInfo()
-	if nil != err {
-		return
-	}
-
 	util.PushEndlessProgress(Conf.Language(116))
 	defer util.PushClearProgress()
-	uploadFileCount, uploadChunkCount, uploadBytes, err := repo.UploadTagIndex(tag, id, cloudInfo, map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress})
+	uploadFileCount, uploadChunkCount, uploadBytes, err := repo.UploadTagIndex(tag, id, map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress})
 	if nil != err {
 		if errors.Is(err, dejavu.ErrCloudBackupCountExceeded) {
 			err = fmt.Errorf(Conf.Language(84), Conf.Language(154))
@@ -324,12 +314,7 @@ func RemoveCloudRepoTag(tag string) (err error) {
 		return
 	}
 
-	cloudInfo, err := buildCloudInfo()
-	if nil != err {
-		return
-	}
-
-	err = repo.RemoveCloudRepoTag(tag, cloudInfo, map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar})
+	err = repo.RemoveCloudRepoTag(tag, map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar})
 	if nil != err {
 		return
 	}
@@ -347,11 +332,7 @@ func GetCloudRepoTagSnapshots() (ret []*dejavu.Log, err error) {
 		return
 	}
 
-	cloudInfo, err := buildCloudInfo()
-	if nil != err {
-		return
-	}
-	ret, err = repo.GetCloudRepoTagLogs(cloudInfo, map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar})
+	ret, err = repo.GetCloudRepoTagLogs(map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar})
 	if 1 > len(ret) {
 		ret = []*dejavu.Log{}
 	}
@@ -509,17 +490,12 @@ func bootSyncRepo() (err error) {
 		return
 	}
 
-	cloudInfo, err := buildCloudInfo()
-	if nil != err {
-		return
-	}
-
 	syncContext := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar}
-	fetchedFiles, err := repo.GetSyncCloudFiles(cloudInfo, syncContext)
+	fetchedFiles, err := repo.GetSyncCloudFiles(syncContext)
 	if errors.Is(err, dejavu.ErrRepoFatalErr) {
 		// 重置仓库并再次尝试同步
 		if _, resetErr := resetRepository(repo); nil == resetErr {
-			fetchedFiles, err = repo.GetSyncCloudFiles(cloudInfo, syncContext)
+			fetchedFiles, err = repo.GetSyncCloudFiles(syncContext)
 		}
 	}
 
@@ -601,17 +577,12 @@ func syncRepo(exit, byHand bool) (err error) {
 		return
 	}
 
-	cloudInfo, err := buildCloudInfo()
-	if nil != err {
-		return
-	}
-
 	syncContext := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar}
-	mergeResult, trafficStat, err := repo.Sync(cloudInfo, syncContext)
+	mergeResult, trafficStat, err := repo.Sync(syncContext)
 	if errors.Is(err, dejavu.ErrRepoFatalErr) {
 		// 重置仓库并再次尝试同步
 		if _, resetErr := resetRepository(repo); nil == resetErr {
-			mergeResult, trafficStat, err = repo.Sync(cloudInfo, syncContext)
+			mergeResult, trafficStat, err = repo.Sync(syncContext)
 		}
 	}
 	elapsed := time.Since(start)
@@ -838,9 +809,17 @@ func resetRepository(repo *dejavu.Repo) (index *entity.Index, err error) {
 }
 
 func newRepository() (ret *dejavu.Repo, err error) {
+	transportConf, err := buildRepoTransportConf()
+	if nil != err {
+		return
+	}
+
+	// TODO: 数据同步支持接入第三方对象存储服务 https://github.com/siyuan-note/siyuan/issues/6426
+	siyuanTransport := &transport.SiYuan{Conf: transportConf}
+
 	ignoreLines := getIgnoreLines()
 	ignoreLines = append(ignoreLines, "/.siyuan/conf.json") // 忽略旧版同步配置
-	ret, err = dejavu.NewRepo(util.DataDir, util.RepoDir, util.HistoryDir, util.TempDir, Conf.Repo.Key, ignoreLines)
+	ret, err = dejavu.NewRepo(util.DataDir, util.RepoDir, util.HistoryDir, util.TempDir, Conf.Repo.Key, ignoreLines, siyuanTransport)
 	if nil != err {
 		logging.LogErrorf("init data repo failed: %s", err)
 	}
@@ -1019,25 +998,25 @@ func subscribeEvents() {
 	})
 }
 
-func buildCloudInfo() (ret *dejavu.CloudInfo, err error) {
+func buildRepoTransportConf() (ret *transport.Conf, err error) {
 	if !IsValidCloudDirName(Conf.Sync.CloudName) {
 		logging.LogWarnf("invalid cloud repo name, rename it to [main]")
 		Conf.Sync.CloudName = "main"
 		Conf.Save()
 	}
 
-	if nil == Conf.User {
-		err = errors.New("user auth failed")
-		return
+	userId, token := "0", ""
+	if nil != Conf.User {
+		userId = Conf.User.UserId
+		token = Conf.User.UserToken
 	}
 
-	ret = &dejavu.CloudInfo{
+	ret = &transport.Conf{
 		Dir:       Conf.Sync.CloudName,
-		UserID:    Conf.User.UserId,
-		Token:     Conf.User.UserToken,
+		UserID:    userId,
+		Token:     token,
 		LimitSize: int64(Conf.User.UserSiYuanRepoSize - Conf.User.UserSiYuanAssetSize),
 		Server:    util.AliyunServer,
-		Zone:      &storage.ZoneHuadong, // TODO: 海外版需要条件编译
 	}
 	return
 }

+ 6 - 9
kernel/model/sync.go

@@ -301,13 +301,12 @@ func CreateCloudSyncDir(name string) (err error) {
 		return errors.New(Conf.Language(37))
 	}
 
-	var cloudInfo *dejavu.CloudInfo
-	cloudInfo, err = buildCloudInfo()
+	repo, err := newRepository()
 	if nil != err {
 		return
 	}
 
-	err = dejavu.CreateCloudRepo(name, cloudInfo)
+	err = repo.CreateCloudRepo(name)
 	return
 }
 
@@ -321,13 +320,12 @@ func RemoveCloudSyncDir(name string) (err error) {
 		return
 	}
 
-	var cloudInfo *dejavu.CloudInfo
-	cloudInfo, err = buildCloudInfo()
+	repo, err := newRepository()
 	if nil != err {
 		return
 	}
 
-	err = dejavu.RemoveCloudRepo(name, cloudInfo)
+	err = repo.RemoveCloudRepo(name)
 	if nil != err {
 		err = errors.New(formatErrorMsg(err))
 		return
@@ -348,13 +346,12 @@ func ListCloudSyncDir() (syncDirs []*Sync, hSize string, err error) {
 	var dirs []map[string]interface{}
 	var size int64
 
-	var cloudInfo *dejavu.CloudInfo
-	cloudInfo, err = buildCloudInfo()
+	repo, err := newRepository()
 	if nil != err {
 		return
 	}
 
-	dirs, size, err = dejavu.GetCloudRepos(cloudInfo)
+	dirs, size, err = repo.GetCloudRepos()
 	if nil != err {
 		err = errors.New(formatErrorMsg(err))
 		return