Merge remote-tracking branch 'origin/dev' into dev

This commit is contained in:
Vanessa 2023-11-12 12:11:14 +08:00
commit 2c71478d83
5 changed files with 102 additions and 67 deletions

View file

@ -49,7 +49,7 @@ require (
github.com/radovskyb/watcher v1.0.7
github.com/sashabaranov/go-openai v1.17.5
github.com/shirou/gopsutil/v3 v3.23.10
github.com/siyuan-note/dejavu v0.0.0-20231110145452-59dbd8bb6c2f
github.com/siyuan-note/dejavu v0.0.0-20231111144625-3b36f1124c31
github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75
github.com/siyuan-note/eventbus v0.0.0-20230804030110-cf250f838c80
github.com/siyuan-note/filelock v0.0.0-20231107122348-6ed75b0b525a
@ -77,7 +77,7 @@ require (
github.com/andybalholm/brotli v1.0.6 // indirect
github.com/andybalholm/cascadia v1.3.2 // indirect
github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect
github.com/aws/aws-sdk-go v1.47.8 // indirect
github.com/aws/aws-sdk-go v1.47.9 // indirect
github.com/bytedance/sonic v1.10.2 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
@ -125,7 +125,7 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/onsi/ginkgo/v2 v2.13.0 // indirect
github.com/onsi/ginkgo/v2 v2.13.1 // indirect
github.com/otiai10/gosseract/v2 v2.4.1 // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
@ -157,7 +157,7 @@ require (
go.uber.org/mock v0.3.0 // indirect
golang.org/x/arch v0.6.0 // indirect
golang.org/x/crypto v0.15.0 // indirect
golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 // indirect
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
golang.org/x/net v0.18.0 // indirect
golang.org/x/sync v0.5.0 // indirect
golang.org/x/sys v0.14.0 // indirect

View file

@ -49,8 +49,8 @@ github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhP
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=
github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM=
github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII=
github.com/aws/aws-sdk-go v1.47.8 h1:VCFyO5UTREnhR0HRf9roqFfJeeRVin58zUy+pBMhwjY=
github.com/aws/aws-sdk-go v1.47.8/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
github.com/aws/aws-sdk-go v1.47.9 h1:rarTsos0mA16q+huicGx0e560aYRtOucV5z2Mw23JRY=
github.com/aws/aws-sdk-go v1.47.9/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM=
github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE=
@ -121,8 +121,8 @@ github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
@ -278,10 +278,10 @@ github.com/olahol/melody v1.1.4/go.mod h1:GgkTl6Y7yWj/HtfD48Q5vLKPVoZOH+Qqgfa7Cv
github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
github.com/onsi/gomega v1.28.0 h1:i2rg/p9n/UqIDAMFUJ6qIUUMcsqOuUHgbpbu235Vr1c=
github.com/onsi/gomega v1.28.0/go.mod h1:A1H2JE76sI14WIP57LMKj7FVfCHx3g3BcZVjJG8bjX8=
github.com/onsi/ginkgo/v2 v2.13.1 h1:LNGfMbR2OVGBfXjvRZIZ2YCTQdGKtPLvuI1rMCCj3OU=
github.com/onsi/ginkgo/v2 v2.13.1/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM=
github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg=
github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
github.com/open-spaced-repetition/go-fsrs v1.0.0 h1:FIH5MW29EEHTTLwXPeDWJqO4QA/b7L9i2GEH4lAEl0U=
github.com/open-spaced-repetition/go-fsrs v1.0.0/go.mod h1:WpbNs4TTKZChOHFO+ME0B9femUVZsepFT5mhAioszRg=
github.com/otiai10/gosseract/v2 v2.4.1 h1:G8AyBpXEeSlcq8TI85LH/pM5SXk8Djy2GEXisgyblRw=
@ -352,10 +352,8 @@ github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR
github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d h1:lvCTyBbr36+tqMccdGMwuEU+hjux/zL6xSmf5S9ITaA=
github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8=
github.com/siyuan-note/dejavu v0.0.0-20231110140645-e32f6a32cffa h1:m7+UGC/nVyLKGy3NxWFJ1cT1sJDLUJ+D0sO/8cyN+xo=
github.com/siyuan-note/dejavu v0.0.0-20231110140645-e32f6a32cffa/go.mod h1:6RdIofCDZP8vL0qYnVZfB12NhK7cUgh1ooUxRvRdlmI=
github.com/siyuan-note/dejavu v0.0.0-20231110145452-59dbd8bb6c2f h1:sq/GK4HappFB+VID9YphmZ1oVtj+Cc/4l6ZPSUy8r/g=
github.com/siyuan-note/dejavu v0.0.0-20231110145452-59dbd8bb6c2f/go.mod h1:6RdIofCDZP8vL0qYnVZfB12NhK7cUgh1ooUxRvRdlmI=
github.com/siyuan-note/dejavu v0.0.0-20231111144625-3b36f1124c31 h1:hfXp7QCYy+iTJjPT2r+BBM4kTTHQ7No5Rtx2zxOK+9w=
github.com/siyuan-note/dejavu v0.0.0-20231111144625-3b36f1124c31/go.mod h1:6RdIofCDZP8vL0qYnVZfB12NhK7cUgh1ooUxRvRdlmI=
github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 h1:Bi7/7f29LW+Fm0cHc0J1NO1cZqyJwljSWVmfOqVZgaE=
github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw=
github.com/siyuan-note/eventbus v0.0.0-20230804030110-cf250f838c80 h1:XghjHKJd+SiL0DkGYFVC+UGUDFtnR4v9gkAbPeh9Eq8=
@ -443,8 +441,8 @@ golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98y
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 h1:mchzmB1XO2pMaKFRqk/+MV3mgGG96aqaPXaMifQU47w=
golang.org/x/exp v0.0.0-20231108232855-2478ac86f678/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/image v0.11.0/go.mod h1:bglhjqbqVuEb9e9+eNR45Jfu7D+T4Qan+NhQk8Ck2P8=
golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4=

View file

@ -966,7 +966,7 @@ func syncRepoDownload() (err error) {
return
}
logging.LogInfof("downloading data repo [kernel=%s, provider=%d, mode=%s/%t]", KernelID, Conf.Sync.Provider, "d", true)
logging.LogInfof("downloading data repo [device=%s, kernel=%s, provider=%d, mode=%s/%t]", Conf.System.ID, KernelID, Conf.Sync.Provider, "d", true)
start := time.Now()
_, _, err = indexRepoBeforeCloudSync(repo)
if nil != err {
@ -1035,7 +1035,7 @@ func syncRepoUpload() (err error) {
return
}
logging.LogInfof("uploading data repo [kernel=%s, provider=%d, mode=%s/%t]", KernelID, Conf.Sync.Provider, "u", true)
logging.LogInfof("uploading data repo [device=%s, kernel=%s, provider=%d, mode=%s/%t]", Conf.System.ID, KernelID, Conf.Sync.Provider, "u", true)
start := time.Now()
_, _, err = indexRepoBeforeCloudSync(repo)
if nil != err {
@ -1195,7 +1195,7 @@ func syncRepo(exit, byHand bool) (dataChanged bool, err error) {
return
}
logging.LogInfof("syncing data repo [kernel=%s, provider=%d, mode=%s/%t]", KernelID, Conf.Sync.Provider, "a", byHand)
logging.LogInfof("syncing data repo [device=%s, kernel=%s, provider=%d, mode=%s/%t]", Conf.System.ID, KernelID, Conf.Sync.Provider, "a", byHand)
start := time.Now()
beforeIndex, afterIndex, err := indexRepoBeforeCloudSync(repo)
if nil != err {
@ -1251,8 +1251,8 @@ func syncRepo(exit, byHand bool) (dataChanged bool, err error) {
}
func processSyncMergeResult(exit, byHand bool, mergeResult *dejavu.MergeResult, trafficStat *dejavu.TrafficStat, mode string, elapsed time.Duration) {
logging.LogInfof("synced data repo [kernel=%s, provider=%d, mode=%s/%t, ufc=%d, dfc=%d, ucc=%d, dcc=%d, ub=%s, db=%s] in [%.2fs], merge result [conflicts=%d, upserts=%d, removes=%d]\n\n",
KernelID, Conf.Sync.Provider, mode, byHand,
logging.LogInfof("synced data repo [device=%s, kernel=%s, provider=%d, mode=%s/%t, ufc=%d, dfc=%d, ucc=%d, dcc=%d, ub=%s, db=%s] in [%.2fs], merge result [conflicts=%d, upserts=%d, removes=%d]\n\n",
Conf.System.ID, KernelID, Conf.Sync.Provider, mode, byHand,
trafficStat.UploadFileCount, trafficStat.DownloadFileCount, trafficStat.UploadChunkCount, trafficStat.DownloadChunkCount, humanize.Bytes(uint64(trafficStat.UploadBytes)), humanize.Bytes(uint64(trafficStat.DownloadBytes)),
elapsed.Seconds(),
len(mergeResult.Conflicts), len(mergeResult.Upserts), len(mergeResult.Removes))

View file

@ -98,7 +98,7 @@ func LoginAuth(c *gin.Context) {
if err := session.Save(c); nil != err {
logging.LogErrorf("save session failed: " + err.Error())
c.Status(500)
c.Status(http.StatusInternalServerError)
return
}
return
@ -109,7 +109,7 @@ func LoginAuth(c *gin.Context) {
workspaceSession.Captcha = gulu.Rand.String(7)
if err := session.Save(c); nil != err {
logging.LogErrorf("save session failed: " + err.Error())
c.Status(500)
c.Status(http.StatusInternalServerError)
return
}
}
@ -123,7 +123,7 @@ func GetCaptcha(c *gin.Context) {
})
if nil != err {
logging.LogErrorf("generates captcha failed: " + err.Error())
c.Status(500)
c.Status(http.StatusInternalServerError)
return
}
@ -132,16 +132,16 @@ func GetCaptcha(c *gin.Context) {
workspaceSession.Captcha = img.Text
if err = session.Save(c); nil != err {
logging.LogErrorf("save session failed: " + err.Error())
c.Status(500)
c.Status(http.StatusInternalServerError)
return
}
if err = img.WriteImage(c.Writer); nil != err {
logging.LogErrorf("writes captcha image failed: " + err.Error())
c.Status(500)
c.Status(http.StatusInternalServerError)
return
}
c.Status(200)
c.Status(http.StatusOK)
}
func CheckReadonly(c *gin.Context) {
@ -150,7 +150,7 @@ func CheckReadonly(c *gin.Context) {
result.Code = -1
result.Msg = Conf.Language(34)
result.Data = map[string]interface{}{"closeTimeout": 5000}
c.JSON(200, result)
c.JSON(http.StatusOK, result)
c.Abort()
return
}
@ -158,38 +158,21 @@ func CheckReadonly(c *gin.Context) {
func CheckAuth(c *gin.Context) {
//logging.LogInfof("check auth for [%s]", c.Request.RequestURI)
localhost := util.IsLocalHost(c.Request.RemoteAddr)
// 未设置访问授权码
if "" == Conf.AccessAuthCode {
if origin := c.GetHeader("Origin"); "" != origin {
// Authenticate requests with the Origin header other than 127.0.0.1 https://github.com/siyuan-note/siyuan/issues/9180
u, parseErr := url.Parse(origin)
if nil != parseErr {
logging.LogWarnf("parse origin [%s] failed: %s", origin, parseErr)
c.JSON(401, map[string]interface{}{"code": -1, "msg": "Auth failed: parse req header [Origin] failed"})
c.Abort()
return
}
if "chrome-extension" == strings.ToLower(u.Scheme) {
c.Next()
return
}
if !strings.HasPrefix(u.Host, util.LocalHost) && !strings.HasPrefix(u.Host, "[::1]") {
c.JSON(401, map[string]interface{}{"code": -1, "msg": "Auth failed: for security reasons, please set [Access authorization code] when using non-127.0.0.1 access\n\n为安全起见使用非 127.0.0.1 访问时请设置 [访问授权码]"})
c.Abort()
return
}
}
if !strings.HasPrefix(c.Request.RemoteAddr, util.LocalHost) && !strings.HasPrefix(c.Request.RemoteAddr, "[::1]") {
// Authenticate requests of assets other than 127.0.0.1 https://github.com/siyuan-note/siyuan/issues/9388
if strings.HasPrefix(c.Request.RequestURI, "/assets/") {
c.JSON(401, map[string]interface{}{"code": -1, "msg": "Auth failed: for security reasons, please set [Access authorization code] when using non-127.0.0.1 access\n\n为安全起见使用非 127.0.0.1 访问时请设置 [访问授权码]"})
c.Abort()
return
}
// Authenticate requests with the Origin header other than 127.0.0.1 https://github.com/siyuan-note/siyuan/issues/9180
host := c.GetHeader("Host")
origin := c.GetHeader("Origin")
forwardedHost := c.GetHeader("X-Forwarded-Host")
if !localhost ||
("" != host && !util.IsLocalHost(host)) ||
("" != origin && !util.IsLocalOrigin(origin) && !strings.HasPrefix(origin, "chrome-extension://")) ||
("" != forwardedHost && !util.IsLocalHost(forwardedHost)) {
c.JSON(http.StatusUnauthorized, map[string]interface{}{"code": -1, "msg": "Auth failed: for security reasons, please set [Access authorization code] when using non-127.0.0.1 access\n\n为安全起见使用非 127.0.0.1 访问时请设置 [访问授权码]"})
c.Abort()
return
}
c.Next()
@ -206,7 +189,7 @@ func CheckAuth(c *gin.Context) {
}
// 放过来自本机的某些请求
if strings.HasPrefix(c.Request.RemoteAddr, util.LocalHost) || strings.HasPrefix(c.Request.RemoteAddr, "[::1]") {
if localhost {
if strings.HasPrefix(c.Request.RequestURI, "/assets/") {
c.Next()
return
@ -234,7 +217,7 @@ func CheckAuth(c *gin.Context) {
return
}
c.JSON(401, map[string]interface{}{"code": -1, "msg": "Auth failed"})
c.JSON(http.StatusUnauthorized, map[string]interface{}{"code": -1, "msg": "Auth failed"})
c.Abort()
return
}
@ -247,7 +230,7 @@ func CheckAuth(c *gin.Context) {
return
}
c.JSON(401, map[string]interface{}{"code": -1, "msg": "Auth failed"})
c.JSON(http.StatusUnauthorized, map[string]interface{}{"code": -1, "msg": "Auth failed"})
c.Abort()
return
}
@ -261,7 +244,7 @@ func CheckAuth(c *gin.Context) {
userAgentHeader := c.GetHeader("User-Agent")
if strings.HasPrefix(userAgentHeader, "SiYuan/") || strings.HasPrefix(userAgentHeader, "Mozilla/") {
if "GET" != c.Request.Method {
c.JSON(401, map[string]interface{}{"code": -1, "msg": Conf.Language(156)})
c.JSON(http.StatusUnauthorized, map[string]interface{}{"code": -1, "msg": Conf.Language(156)})
c.Abort()
return
}
@ -271,12 +254,13 @@ func CheckAuth(c *gin.Context) {
queryParams.Set("to", c.Request.URL.String())
location.RawQuery = queryParams.Encode()
location.Path = "/check-auth"
c.Redirect(302, location.String())
c.Redirect(http.StatusFound, location.String())
c.Abort()
return
}
c.JSON(401, map[string]interface{}{"code": -1, "msg": "Auth failed"})
c.JSON(http.StatusUnauthorized, map[string]interface{}{"code": -1, "msg": "Auth failed"})
c.Abort()
return
}
@ -316,7 +300,7 @@ func Timing(c *gin.Context) {
func Recover(c *gin.Context) {
defer func() {
logging.Recover()
c.Status(500)
c.Status(http.StatusInternalServerError)
}()
c.Next()

View file

@ -17,6 +17,7 @@
package util
import (
"net"
"net/http"
"net/url"
"strings"
@ -31,6 +32,58 @@ import (
"github.com/siyuan-note/logging"
)
func ValidOptionalPort(port string) bool {
if port == "" {
return true
}
if port[0] != ':' {
return false
}
for _, b := range port[1:] {
if b < '0' || b > '9' {
return false
}
}
return true
}
func SplitHost(host string) (hostname, port string) {
hostname = host
colon := strings.LastIndexByte(hostname, ':')
if colon != -1 && ValidOptionalPort(hostname[colon:]) {
hostname, port = hostname[:colon], hostname[colon+1:]
}
if strings.HasPrefix(hostname, "[") && strings.HasSuffix(hostname, "]") {
hostname = hostname[1 : len(hostname)-1]
}
return
}
func IsLocalHostname(hostname string) bool {
if "localhost" == hostname {
return true
}
if ip := net.ParseIP(hostname); nil != ip {
return ip.IsLoopback()
}
return false
}
func IsLocalHost(host string) bool {
hostname, _ := SplitHost(host)
return IsLocalHostname(hostname)
}
func IsLocalOrigin(origin string) bool {
if url, err := url.Parse(origin); nil == err {
return IsLocalHostname(url.Hostname())
}
return false
}
func IsOnline(checkURL string, skipTlsVerify bool) bool {
_, err := url.Parse(checkURL)
if nil != err {