diff --git a/docs/webdav.md b/docs/webdav.md index 8ba94cb7..25922579 100644 --- a/docs/webdav.md +++ b/docs/webdav.md @@ -29,6 +29,8 @@ Know issues: - if a file or a directory cannot be accessed, for example due to OS permissions issues or because a mapped path for a virtual folder is a missing, it will be omitted from the directory listing. If there is a different error then the whole directory listing will fail. This behavior is different from SFTP/FTP where you will be able to see the problematic file/directory in the directory listing, you will only get an error if you try to access it - if you use the native Windows client please check its usage and pay particular attention to the [registry settings](https://docs.microsoft.com/en-us/iis/publish/using-webdav/using-the-webdav-redirector#webdav-redirector-registry-settings). The default file size limit is 50MB and if you don't configure SFTPGo to use HTTPS you have to set `BasicAuthLevel` to `2` -We plan to add [Dead Properties](https://tools.ietf.org/html/rfc4918#section-3) support in future releases. We need a design decision here, probably the best solution is to store dead properties inside the data provider but this could increase a lot its size. Alternately we could store them on disk for local filesystem and add as metadata for Cloud Storage, this means that we need to do a separate `HEAD` request to retrieve dead properties for an S3 file. For big folders will do a lot of requests to the Cloud Provider, I don't like this solution. Another option is to expose a hook and allow you to implement `dead properties` outside SFTPGo. +SFTPGo has a minimal implementation for [Dead Properties](https://tools.ietf.org/html/rfc4918#section-3). We support setting the last modification time and we return the value in the "live" properties, so basically we don't store anything. + +To properly support dead properties we need a design decision, probably the best solution is to write a plugin and store them inside a supported data provider. If you find any other quirks or problems please let us know opening a GitHub issue, thank you! diff --git a/go.mod b/go.mod index 8403b9f7..e1b4655d 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( cloud.google.com/go/storage v1.27.0 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 github.com/alexedwards/argon2id v0.0.0-20211130144151-3585854a6387 github.com/aws/aws-sdk-go-v2 v1.16.16 @@ -54,7 +54,7 @@ require ( github.com/sftpgo/sdk v0.1.2-0.20220913155952-81743fa5ded5 github.com/shirou/gopsutil/v3 v3.22.9 github.com/spf13/afero v1.9.2 - github.com/spf13/cobra v1.5.0 + github.com/spf13/cobra v1.6.0 github.com/spf13/viper v1.13.0 github.com/stretchr/testify v1.8.0 github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62 @@ -65,7 +65,7 @@ require ( go.etcd.io/bbolt v1.3.6 go.uber.org/automaxprocs v1.5.1 gocloud.dev v0.27.0 - golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b + golang.org/x/crypto v0.0.0-20221010152910-d6f0a8c073c2 golang.org/x/net v0.0.0-20221004154528-8021a29435af golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1 golang.org/x/sys v0.0.0-20221010170243-090e33056c14 @@ -155,7 +155,7 @@ require ( github.com/yusufpapurcu/wmi v1.2.2 // indirect go.opencensus.io v0.23.0 // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/text v0.3.7 // indirect + golang.org/x/text v0.3.8 // indirect golang.org/x/tools v0.1.12 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect @@ -171,6 +171,6 @@ require ( replace ( github.com/jlaffaye/ftp => github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9 github.com/pkg/sftp => github.com/drakkan/sftp v0.0.0-20220930161944-e8c89afc13a7 - golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20221007045301-0163d4cbe5a2 - golang.org/x/net => github.com/drakkan/net v0.0.0-20221007045029-3c39cb455af9 + golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20221011170652-7c454d6a47a0 + golang.org/x/net => github.com/drakkan/net v0.0.0-20221011170324-793589996ca2 ) diff --git a/go.sum b/go.sum index d58cd7d9..65a599f6 100644 --- a/go.sum +++ b/go.sum @@ -109,8 +109,8 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQi github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.0.2/go.mod h1:LH9XQnMr2ZYxQdVdCrzLO9mxeDyrDFa6wbSI3x5zCZk= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1/go.mod h1:eZ4g6GUvXiGulfIbbhh1Xr4XwUYaYaWMqzGD/284wCA= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.0 h1:fe+kSd9btgTTeHeUlMTyEsjoe6L/zd+Q61iWEMPwHmc= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.0/go.mod h1:T7nxmZ9i42Dqy7kwnn8AZYNjqxd4TloKXdIbhosHSqo= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= github.com/Azure/go-amqp v0.17.0/go.mod h1:9YJ3RhxRT1gquYnzpZO1vcYMMpAdJT+QEg6fwmw9Zlg= github.com/Azure/go-amqp v0.17.5/go.mod h1:9YJ3RhxRT1gquYnzpZO1vcYMMpAdJT+QEg6fwmw9Zlg= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= @@ -535,12 +535,12 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/drakkan/crypto v0.0.0-20221007045301-0163d4cbe5a2 h1:VW04POuNbYg0uLKufFxgr9RnvvNKVc/Mhqq3EUx5tnE= -github.com/drakkan/crypto v0.0.0-20221007045301-0163d4cbe5a2/go.mod h1:SiM6ypd8Xu1xldObYtbDztuUU7xUzMnUULfphXFZmro= +github.com/drakkan/crypto v0.0.0-20221011170652-7c454d6a47a0 h1:H8T0AOkrwrWacTEY8nP1PDQ+kUMeTQbCu8OY+x0/mXY= +github.com/drakkan/crypto v0.0.0-20221011170652-7c454d6a47a0/go.mod h1:SiM6ypd8Xu1xldObYtbDztuUU7xUzMnUULfphXFZmro= github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9 h1:LPH1dEblAOO/LoG7yHPMtBLXhQmjaga91/DDjWk9jWA= github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU= -github.com/drakkan/net v0.0.0-20221007045029-3c39cb455af9 h1:LuHrwSqNqg+MNxDUgg+5uSoxADH1z/PMU3zKM4V/bqM= -github.com/drakkan/net v0.0.0-20221007045029-3c39cb455af9/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +github.com/drakkan/net v0.0.0-20221011170324-793589996ca2 h1:nFZmCADOhW2YZ51/IKcODGsJsg3cX1VGVgjFDQYuVKs= +github.com/drakkan/net v0.0.0-20221011170324-793589996ca2/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= github.com/drakkan/sftp v0.0.0-20220930161944-e8c89afc13a7 h1:Hj7AAfZ5yt9QuCxSQDllRygmL33xJ2sZLOmcyyOAdYU= github.com/drakkan/sftp v0.0.0-20220930161944-e8c89afc13a7/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -1490,8 +1490,8 @@ github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKv github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= -github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= +github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI= +github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -1931,8 +1931,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/internal/dataprovider/mysql.go b/internal/dataprovider/mysql.go index b064fc98..948052bd 100644 --- a/internal/dataprovider/mysql.go +++ b/internal/dataprovider/mysql.go @@ -220,6 +220,7 @@ func initializeMySQLProvider() error { dbHandle.SetMaxIdleConns(2) } dbHandle.SetConnMaxLifetime(240 * time.Second) + dbHandle.SetConnMaxIdleTime(120 * time.Second) provider = &MySQLProvider{dbHandle: dbHandle} } else { providerLog(logger.LevelError, "error creating mysql database handler, connection string: %#v, error: %v", diff --git a/internal/dataprovider/pgsql.go b/internal/dataprovider/pgsql.go index 88db99f1..0e2d5c43 100644 --- a/internal/dataprovider/pgsql.go +++ b/internal/dataprovider/pgsql.go @@ -226,6 +226,7 @@ func initializePGSQLProvider() error { dbHandle.SetMaxIdleConns(2) } dbHandle.SetConnMaxLifetime(240 * time.Second) + dbHandle.SetConnMaxIdleTime(120 * time.Second) provider = &PGSQLProvider{dbHandle: dbHandle} } else { providerLog(logger.LevelError, "error creating postgres database handler, connection string: %#v, error: %v", diff --git a/internal/webdavd/file.go b/internal/webdavd/file.go index 50af68b0..adb92c00 100644 --- a/internal/webdavd/file.go +++ b/internal/webdavd/file.go @@ -16,9 +16,11 @@ package webdavd import ( "context" + "encoding/xml" "errors" "io" "mime" + "net/http" "os" "path" "sync/atomic" @@ -30,10 +32,14 @@ import ( "github.com/drakkan/sftpgo/v2/internal/common" "github.com/drakkan/sftpgo/v2/internal/dataprovider" "github.com/drakkan/sftpgo/v2/internal/logger" + "github.com/drakkan/sftpgo/v2/internal/util" "github.com/drakkan/sftpgo/v2/internal/vfs" ) -var errTransferAborted = errors.New("transfer aborted") +var ( + errTransferAborted = errors.New("transfer aborted") + lastModifiedProps = []string{"Win32LastModifiedTime", "getlastmodified"} +) type webDavFile struct { *common.BaseTransfer @@ -374,3 +380,53 @@ func (f *webDavFile) isTransfer() bool { } return true } + +// DeadProps returns a copy of the dead properties held. +// We always return nil for now, we only support the last modification time +// and it is already included in "live" properties +func (f *webDavFile) DeadProps() (map[xml.Name]webdav.Property, error) { + return nil, nil +} + +// Patch patches the dead properties held. +// In our minimal implementation we just support Win32LastModifiedTime and +// getlastmodified to set the the modification time. +// We ignore any other property and just return an OK response if the patch sets +// the modification time, otherwise a Forbidden response +func (f *webDavFile) Patch(patches []webdav.Proppatch) ([]webdav.Propstat, error) { + resp := make([]webdav.Propstat, 0, len(patches)) + hasError := false + for _, patch := range patches { + status := http.StatusForbidden + pstat := webdav.Propstat{} + for _, p := range patch.Props { + if status == http.StatusForbidden && !hasError { + if !patch.Remove && util.Contains(lastModifiedProps, p.XMLName.Local) { + parsed, err := http.ParseTime(string(p.InnerXML)) + if err != nil { + f.Connection.Log(logger.LevelWarn, "unsupported last modification time: %q, err: %v", + string(p.InnerXML), err) + hasError = true + continue + } + attrs := &common.StatAttributes{ + Flags: common.StatAttrTimes, + Atime: parsed, + Mtime: parsed, + } + if err := f.Connection.SetStat(f.GetVirtualPath(), attrs); err != nil { + f.Connection.Log(logger.LevelWarn, "unable to set modification time for %q, err :%v", + f.GetVirtualPath(), err) + hasError = true + continue + } + status = http.StatusOK + } + } + pstat.Props = append(pstat.Props, webdav.Property{XMLName: p.XMLName}) + } + pstat.Status = status + resp = append(resp, pstat) + } + return resp, nil +} diff --git a/internal/webdavd/internal_test.go b/internal/webdavd/internal_test.go index 4209927f..ee479adc 100644 --- a/internal/webdavd/internal_test.go +++ b/internal/webdavd/internal_test.go @@ -18,6 +18,7 @@ import ( "context" "crypto/tls" "crypto/x509" + "encoding/xml" "errors" "fmt" "io" @@ -663,10 +664,46 @@ func TestFileAccessErrors(t *testing.T) { assert.NoError(t, err) assert.Equal(t, 0, len(connection.GetTransfers())) } - } + // test PROPPATCH date parsing error + pstats, err := transfer.Patch([]webdav.Proppatch{ + { + Props: []webdav.Property{ + { + XMLName: xml.Name{ + Space: "DAV", + Local: "getlastmodified", + }, + InnerXML: []byte(`Wid, 04 Nov 2020 13:25:51 GMT`), + }, + }, + }, + }) + assert.NoError(t, err) + for _, pstat := range pstats { + assert.Equal(t, http.StatusForbidden, pstat.Status) + } - err = os.Remove(f.Name()) - assert.NoError(t, err) + err = os.Remove(f.Name()) + assert.NoError(t, err) + // the file is deleted PROPPATCH should fail + pstats, err = transfer.Patch([]webdav.Proppatch{ + { + Props: []webdav.Property{ + { + XMLName: xml.Name{ + Space: "DAV", + Local: "getlastmodified", + }, + InnerXML: []byte(`Wed, 04 Nov 2020 13:25:51 GMT`), + }, + }, + }, + }) + assert.NoError(t, err) + for _, pstat := range pstats { + assert.Equal(t, http.StatusForbidden, pstat.Status) + } + } } func TestRemoveDirTree(t *testing.T) { diff --git a/internal/webdavd/webdavd_test.go b/internal/webdavd/webdavd_test.go index 173d43f9..f1100837 100644 --- a/internal/webdavd/webdavd_test.go +++ b/internal/webdavd/webdavd_test.go @@ -895,13 +895,26 @@ func TestPropPatch(t *testing.T) { req.SetBasicAuth(u.Username, u.Password) resp, err := httpClient.Do(req) assert.NoError(t, err) - assert.Equal(t, 207, resp.StatusCode) + assert.Equal(t, http.StatusMultiStatus, resp.StatusCode) err = resp.Body.Close() assert.NoError(t, err) info, err := client.Stat(testFileName) if assert.NoError(t, err) { + expected, err := http.ParseTime("Wed, 04 Nov 2020 13:25:51 GMT") + assert.NoError(t, err) assert.Equal(t, testFileSize, info.Size()) + assert.Equal(t, expected.Format(http.TimeFormat), info.ModTime().Format(http.TimeFormat)) } + // wrong date + propatchBody = `Wed, 04 Nov 2020 13:25:51 GMTSat, 05 Dec 2020 21:16:12 GMTWid, 04 Nov 2020 13:25:51 GMT00000000` + req, err = http.NewRequest("PROPPATCH", fmt.Sprintf("http://%v/%v", webDavServerAddr, testFileName), bytes.NewReader([]byte(propatchBody))) + assert.NoError(t, err) + req.SetBasicAuth(u.Username, u.Password) + resp, err = httpClient.Do(req) + assert.NoError(t, err) + assert.Equal(t, http.StatusMultiStatus, resp.StatusCode) + err = resp.Body.Close() + assert.NoError(t, err) err = os.Remove(testFilePath) assert.NoError(t, err) _, err = httpdtest.RemoveUser(user, http.StatusOK)