WebDAV: allow to set last modification time

This commit add a minimal dead properties implementation

Fixes #1018

Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
This commit is contained in:
Nicola Murino 2022-10-11 19:20:58 +02:00
parent 0e54fa5655
commit 07012aa812
No known key found for this signature in database
GPG key ID: 935D2952DEC4EECF
8 changed files with 132 additions and 21 deletions

View file

@ -29,6 +29,8 @@ Know issues:
- if a file or a directory cannot be accessed, for example due to OS permissions issues or because a mapped path for a virtual folder is a missing, it will be omitted from the directory listing. If there is a different error then the whole directory listing will fail. This behavior is different from SFTP/FTP where you will be able to see the problematic file/directory in the directory listing, you will only get an error if you try to access it
- if you use the native Windows client please check its usage and pay particular attention to the [registry settings](https://docs.microsoft.com/en-us/iis/publish/using-webdav/using-the-webdav-redirector#webdav-redirector-registry-settings). The default file size limit is 50MB and if you don't configure SFTPGo to use HTTPS you have to set `BasicAuthLevel` to `2`
We plan to add [Dead Properties](https://tools.ietf.org/html/rfc4918#section-3) support in future releases. We need a design decision here, probably the best solution is to store dead properties inside the data provider but this could increase a lot its size. Alternately we could store them on disk for local filesystem and add as metadata for Cloud Storage, this means that we need to do a separate `HEAD` request to retrieve dead properties for an S3 file. For big folders will do a lot of requests to the Cloud Provider, I don't like this solution. Another option is to expose a hook and allow you to implement `dead properties` outside SFTPGo.
SFTPGo has a minimal implementation for [Dead Properties](https://tools.ietf.org/html/rfc4918#section-3). We support setting the last modification time and we return the value in the "live" properties, so basically we don't store anything.
To properly support dead properties we need a design decision, probably the best solution is to write a plugin and store them inside a supported data provider.
If you find any other quirks or problems please let us know opening a GitHub issue, thank you!

12
go.mod
View file

@ -5,7 +5,7 @@ go 1.19
require (
cloud.google.com/go/storage v1.27.0
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1
github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962
github.com/alexedwards/argon2id v0.0.0-20211130144151-3585854a6387
github.com/aws/aws-sdk-go-v2 v1.16.16
@ -54,7 +54,7 @@ require (
github.com/sftpgo/sdk v0.1.2-0.20220913155952-81743fa5ded5
github.com/shirou/gopsutil/v3 v3.22.9
github.com/spf13/afero v1.9.2
github.com/spf13/cobra v1.5.0
github.com/spf13/cobra v1.6.0
github.com/spf13/viper v1.13.0
github.com/stretchr/testify v1.8.0
github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62
@ -65,7 +65,7 @@ require (
go.etcd.io/bbolt v1.3.6
go.uber.org/automaxprocs v1.5.1
gocloud.dev v0.27.0
golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b
golang.org/x/crypto v0.0.0-20221010152910-d6f0a8c073c2
golang.org/x/net v0.0.0-20221004154528-8021a29435af
golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1
golang.org/x/sys v0.0.0-20221010170243-090e33056c14
@ -155,7 +155,7 @@ require (
github.com/yusufpapurcu/wmi v1.2.2 // indirect
go.opencensus.io v0.23.0 // indirect
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/text v0.3.8 // indirect
golang.org/x/tools v0.1.12 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect
@ -171,6 +171,6 @@ require (
replace (
github.com/jlaffaye/ftp => github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9
github.com/pkg/sftp => github.com/drakkan/sftp v0.0.0-20220930161944-e8c89afc13a7
golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20221007045301-0163d4cbe5a2
golang.org/x/net => github.com/drakkan/net v0.0.0-20221007045029-3c39cb455af9
golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20221011170652-7c454d6a47a0
golang.org/x/net => github.com/drakkan/net v0.0.0-20221011170324-793589996ca2
)

19
go.sum
View file

@ -109,8 +109,8 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQi
github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.0.2/go.mod h1:LH9XQnMr2ZYxQdVdCrzLO9mxeDyrDFa6wbSI3x5zCZk=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1/go.mod h1:eZ4g6GUvXiGulfIbbhh1Xr4XwUYaYaWMqzGD/284wCA=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.0 h1:fe+kSd9btgTTeHeUlMTyEsjoe6L/zd+Q61iWEMPwHmc=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.0/go.mod h1:T7nxmZ9i42Dqy7kwnn8AZYNjqxd4TloKXdIbhosHSqo=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU=
github.com/Azure/go-amqp v0.17.0/go.mod h1:9YJ3RhxRT1gquYnzpZO1vcYMMpAdJT+QEg6fwmw9Zlg=
github.com/Azure/go-amqp v0.17.5/go.mod h1:9YJ3RhxRT1gquYnzpZO1vcYMMpAdJT+QEg6fwmw9Zlg=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
@ -535,12 +535,12 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/drakkan/crypto v0.0.0-20221007045301-0163d4cbe5a2 h1:VW04POuNbYg0uLKufFxgr9RnvvNKVc/Mhqq3EUx5tnE=
github.com/drakkan/crypto v0.0.0-20221007045301-0163d4cbe5a2/go.mod h1:SiM6ypd8Xu1xldObYtbDztuUU7xUzMnUULfphXFZmro=
github.com/drakkan/crypto v0.0.0-20221011170652-7c454d6a47a0 h1:H8T0AOkrwrWacTEY8nP1PDQ+kUMeTQbCu8OY+x0/mXY=
github.com/drakkan/crypto v0.0.0-20221011170652-7c454d6a47a0/go.mod h1:SiM6ypd8Xu1xldObYtbDztuUU7xUzMnUULfphXFZmro=
github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9 h1:LPH1dEblAOO/LoG7yHPMtBLXhQmjaga91/DDjWk9jWA=
github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU=
github.com/drakkan/net v0.0.0-20221007045029-3c39cb455af9 h1:LuHrwSqNqg+MNxDUgg+5uSoxADH1z/PMU3zKM4V/bqM=
github.com/drakkan/net v0.0.0-20221007045029-3c39cb455af9/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
github.com/drakkan/net v0.0.0-20221011170324-793589996ca2 h1:nFZmCADOhW2YZ51/IKcODGsJsg3cX1VGVgjFDQYuVKs=
github.com/drakkan/net v0.0.0-20221011170324-793589996ca2/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
github.com/drakkan/sftp v0.0.0-20220930161944-e8c89afc13a7 h1:Hj7AAfZ5yt9QuCxSQDllRygmL33xJ2sZLOmcyyOAdYU=
github.com/drakkan/sftp v0.0.0-20220930161944-e8c89afc13a7/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg=
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@ -1490,8 +1490,8 @@ github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKv
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI=
github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
@ -1931,8 +1931,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

View file

@ -220,6 +220,7 @@ func initializeMySQLProvider() error {
dbHandle.SetMaxIdleConns(2)
}
dbHandle.SetConnMaxLifetime(240 * time.Second)
dbHandle.SetConnMaxIdleTime(120 * time.Second)
provider = &MySQLProvider{dbHandle: dbHandle}
} else {
providerLog(logger.LevelError, "error creating mysql database handler, connection string: %#v, error: %v",

View file

@ -226,6 +226,7 @@ func initializePGSQLProvider() error {
dbHandle.SetMaxIdleConns(2)
}
dbHandle.SetConnMaxLifetime(240 * time.Second)
dbHandle.SetConnMaxIdleTime(120 * time.Second)
provider = &PGSQLProvider{dbHandle: dbHandle}
} else {
providerLog(logger.LevelError, "error creating postgres database handler, connection string: %#v, error: %v",

View file

@ -16,9 +16,11 @@ package webdavd
import (
"context"
"encoding/xml"
"errors"
"io"
"mime"
"net/http"
"os"
"path"
"sync/atomic"
@ -30,10 +32,14 @@ import (
"github.com/drakkan/sftpgo/v2/internal/common"
"github.com/drakkan/sftpgo/v2/internal/dataprovider"
"github.com/drakkan/sftpgo/v2/internal/logger"
"github.com/drakkan/sftpgo/v2/internal/util"
"github.com/drakkan/sftpgo/v2/internal/vfs"
)
var errTransferAborted = errors.New("transfer aborted")
var (
errTransferAborted = errors.New("transfer aborted")
lastModifiedProps = []string{"Win32LastModifiedTime", "getlastmodified"}
)
type webDavFile struct {
*common.BaseTransfer
@ -374,3 +380,53 @@ func (f *webDavFile) isTransfer() bool {
}
return true
}
// DeadProps returns a copy of the dead properties held.
// We always return nil for now, we only support the last modification time
// and it is already included in "live" properties
func (f *webDavFile) DeadProps() (map[xml.Name]webdav.Property, error) {
return nil, nil
}
// Patch patches the dead properties held.
// In our minimal implementation we just support Win32LastModifiedTime and
// getlastmodified to set the the modification time.
// We ignore any other property and just return an OK response if the patch sets
// the modification time, otherwise a Forbidden response
func (f *webDavFile) Patch(patches []webdav.Proppatch) ([]webdav.Propstat, error) {
resp := make([]webdav.Propstat, 0, len(patches))
hasError := false
for _, patch := range patches {
status := http.StatusForbidden
pstat := webdav.Propstat{}
for _, p := range patch.Props {
if status == http.StatusForbidden && !hasError {
if !patch.Remove && util.Contains(lastModifiedProps, p.XMLName.Local) {
parsed, err := http.ParseTime(string(p.InnerXML))
if err != nil {
f.Connection.Log(logger.LevelWarn, "unsupported last modification time: %q, err: %v",
string(p.InnerXML), err)
hasError = true
continue
}
attrs := &common.StatAttributes{
Flags: common.StatAttrTimes,
Atime: parsed,
Mtime: parsed,
}
if err := f.Connection.SetStat(f.GetVirtualPath(), attrs); err != nil {
f.Connection.Log(logger.LevelWarn, "unable to set modification time for %q, err :%v",
f.GetVirtualPath(), err)
hasError = true
continue
}
status = http.StatusOK
}
}
pstat.Props = append(pstat.Props, webdav.Property{XMLName: p.XMLName})
}
pstat.Status = status
resp = append(resp, pstat)
}
return resp, nil
}

View file

@ -18,6 +18,7 @@ import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/xml"
"errors"
"fmt"
"io"
@ -663,10 +664,46 @@ func TestFileAccessErrors(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, 0, len(connection.GetTransfers()))
}
// test PROPPATCH date parsing error
pstats, err := transfer.Patch([]webdav.Proppatch{
{
Props: []webdav.Property{
{
XMLName: xml.Name{
Space: "DAV",
Local: "getlastmodified",
},
InnerXML: []byte(`Wid, 04 Nov 2020 13:25:51 GMT`),
},
},
},
})
assert.NoError(t, err)
for _, pstat := range pstats {
assert.Equal(t, http.StatusForbidden, pstat.Status)
}
err = os.Remove(f.Name())
assert.NoError(t, err)
// the file is deleted PROPPATCH should fail
pstats, err = transfer.Patch([]webdav.Proppatch{
{
Props: []webdav.Property{
{
XMLName: xml.Name{
Space: "DAV",
Local: "getlastmodified",
},
InnerXML: []byte(`Wed, 04 Nov 2020 13:25:51 GMT`),
},
},
},
})
assert.NoError(t, err)
for _, pstat := range pstats {
assert.Equal(t, http.StatusForbidden, pstat.Status)
}
}
}
func TestRemoveDirTree(t *testing.T) {

View file

@ -895,13 +895,26 @@ func TestPropPatch(t *testing.T) {
req.SetBasicAuth(u.Username, u.Password)
resp, err := httpClient.Do(req)
assert.NoError(t, err)
assert.Equal(t, 207, resp.StatusCode)
assert.Equal(t, http.StatusMultiStatus, resp.StatusCode)
err = resp.Body.Close()
assert.NoError(t, err)
info, err := client.Stat(testFileName)
if assert.NoError(t, err) {
expected, err := http.ParseTime("Wed, 04 Nov 2020 13:25:51 GMT")
assert.NoError(t, err)
assert.Equal(t, testFileSize, info.Size())
assert.Equal(t, expected.Format(http.TimeFormat), info.ModTime().Format(http.TimeFormat))
}
// wrong date
propatchBody = `<?xml version="1.0" encoding="utf-8" ?><D:propertyupdate xmlns:D="DAV:" xmlns:Z="urn:schemas-microsoft-com:"><D:set><D:prop><Z:Win32CreationTime>Wed, 04 Nov 2020 13:25:51 GMT</Z:Win32CreationTime><Z:Win32LastAccessTime>Sat, 05 Dec 2020 21:16:12 GMT</Z:Win32LastAccessTime><Z:Win32LastModifiedTime>Wid, 04 Nov 2020 13:25:51 GMT</Z:Win32LastModifiedTime><Z:Win32FileAttributes>00000000</Z:Win32FileAttributes></D:prop></D:set></D:propertyupdate>`
req, err = http.NewRequest("PROPPATCH", fmt.Sprintf("http://%v/%v", webDavServerAddr, testFileName), bytes.NewReader([]byte(propatchBody)))
assert.NoError(t, err)
req.SetBasicAuth(u.Username, u.Password)
resp, err = httpClient.Do(req)
assert.NoError(t, err)
assert.Equal(t, http.StatusMultiStatus, resp.StatusCode)
err = resp.Body.Close()
assert.NoError(t, err)
err = os.Remove(testFilePath)
assert.NoError(t, err)
_, err = httpdtest.RemoveUser(user, http.StatusOK)