From 81433e00d15e52177a4f3721305be23e3e5dbe29 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Fri, 9 Aug 2024 20:18:33 +0200 Subject: [PATCH] event action: add update modtime to fs rename Signed-off-by: Nicola Murino --- go.mod | 16 ++-- go.sum | 32 +++---- internal/common/connection.go | 8 +- internal/common/eventmanager.go | 10 +- internal/common/eventmanager_test.go | 24 +++-- internal/common/protocol_test.go | 137 ++++++++++++++++----------- internal/common/transfer.go | 2 +- internal/dataprovider/eventrule.go | 44 +++++++-- internal/ftpd/handler.go | 2 +- internal/ftpd/internal_test.go | 2 +- internal/httpd/handler.go | 2 +- internal/httpd/httpd_test.go | 37 +++++--- internal/httpd/webadmin.go | 26 ++++- internal/httpdtest/httpdtest.go | 25 ++++- internal/sftpd/handler.go | 2 +- internal/sftpd/internal_test.go | 2 +- internal/sftpd/scp.go | 2 +- internal/vfs/azblobfs.go | 61 ++++++++---- internal/vfs/gcsfs.go | 29 ++++-- internal/vfs/httpfs.go | 5 +- internal/vfs/osfs.go | 8 +- internal/vfs/s3fs.go | 30 ++++-- internal/vfs/sftpfs.go | 8 +- internal/vfs/vfs.go | 15 +-- internal/webdavd/handler.go | 2 +- internal/webdavd/internal_test.go | 2 +- openapi/openapi.yaml | 10 +- static/locales/en/translation.json | 1 + static/locales/it/translation.json | 1 + templates/webadmin/eventaction.html | 20 +++- 30 files changed, 383 insertions(+), 182 deletions(-) diff --git a/go.mod b/go.mod index d8dfbb45..169f8afc 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22.2 require ( cloud.google.com/go/storage v1.43.0 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0 github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 github.com/alexedwards/argon2id v1.0.0 @@ -70,7 +70,7 @@ require ( golang.org/x/crypto v0.26.0 golang.org/x/net v0.28.0 golang.org/x/oauth2 v0.22.0 - golang.org/x/sys v0.23.0 + golang.org/x/sys v0.24.0 golang.org/x/term v0.23.0 golang.org/x/time v0.6.0 google.golang.org/api v0.191.0 @@ -80,9 +80,9 @@ require ( require ( cloud.google.com/go v0.115.0 // indirect cloud.google.com/go/auth v0.8.0 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect cloud.google.com/go/compute/metadata v0.5.0 // indirect - cloud.google.com/go/iam v1.1.12 // indirect + cloud.google.com/go/iam v1.1.13 // indirect filippo.io/edwards25519 v1.1.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect github.com/ajg/form v1.5.1 // indirect @@ -167,15 +167,15 @@ require ( go.opentelemetry.io/otel/metric v1.28.0 // indirect go.opentelemetry.io/otel/trace v1.28.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect golang.org/x/mod v0.20.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/text v0.17.0 // indirect golang.org/x/tools v0.24.0 // indirect golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect - google.golang.org/genproto v0.0.0-20240805194559-2c9e96a0b5d4 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240805194559-2c9e96a0b5d4 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240805194559-2c9e96a0b5d4 // indirect + google.golang.org/genproto v0.0.0-20240808171019-573a1156607a // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240808171019-573a1156607a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240808171019-573a1156607a // indirect google.golang.org/grpc v1.65.0 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index 39d7f1bd..1333af0c 100644 --- a/go.sum +++ b/go.sum @@ -3,12 +3,12 @@ cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= cloud.google.com/go/auth v0.8.0 h1:y8jUJLl/Fg+qNBWxP/Hox2ezJvjkrPb952PC1p0G6A4= cloud.google.com/go/auth v0.8.0/go.mod h1:qGVp/Y3kDRSDZ5gFD/XPUfYQ9xW1iI7q8RIRoCyBbJc= -cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI= -cloud.google.com/go/auth/oauth2adapt v0.2.3/go.mod h1:tMQXOfZzFuNuUxOypHlQEXgdfX5cuhwU+ffUuXRJE8I= +cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= +cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= -cloud.google.com/go/iam v1.1.12 h1:JixGLimRrNGcxvJEQ8+clfLxPlbeZA6MuRJ+qJNQ5Xw= -cloud.google.com/go/iam v1.1.12/go.mod h1:9LDX8J7dN5YRyzVHxwQzrQs9opFFqn0Mxs9nAeB+Hhg= +cloud.google.com/go/iam v1.1.13 h1:7zWBXG9ERbMLrzQBRhFliAV+kjcRToDTgQT3CTwYyv4= +cloud.google.com/go/iam v1.1.13/go.mod h1:K8mY0uSXwEXS30KrnVb+j54LB/ntfZu1dr+4zFMNbus= cloud.google.com/go/kms v1.18.4 h1:dYN3OCsQ6wJLLtOnI8DGUwQ5shMusXsWCCC+s09ATsk= cloud.google.com/go/kms v1.18.4/go.mod h1:SG1bgQ3UWW6/KdPo9uuJnzELXY5YTTMJtDYvajiQ22g= cloud.google.com/go/longrunning v0.5.11 h1:Havn1kGjz3whCfoD8dxMLP73Ph5w+ODyZB9RUsDxtGk= @@ -18,8 +18,8 @@ cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502Jw filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0 h1:GJHeeA2N7xrG3q30L2UXDyuWRzDM900/65j70wcM4Ww= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= @@ -419,8 +419,8 @@ go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8 gocloud.dev v0.38.0 h1:SpxfaOc/Fp4PeO8ui7wRcCZV0EgXZ+IWcVSLn6ZMSw0= gocloud.dev v0.38.0/go.mod h1:3XjKvd2E5iVNu/xFImRzjN0d/fkNHe4s0RiKidpEUMQ= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI= +golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -479,8 +479,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -525,12 +525,12 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20240805194559-2c9e96a0b5d4 h1:g+rQ3aqOyXK/0qwnC5TGUXnyIeipstP5SsniB9uPJ2c= -google.golang.org/genproto v0.0.0-20240805194559-2c9e96a0b5d4/go.mod h1:7uvplUBj4RjHAxIZ//98LzOvrQ04JBkaixRmCMI29hc= -google.golang.org/genproto/googleapis/api v0.0.0-20240805194559-2c9e96a0b5d4 h1:ABEBT/sZ7We8zd7A5f3KO6zMQe+s3901H7l8Whhijt0= -google.golang.org/genproto/googleapis/api v0.0.0-20240805194559-2c9e96a0b5d4/go.mod h1:4+X6GvPs+25wZKbQq9qyAXrwIRExv7w0Ea6MgZLZiDM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240805194559-2c9e96a0b5d4 h1:OsSGQeIIsyOEOimVxLEIL4rwGcnrjOydQaiA2bOnZUM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240805194559-2c9e96a0b5d4/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/genproto v0.0.0-20240808171019-573a1156607a h1:3JVv3Ujh+kGiajpSqHWnbWPuu0nQqMZ3hASNDDF9974= +google.golang.org/genproto v0.0.0-20240808171019-573a1156607a/go.mod h1:7uvplUBj4RjHAxIZ//98LzOvrQ04JBkaixRmCMI29hc= +google.golang.org/genproto/googleapis/api v0.0.0-20240808171019-573a1156607a h1:KyUe15n7B1YCu+kMmPtlXxgkLQbp+Dw0tCRZf9Sd+CE= +google.golang.org/genproto/googleapis/api v0.0.0-20240808171019-573a1156607a/go.mod h1:4+X6GvPs+25wZKbQq9qyAXrwIRExv7w0Ea6MgZLZiDM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240808171019-573a1156607a h1:EKiZZXueP9/T68B8Nl0GAx9cjbQnCId0yP3qPMgaaHs= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240808171019-573a1156607a/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= diff --git a/internal/common/connection.go b/internal/common/connection.go index 5b7a0953..58293e22 100644 --- a/internal/common/connection.go +++ b/internal/common/connection.go @@ -787,10 +787,12 @@ func (c *BaseConnection) Copy(virtualSourcePath, virtualTargetPath string) error // Rename renames (moves) virtualSourcePath to virtualTargetPath func (c *BaseConnection) Rename(virtualSourcePath, virtualTargetPath string) error { - return c.renameInternal(virtualSourcePath, virtualTargetPath, false) + return c.renameInternal(virtualSourcePath, virtualTargetPath, false, vfs.CheckParentDir) } -func (c *BaseConnection) renameInternal(virtualSourcePath, virtualTargetPath string, checkParentDestination bool) error { +func (c *BaseConnection) renameInternal(virtualSourcePath, virtualTargetPath string, + checkParentDestination bool, checks int, +) error { if virtualSourcePath == virtualTargetPath { return fmt.Errorf("the rename source and target cannot be the same: %w", c.GetOpUnsupportedError()) } @@ -844,7 +846,7 @@ func (c *BaseConnection) renameInternal(virtualSourcePath, virtualTargetPath str defer close(done) go keepConnectionAlive(c, done, 2*time.Minute) - files, size, err := fsDst.Rename(fsSourcePath, fsTargetPath) + files, size, err := fsDst.Rename(fsSourcePath, fsTargetPath, checks) if err != nil { c.Log(logger.LevelError, "failed to rename %q -> %q: %+v", fsSourcePath, fsTargetPath, err) return c.GetFsError(fsSrc, err) diff --git a/internal/common/eventmanager.go b/internal/common/eventmanager.go index b0ed9b7c..6b715d7c 100644 --- a/internal/common/eventmanager.go +++ b/internal/common/eventmanager.go @@ -1753,7 +1753,7 @@ func executeMkdirFsRuleAction(dirs []string, replacer *strings.Replacer, return nil } -func executeRenameFsActionForUser(renames []dataprovider.KeyValue, replacer *strings.Replacer, +func executeRenameFsActionForUser(renames []dataprovider.RenameConfig, replacer *strings.Replacer, user dataprovider.User, ) error { user, err := getUserForEventAction(user) @@ -1770,7 +1770,11 @@ func executeRenameFsActionForUser(renames []dataprovider.KeyValue, replacer *str for _, item := range renames { source := util.CleanPath(replaceWithReplacer(item.Key, replacer)) target := util.CleanPath(replaceWithReplacer(item.Value, replacer)) - if err = conn.renameInternal(source, target, true); err != nil { + checks := 0 + if item.UpdateModTime { + checks += vfs.CheckUpdateModTime + } + if err = conn.renameInternal(source, target, true, checks); err != nil { return fmt.Errorf("unable to rename %q->%q, user %q: %w", source, target, user.Username, err) } eventManagerLog(logger.LevelDebug, "rename %q->%q ok, user %q", source, target, user.Username) @@ -1832,7 +1836,7 @@ func executeExistFsActionForUser(exist []string, replacer *strings.Replacer, return nil } -func executeRenameFsRuleAction(renames []dataprovider.KeyValue, replacer *strings.Replacer, +func executeRenameFsRuleAction(renames []dataprovider.RenameConfig, replacer *strings.Replacer, conditions dataprovider.ConditionOptions, params *EventParams, ) error { users, err := params.getUsers() diff --git a/internal/common/eventmanager_test.go b/internal/common/eventmanager_test.go index c874a2b3..4f66d16c 100644 --- a/internal/common/eventmanager_test.go +++ b/internal/common/eventmanager_test.go @@ -1235,10 +1235,12 @@ func TestEventRuleActions(t *testing.T) { action.Options = dataprovider.BaseEventActionOptions{ FsConfig: dataprovider.EventActionFilesystemConfig{ Type: dataprovider.FilesystemActionRename, - Renames: []dataprovider.KeyValue{ + Renames: []dataprovider.RenameConfig{ { - Key: "/source", - Value: "/target", + KeyValue: dataprovider.KeyValue{ + Key: "/source", + Value: "/target", + }, }, }, }, @@ -1778,10 +1780,12 @@ func TestFilesystemActionErrors(t *testing.T) { assert.NoError(t, err) err = dataprovider.AddUser(&user, "", "", "") assert.NoError(t, err) - err = executeRenameFsActionForUser([]dataprovider.KeyValue{ + err = executeRenameFsActionForUser([]dataprovider.RenameConfig{ { - Key: "/p1", - Value: "/p1", + KeyValue: dataprovider.KeyValue{ + Key: "/p1", + Value: "/p1", + }, }, }, testReplacer, user) if assert.Error(t, err) { @@ -1792,10 +1796,12 @@ func TestFilesystemActionErrors(t *testing.T) { Options: dataprovider.BaseEventActionOptions{ FsConfig: dataprovider.EventActionFilesystemConfig{ Type: dataprovider.FilesystemActionRename, - Renames: []dataprovider.KeyValue{ + Renames: []dataprovider.RenameConfig{ { - Key: "/p2", - Value: "/p2", + KeyValue: dataprovider.KeyValue{ + Key: "/p2", + Value: "/p2", + }, }, }, }, diff --git a/internal/common/protocol_test.go b/internal/common/protocol_test.go index 350ae4b3..73ff93d8 100644 --- a/internal/common/protocol_test.go +++ b/internal/common/protocol_test.go @@ -4301,10 +4301,12 @@ func TestEventRuleFsActions(t *testing.T) { Options: dataprovider.BaseEventActionOptions{ FsConfig: dataprovider.EventActionFilesystemConfig{ Type: dataprovider.FilesystemActionRename, - Renames: []dataprovider.KeyValue{ + Renames: []dataprovider.RenameConfig{ { - Key: "/{{VirtualDirPath}}/{{ObjectName}}", - Value: "/{{ObjectName}}_renamed", + KeyValue: dataprovider.KeyValue{ + Key: "/{{VirtualDirPath}}/{{ObjectName}}", + Value: "/{{ObjectName}}_renamed", + }, }, }, }, @@ -4711,10 +4713,13 @@ func TestEventRulePreDelete(t *testing.T) { Options: dataprovider.BaseEventActionOptions{ FsConfig: dataprovider.EventActionFilesystemConfig{ Type: dataprovider.FilesystemActionRename, - Renames: []dataprovider.KeyValue{ + Renames: []dataprovider.RenameConfig{ { - Key: "/{{VirtualPath}}", - Value: fmt.Sprintf("/%s/{{VirtualPath}}", movePath), + KeyValue: dataprovider.KeyValue{ + Key: "/{{VirtualPath}}", + Value: fmt.Sprintf("/%s/{{VirtualPath}}", movePath), + }, + UpdateModTime: true, }, }, }, @@ -4768,59 +4773,83 @@ func TestEventRulePreDelete(t *testing.T) { QuotaFiles: 1000, }, } - user, _, err := httpdtest.AddUser(u, http.StatusCreated) + localUser, _, err := httpdtest.AddUser(u, http.StatusCreated) + assert.NoError(t, err) + u = getTestSFTPUser() + u.QuotaFiles = 1000 + sftpUser, _, err := httpdtest.AddUser(u, http.StatusCreated) assert.NoError(t, err) - conn, client, err := getSftpClient(user) - if assert.NoError(t, err) { - defer conn.Close() - defer client.Close() - testDir := "sub dir" - err = client.MkdirAll(testDir) - assert.NoError(t, err) - err = writeSFTPFile(testFileName, 100, client) - assert.NoError(t, err) - err = writeSFTPFile(path.Join(testDir, testFileName), 100, client) - assert.NoError(t, err) - err = client.Remove(testFileName) - assert.NoError(t, err) - err = client.Remove(path.Join(testDir, testFileName)) - assert.NoError(t, err) - // check files - _, err = client.Stat(testFileName) - assert.ErrorIs(t, err, os.ErrNotExist) - _, err = client.Stat(path.Join(testDir, testFileName)) - assert.ErrorIs(t, err, os.ErrNotExist) - _, err = client.Stat(path.Join("/", movePath, testFileName)) - assert.NoError(t, err) - _, err = client.Stat(path.Join("/", movePath, testDir, testFileName)) - assert.NoError(t, err) - // check quota - user, _, err = httpdtest.GetUserByUsername(user.Username, http.StatusOK) - assert.NoError(t, err) - assert.Equal(t, 0, user.UsedQuotaFiles) - assert.Equal(t, int64(0), user.UsedQuotaSize) - folder, _, err := httpdtest.GetFolderByName(movePath, http.StatusOK) - assert.NoError(t, err) - assert.Equal(t, 2, folder.UsedQuotaFiles) - assert.Equal(t, int64(200), folder.UsedQuotaSize) - // pre-delete action is not executed in movePath - err = client.Remove(path.Join("/", movePath, testFileName)) - assert.NoError(t, err) - // check quota - folder, _, err = httpdtest.GetFolderByName(movePath, http.StatusOK) - assert.NoError(t, err) - assert.Equal(t, 1, folder.UsedQuotaFiles) - assert.Equal(t, int64(100), folder.UsedQuotaSize) + for _, user := range []dataprovider.User{localUser, sftpUser} { + conn, client, err := getSftpClient(user) + if assert.NoError(t, err) { + defer conn.Close() + defer client.Close() + + testDir := "sub dir" + err = client.MkdirAll(testDir) + assert.NoError(t, err) + err = writeSFTPFile(testFileName, 100, client) + assert.NoError(t, err) + err = writeSFTPFile(path.Join(testDir, testFileName), 100, client) + assert.NoError(t, err) + modTime := time.Now().Add(-36 * time.Hour) + err = client.Chtimes(testFileName, modTime, modTime) + assert.NoError(t, err) + err = client.Remove(testFileName) + assert.NoError(t, err) + err = client.Remove(path.Join(testDir, testFileName)) + assert.NoError(t, err) + // check files + _, err = client.Stat(testFileName) + assert.ErrorIs(t, err, os.ErrNotExist) + _, err = client.Stat(path.Join(testDir, testFileName)) + assert.ErrorIs(t, err, os.ErrNotExist) + info, err := client.Stat(path.Join("/", movePath, testFileName)) + assert.NoError(t, err) + diff := math.Abs(time.Until(info.ModTime()).Seconds()) + assert.LessOrEqual(t, diff, float64(2)) + + _, err = client.Stat(path.Join("/", movePath, testDir, testFileName)) + assert.NoError(t, err) + // check quota + user, _, err = httpdtest.GetUserByUsername(user.Username, http.StatusOK) + assert.NoError(t, err) + if user.Username == localUser.Username { + assert.Equal(t, 0, user.UsedQuotaFiles) + assert.Equal(t, int64(0), user.UsedQuotaSize) + folder, _, err := httpdtest.GetFolderByName(movePath, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, 2, folder.UsedQuotaFiles) + assert.Equal(t, int64(200), folder.UsedQuotaSize) + } else { + assert.Equal(t, 1, user.UsedQuotaFiles) + assert.Equal(t, int64(100), user.UsedQuotaSize) + } + // pre-delete action is not executed in movePath + err = client.Remove(path.Join("/", movePath, testFileName)) + assert.NoError(t, err) + if user.Username == localUser.Username { + // check quota + folder, _, err := httpdtest.GetFolderByName(movePath, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, 1, folder.UsedQuotaFiles) + assert.Equal(t, int64(100), folder.UsedQuotaSize) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) + } + } } _, err = httpdtest.RemoveEventRule(rule1, http.StatusOK) assert.NoError(t, err) _, err = httpdtest.RemoveEventAction(action1, http.StatusOK) assert.NoError(t, err) - _, err = httpdtest.RemoveUser(user, http.StatusOK) + _, err = httpdtest.RemoveUser(sftpUser, http.StatusOK) assert.NoError(t, err) - err = os.RemoveAll(user.GetHomeDir()) + _, err = httpdtest.RemoveUser(localUser, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(localUser.GetHomeDir()) assert.NoError(t, err) _, err = httpdtest.RemoveFolder(vfs.BaseVirtualFolder{Name: movePath}, http.StatusOK) assert.NoError(t, err) @@ -4848,10 +4877,12 @@ func TestEventRulePreDownloadUpload(t *testing.T) { Options: dataprovider.BaseEventActionOptions{ FsConfig: dataprovider.EventActionFilesystemConfig{ Type: dataprovider.FilesystemActionRename, - Renames: []dataprovider.KeyValue{ + Renames: []dataprovider.RenameConfig{ { - Key: "/missing source", - Value: "/missing target", + KeyValue: dataprovider.KeyValue{ + Key: "/missing source", + Value: "/missing target", + }, }, }, }, diff --git a/internal/common/transfer.go b/internal/common/transfer.go index ae1ed432..2ff61b29 100644 --- a/internal/common/transfer.go +++ b/internal/common/transfer.go @@ -394,7 +394,7 @@ func (t *BaseTransfer) Close() error { t.effectiveFsPath, err) } else if t.isAtomicUpload() { if t.ErrTransfer == nil || Config.UploadMode&UploadModeAtomicWithResume != 0 { - _, _, err = t.Fs.Rename(t.effectiveFsPath, t.fsPath) + _, _, err = t.Fs.Rename(t.effectiveFsPath, t.fsPath, 0) t.Connection.Log(logger.LevelDebug, "atomic upload completed, rename: %q -> %q, error: %v", t.effectiveFsPath, t.fsPath, err) // the file must be removed if it is uploaded to a path outside the home dir and cannot be renamed diff --git a/internal/dataprovider/eventrule.go b/internal/dataprovider/eventrule.go index ad705c34..52d14262 100644 --- a/internal/dataprovider/eventrule.go +++ b/internal/dataprovider/eventrule.go @@ -660,12 +660,21 @@ func (c *EventActionFsCompress) validate() error { return nil } +// RenameConfig defines the configuration for a filesystem rename +type RenameConfig struct { + // key is the source and target the value + KeyValue + // This setting only applies to storage providers that support + // changing modification times. + UpdateModTime bool `json:"update_modtime,omitempty"` +} + // EventActionFilesystemConfig defines the configuration for filesystem actions type EventActionFilesystemConfig struct { // Filesystem actions, see the above enum Type int `json:"type,omitempty"` - // files/dirs to rename, key is the source and target the value - Renames []KeyValue `json:"renames,omitempty"` + // files/dirs to rename + Renames []RenameConfig `json:"renames,omitempty"` // directories to create MkDirs []string `json:"mkdirs,omitempty"` // files/dirs to delete @@ -706,9 +715,9 @@ func (c *EventActionFilesystemConfig) validateRenames() error { if len(c.Renames) == 0 { return util.NewI18nError(util.NewValidationError("no path to rename specified"), util.I18nErrorPathRequired) } - for idx, kv := range c.Renames { - key := strings.TrimSpace(kv.Key) - value := strings.TrimSpace(kv.Value) + for idx, cfg := range c.Renames { + key := strings.TrimSpace(cfg.Key) + value := strings.TrimSpace(cfg.Value) if key == "" || value == "" { return util.NewValidationError("invalid paths to rename") } @@ -726,9 +735,12 @@ func (c *EventActionFilesystemConfig) validateRenames() error { util.I18nErrorRootNotAllowed, ) } - c.Renames[idx] = KeyValue{ - Key: key, - Value: value, + c.Renames[idx] = RenameConfig{ + KeyValue: KeyValue{ + Key: key, + Value: value, + }, + UpdateModTime: cfg.UpdateModTime, } } return nil @@ -892,7 +904,7 @@ func (c *EventActionFilesystemConfig) getACopy() EventActionFilesystemConfig { return EventActionFilesystemConfig{ Type: c.Type, - Renames: cloneKeyValues(c.Renames), + Renames: cloneRenameConfigs(c.Renames), MkDirs: mkdirs, Deletes: deletes, Exist: exist, @@ -1833,6 +1845,20 @@ func (r *EventRule) RenderAsJSON(reload bool) ([]byte, error) { return json.Marshal(r) } +func cloneRenameConfigs(renames []RenameConfig) []RenameConfig { + res := make([]RenameConfig, 0, len(renames)) + for _, c := range renames { + res = append(res, RenameConfig{ + KeyValue: KeyValue{ + Key: c.Key, + Value: c.Value, + }, + UpdateModTime: c.UpdateModTime, + }) + } + return res +} + func cloneKeyValues(keyVals []KeyValue) []KeyValue { res := make([]KeyValue, 0, len(keyVals)) for _, kv := range keyVals { diff --git a/internal/ftpd/handler.go b/internal/ftpd/handler.go index 5a997f91..8ed36a6b 100644 --- a/internal/ftpd/handler.go +++ b/internal/ftpd/handler.go @@ -465,7 +465,7 @@ func (c *Connection) handleFTPUploadToExistingFile(fs vfs.Fs, flags int, resolve } if common.Config.IsAtomicUploadEnabled() && fs.IsAtomicUploadSupported() { - _, _, err = fs.Rename(resolvedPath, filePath) + _, _, err = fs.Rename(resolvedPath, filePath, 0) if err != nil { c.Log(logger.LevelError, "error renaming existing file for atomic upload, source: %q, dest: %q, err: %+v", resolvedPath, filePath, err) diff --git a/internal/ftpd/internal_test.go b/internal/ftpd/internal_test.go index 6d9dd8f0..ee877e7d 100644 --- a/internal/ftpd/internal_test.go +++ b/internal/ftpd/internal_test.go @@ -404,7 +404,7 @@ func (fs MockOsFs) Remove(name string, _ bool) error { } // Rename renames (moves) source to target -func (fs MockOsFs) Rename(source, target string) (int, int64, error) { +func (fs MockOsFs) Rename(source, target string, _ int) (int, int64, error) { if fs.err != nil { return -1, -1, fs.err } diff --git a/internal/httpd/handler.go b/internal/httpd/handler.go index 4fb70fde..bc936a8e 100644 --- a/internal/httpd/handler.go +++ b/internal/httpd/handler.go @@ -176,7 +176,7 @@ func (c *Connection) getFileWriter(name string) (io.WriteCloser, error) { } if common.Config.IsAtomicUploadEnabled() && fs.IsAtomicUploadSupported() { - _, _, err = fs.Rename(p, filePath) + _, _, err = fs.Rename(p, filePath, 0) if err != nil { c.Log(logger.LevelError, "error renaming existing file for atomic upload, source: %q, dest: %q, err: %+v", p, filePath, err) diff --git a/internal/httpd/httpd_test.go b/internal/httpd/httpd_test.go index 83f3242e..704b7cd3 100644 --- a/internal/httpd/httpd_test.go +++ b/internal/httpd/httpd_test.go @@ -2458,28 +2458,34 @@ func TestEventActionValidation(t *testing.T) { _, resp, err = httpdtest.AddEventAction(action, http.StatusBadRequest) assert.NoError(t, err) assert.Contains(t, string(resp), "no path to rename specified") - action.Options.FsConfig.Renames = []dataprovider.KeyValue{ + action.Options.FsConfig.Renames = []dataprovider.RenameConfig{ { - Key: "", - Value: "/adir", + KeyValue: dataprovider.KeyValue{ + Key: "", + Value: "/adir", + }, }, } _, resp, err = httpdtest.AddEventAction(action, http.StatusBadRequest) assert.NoError(t, err) assert.Contains(t, string(resp), "invalid paths to rename") - action.Options.FsConfig.Renames = []dataprovider.KeyValue{ + action.Options.FsConfig.Renames = []dataprovider.RenameConfig{ { - Key: "adir", - Value: "/adir", + KeyValue: dataprovider.KeyValue{ + Key: "adir", + Value: "/adir", + }, }, } _, resp, err = httpdtest.AddEventAction(action, http.StatusBadRequest) assert.NoError(t, err) assert.Contains(t, string(resp), "rename source and target cannot be equal") - action.Options.FsConfig.Renames = []dataprovider.KeyValue{ + action.Options.FsConfig.Renames = []dataprovider.RenameConfig{ { - Key: "/", - Value: "/dir", + KeyValue: dataprovider.KeyValue{ + Key: "/", + Value: "/dir", + }, }, } _, resp, err = httpdtest.AddEventAction(action, http.StatusBadRequest) @@ -23974,16 +23980,19 @@ func TestWebEventAction(t *testing.T) { action.Options.FsConfig = dataprovider.EventActionFilesystemConfig{ Type: dataprovider.FilesystemActionRename, - Renames: []dataprovider.KeyValue{ + Renames: []dataprovider.RenameConfig{ { - Key: "/src", - Value: "/target", + KeyValue: dataprovider.KeyValue{ + Key: "/src", + Value: "/target", + }, }, }, } form.Set("fs_action_type", fmt.Sprintf("%d", action.Options.FsConfig.Type)) form.Set("fs_rename[0][fs_rename_source]", action.Options.FsConfig.Renames[0].Key) form.Set("fs_rename[0][fs_rename_target]", action.Options.FsConfig.Renames[0].Value) + form.Set("fs_rename[0][fs_rename_options][]", "1") req, err = http.NewRequest(http.MethodPost, path.Join(webAdminEventActionPath, action.Name), bytes.NewBuffer([]byte(form.Encode()))) assert.NoError(t, err) @@ -23995,7 +24004,9 @@ func TestWebEventAction(t *testing.T) { actionGet, _, err = httpdtest.GetEventActionByName(action.Name, http.StatusOK) assert.NoError(t, err) assert.Equal(t, action.Type, actionGet.Type) - assert.Len(t, actionGet.Options.FsConfig.Renames, 1) + if assert.Len(t, actionGet.Options.FsConfig.Renames, 1) { + assert.True(t, actionGet.Options.FsConfig.Renames[0].UpdateModTime) + } action.Options.FsConfig = dataprovider.EventActionFilesystemConfig{ Type: dataprovider.FilesystemActionCopy, diff --git a/internal/httpd/webadmin.go b/internal/httpd/webadmin.go index 7b7933ca..878dafe1 100644 --- a/internal/httpd/webadmin.go +++ b/internal/httpd/webadmin.go @@ -2201,6 +2201,28 @@ func getKeyValsFromPostFields(r *http.Request, key, val string) []dataprovider.K return res } +func getRenameConfigsFromPostFields(r *http.Request) []dataprovider.RenameConfig { + var res []dataprovider.RenameConfig + keys := r.Form["fs_rename_source"] + values := r.Form["fs_rename_target"] + + for idx, k := range keys { + v := values[idx] + if k != "" && v != "" { + opts := r.Form["fs_rename_options"+strconv.Itoa(idx)] + res = append(res, dataprovider.RenameConfig{ + KeyValue: dataprovider.KeyValue{ + Key: k, + Value: v, + }, + UpdateModTime: slices.Contains(opts, "1"), + }) + } + } + + return res +} + func getFoldersRetentionFromPostFields(r *http.Request) ([]dataprovider.FolderRetention, error) { var res []dataprovider.FolderRetention paths := r.Form["folder_retention_path"] @@ -2310,6 +2332,8 @@ func updateRepeaterFormActionFields(r *http.Request) { base, _ := strings.CutSuffix(k, "[fs_rename_source]") r.Form.Add("fs_rename_source", strings.TrimSpace(r.Form.Get(k))) r.Form.Add("fs_rename_target", strings.TrimSpace(r.Form.Get(base+"[fs_rename_target]"))) + r.Form["fs_rename_options"+strconv.Itoa(len(r.Form["fs_rename_source"])-1)] = + r.Form[base+"[fs_rename_options][]"] continue } if hasPrefixAndSuffix(k, "fs_copy[", "][fs_copy_source]") { @@ -2398,7 +2422,7 @@ func getEventActionOptionsFromPostFields(r *http.Request) (dataprovider.BaseEven }, FsConfig: dataprovider.EventActionFilesystemConfig{ Type: fsActionType, - Renames: getKeyValsFromPostFields(r, "fs_rename_source", "fs_rename_target"), + Renames: getRenameConfigsFromPostFields(r), Deletes: getSliceFromDelimitedValues(r.Form.Get("fs_delete_paths"), ","), MkDirs: getSliceFromDelimitedValues(r.Form.Get("fs_mkdir_paths"), ","), Exist: getSliceFromDelimitedValues(r.Form.Get("fs_exist_paths"), ","), diff --git a/internal/httpdtest/httpdtest.go b/internal/httpdtest/httpdtest.go index c78c97b2..af53577b 100644 --- a/internal/httpdtest/httpdtest.go +++ b/internal/httpdtest/httpdtest.go @@ -2601,9 +2601,28 @@ func compareUserFilePatternsFilters(expected sdk.BaseUserFilters, actual sdk.Bas return nil } +func compareRenameConfigs(expected, actual []dataprovider.RenameConfig) error { + if len(expected) != len(actual) { + return errors.New("rename configs mismatch") + } + for _, ex := range expected { + found := false + for _, ac := range actual { + if ac.Key == ex.Key && ac.Value == ex.Value && ac.UpdateModTime == ex.UpdateModTime { + found = true + break + } + } + if !found { + return errors.New("rename configs mismatch") + } + } + return nil +} + func compareKeyValues(expected, actual []dataprovider.KeyValue) error { if len(expected) != len(actual) { - return errors.New("kay values mismatch") + return errors.New("key values mismatch") } for _, ex := range expected { found := false @@ -2614,7 +2633,7 @@ func compareKeyValues(expected, actual []dataprovider.KeyValue) error { } } if !found { - return errors.New("kay values mismatch") + return errors.New("key values mismatch") } } return nil @@ -2731,7 +2750,7 @@ func compareEventActionFsConfigFields(expected, actual dataprovider.EventActionF if expected.Type != actual.Type { return errors.New("fs type mismatch") } - if err := compareKeyValues(expected.Renames, actual.Renames); err != nil { + if err := compareRenameConfigs(expected.Renames, actual.Renames); err != nil { return errors.New("fs renames mismatch") } if err := compareKeyValues(expected.Copy, actual.Copy); err != nil { diff --git a/internal/sftpd/handler.go b/internal/sftpd/handler.go index 742663cf..d404ec0f 100644 --- a/internal/sftpd/handler.go +++ b/internal/sftpd/handler.go @@ -457,7 +457,7 @@ func (c *Connection) handleSFTPUploadToExistingFile(fs vfs.Fs, pflags sftp.FileO } if common.Config.IsAtomicUploadEnabled() && fs.IsAtomicUploadSupported() { - _, _, err = fs.Rename(resolvedPath, filePath) + _, _, err = fs.Rename(resolvedPath, filePath, 0) if err != nil { c.Log(logger.LevelError, "error renaming existing file for atomic upload, source: %q, dest: %q, err: %+v", resolvedPath, filePath, err) diff --git a/internal/sftpd/internal_test.go b/internal/sftpd/internal_test.go index 3ddfe400..890f62ba 100644 --- a/internal/sftpd/internal_test.go +++ b/internal/sftpd/internal_test.go @@ -146,7 +146,7 @@ func (fs MockOsFs) Remove(name string, _ bool) error { } // Rename renames (moves) source to target -func (fs MockOsFs) Rename(source, target string) (int, int64, error) { +func (fs MockOsFs) Rename(source, target string, _ int) (int, int64, error) { if fs.err != nil { return -1, -1, fs.err } diff --git a/internal/sftpd/scp.go b/internal/sftpd/scp.go index 6b97af08..e62332c7 100644 --- a/internal/sftpd/scp.go +++ b/internal/sftpd/scp.go @@ -330,7 +330,7 @@ func (c *scpCommand) handleUpload(uploadFilePath string, sizeToRead int64) error } if common.Config.IsAtomicUploadEnabled() && fs.IsAtomicUploadSupported() { - _, _, err = fs.Rename(p, filePath) + _, _, err = fs.Rename(p, filePath, 0) if err != nil { c.connection.Log(logger.LevelError, "error renaming existing file for atomic upload, source: %q, dest: %q, err: %v", p, filePath, err) diff --git a/internal/vfs/azblobfs.go b/internal/vfs/azblobfs.go index 11129167..1c69680a 100644 --- a/internal/vfs/azblobfs.go +++ b/internal/vfs/azblobfs.go @@ -306,19 +306,21 @@ func (fs *AzureBlobFs) Create(name string, flag, checks int) (File, PipeWriter, } // Rename renames (moves) source to target. -func (fs *AzureBlobFs) Rename(source, target string) (int, int64, error) { +func (fs *AzureBlobFs) Rename(source, target string, checks int) (int, int64, error) { if source == target { return -1, -1, nil } - _, err := fs.Stat(path.Dir(target)) - if err != nil { - return -1, -1, err + if checks&CheckParentDir != 0 { + _, err := fs.Stat(path.Dir(target)) + if err != nil { + return -1, -1, err + } } fi, err := fs.Stat(source) if err != nil { return -1, -1, err } - return fs.renameInternal(source, target, fi, 0) + return fs.renameInternal(source, target, fi, 0, checks&CheckUpdateModTime != 0) } // Remove removes the named file or (empty) directory. @@ -398,7 +400,17 @@ func (fs *AzureBlobFs) Chtimes(name string, _, mtime time.Time, isUploading bool if metadata == nil { metadata = make(map[string]*string) } - metadata[lastModifiedField] = to.Ptr(strconv.FormatInt(mtime.UnixMilli(), 10)) + found := false + for k := range metadata { + if strings.ToLower(k) == lastModifiedField { + metadata[k] = to.Ptr(strconv.FormatInt(mtime.UnixMilli(), 10)) + found = true + break + } + } + if !found { + metadata[lastModifiedField] = to.Ptr(strconv.FormatInt(mtime.UnixMilli(), 10)) + } ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) defer cancelFn() @@ -667,7 +679,7 @@ func (fs *AzureBlobFs) CopyFile(source, target string, srcInfo os.FileInfo) (int return 0, 0, err } } - if err := fs.copyFileInternal(source, target, srcInfo); err != nil { + if err := fs.copyFileInternal(source, target, srcInfo, true); err != nil { return 0, 0, err } return numFiles, sizeDiff, nil @@ -750,13 +762,13 @@ func (fs *AzureBlobFs) setConfigDefaults() { } } -func (fs *AzureBlobFs) copyFileInternal(source, target string, srcInfo os.FileInfo) error { +func (fs *AzureBlobFs) copyFileInternal(source, target string, srcInfo os.FileInfo, updateModTime bool) error { ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout)) defer cancelFn() srcBlob := fs.containerClient.NewBlockBlobClient(source) dstBlob := fs.containerClient.NewBlockBlobClient(target) - resp, err := dstBlob.StartCopyFromURL(ctx, srcBlob.URL(), fs.getCopyOptions(srcInfo)) + resp, err := dstBlob.StartCopyFromURL(ctx, srcBlob.URL(), fs.getCopyOptions(srcInfo, updateModTime)) if err != nil { metric.AZCopyObjectCompleted(err) return err @@ -789,7 +801,9 @@ func (fs *AzureBlobFs) copyFileInternal(source, target string, srcInfo os.FileIn return nil } -func (fs *AzureBlobFs) renameInternal(source, target string, srcInfo os.FileInfo, recursion int) (int, int64, error) { +func (fs *AzureBlobFs) renameInternal(source, target string, srcInfo os.FileInfo, recursion int, + updateModTime bool, +) (int, int64, error) { var numFiles int var filesSize int64 @@ -807,7 +821,7 @@ func (fs *AzureBlobFs) renameInternal(source, target string, srcInfo os.FileInfo return numFiles, filesSize, err } if renameMode == 1 { - files, size, err := doRecursiveRename(fs, source, target, fs.renameInternal, recursion) + files, size, err := doRecursiveRename(fs, source, target, fs.renameInternal, recursion, updateModTime) numFiles += files filesSize += size if err != nil { @@ -815,7 +829,7 @@ func (fs *AzureBlobFs) renameInternal(source, target string, srcInfo os.FileInfo } } } else { - if err := fs.copyFileInternal(source, target, srcInfo); err != nil { + if err := fs.copyFileInternal(source, target, srcInfo, updateModTime); err != nil { return numFiles, filesSize, err } numFiles++ @@ -1102,20 +1116,27 @@ func (*AzureBlobFs) readFill(r io.Reader, buf []byte) (n int, err error) { return n, err } -func (fs *AzureBlobFs) getCopyOptions(srcInfo os.FileInfo) *blob.StartCopyFromURLOptions { +func (fs *AzureBlobFs) getCopyOptions(srcInfo os.FileInfo, updateModTime bool) *blob.StartCopyFromURLOptions { copyOptions := &blob.StartCopyFromURLOptions{} if fs.config.AccessTier != "" { copyOptions.Tier = (*blob.AccessTier)(&fs.config.AccessTier) } - metadata := make(map[string]*string) - for k, v := range getMetadata(srcInfo) { - if v != "" { - metadata[k] = to.Ptr(v) + if updateModTime { + metadata := make(map[string]*string) + for k, v := range getMetadata(srcInfo) { + if v != "" { + if strings.ToLower(k) == lastModifiedField { + metadata[k] = to.Ptr("0") + } else { + metadata[k] = to.Ptr(v) + } + } + } + if len(metadata) > 0 { + copyOptions.Metadata = metadata } } - if len(metadata) > 0 { - copyOptions.Metadata = metadata - } + return copyOptions } diff --git a/internal/vfs/gcsfs.go b/internal/vfs/gcsfs.go index 06b55323..a169d750 100644 --- a/internal/vfs/gcsfs.go +++ b/internal/vfs/gcsfs.go @@ -255,19 +255,21 @@ func (fs *GCSFs) Create(name string, flag, checks int) (File, PipeWriter, func() } // Rename renames (moves) source to target. -func (fs *GCSFs) Rename(source, target string) (int, int64, error) { +func (fs *GCSFs) Rename(source, target string, checks int) (int, int64, error) { if source == target { return -1, -1, nil } - _, err := fs.Stat(path.Dir(target)) - if err != nil { - return -1, -1, err + if checks&CheckParentDir != 0 { + _, err := fs.Stat(path.Dir(target)) + if err != nil { + return -1, -1, err + } } fi, err := fs.getObjectStat(source) if err != nil { return -1, -1, err } - return fs.renameInternal(source, target, fi, 0) + return fs.renameInternal(source, target, fi, 0, checks&CheckUpdateModTime != 0) } // Remove removes the named file or (empty) directory. @@ -651,7 +653,7 @@ func (fs *GCSFs) CopyFile(source, target string, srcInfo os.FileInfo) (int, int6 } conditions = &storage.Conditions{DoesNotExist: true} } - if err := fs.copyFileInternal(source, target, conditions, srcInfo); err != nil { + if err := fs.copyFileInternal(source, target, conditions, srcInfo, true); err != nil { return 0, 0, err } return numFiles, sizeDiff, nil @@ -753,7 +755,9 @@ func (fs *GCSFs) composeObjects(ctx context.Context, dst, partialObject *storage return err } -func (fs *GCSFs) copyFileInternal(source, target string, conditions *storage.Conditions, srcInfo os.FileInfo) error { +func (fs *GCSFs) copyFileInternal(source, target string, conditions *storage.Conditions, + srcInfo os.FileInfo, updateModTime bool, +) error { src := fs.svc.Bucket(fs.config.Bucket).Object(source) dst := fs.svc.Bucket(fs.config.Bucket).Object(target) if conditions != nil { @@ -785,6 +789,9 @@ func (fs *GCSFs) copyFileInternal(source, target string, conditions *storage.Con copier.ContentType = contentType } metadata := getMetadata(srcInfo) + if updateModTime && len(metadata) > 0 { + delete(metadata, lastModifiedField) + } if len(metadata) > 0 { copier.Metadata = metadata } @@ -793,7 +800,9 @@ func (fs *GCSFs) copyFileInternal(source, target string, conditions *storage.Con return err } -func (fs *GCSFs) renameInternal(source, target string, srcInfo os.FileInfo, recursion int) (int, int64, error) { +func (fs *GCSFs) renameInternal(source, target string, srcInfo os.FileInfo, recursion int, + updateModTime bool, +) (int, int64, error) { var numFiles int var filesSize int64 @@ -811,7 +820,7 @@ func (fs *GCSFs) renameInternal(source, target string, srcInfo os.FileInfo, recu return numFiles, filesSize, err } if renameMode == 1 { - files, size, err := doRecursiveRename(fs, source, target, fs.renameInternal, recursion) + files, size, err := doRecursiveRename(fs, source, target, fs.renameInternal, recursion, updateModTime) numFiles += files filesSize += size if err != nil { @@ -819,7 +828,7 @@ func (fs *GCSFs) renameInternal(source, target string, srcInfo os.FileInfo, recu } } } else { - if err := fs.copyFileInternal(source, target, nil, srcInfo); err != nil { + if err := fs.copyFileInternal(source, target, nil, srcInfo, updateModTime); err != nil { return numFiles, filesSize, err } numFiles++ diff --git a/internal/vfs/httpfs.go b/internal/vfs/httpfs.go index 331f72dc..b6ddb6fa 100644 --- a/internal/vfs/httpfs.go +++ b/internal/vfs/httpfs.go @@ -384,7 +384,7 @@ func (fs *HTTPFs) Create(name string, flag, checks int) (File, PipeWriter, func( } // Rename renames (moves) source to target. -func (fs *HTTPFs) Rename(source, target string) (int, int64, error) { +func (fs *HTTPFs) Rename(source, target string, checks int) (int, int64, error) { if source == target { return -1, -1, nil } @@ -397,6 +397,9 @@ func (fs *HTTPFs) Rename(source, target string) (int, int64, error) { return -1, -1, err } defer resp.Body.Close() + if checks&CheckUpdateModTime != 0 { + fs.Chtimes(target, time.Now(), time.Now(), false) //nolint:errcheck + } return -1, -1, nil } diff --git a/internal/vfs/osfs.go b/internal/vfs/osfs.go index 73c552a9..f885cb5b 100644 --- a/internal/vfs/osfs.go +++ b/internal/vfs/osfs.go @@ -176,7 +176,7 @@ func (fs *OsFs) Create(name string, flag, _ int) (File, PipeWriter, func(), erro } // Rename renames (moves) source to target -func (fs *OsFs) Rename(source, target string) (int, int64, error) { +func (fs *OsFs) Rename(source, target string, checks int) (int, int64, error) { if source == target { return -1, -1, nil } @@ -199,9 +199,15 @@ func (fs *OsFs) Rename(source, target string) (int, int64, error) { fsLog(fs, logger.LevelError, "cross device copy error: %v", err) return -1, -1, err } + if checks&CheckUpdateModTime != 0 { + fs.Chtimes(target, time.Now(), time.Now(), false) //nolint:errcheck + } err = os.RemoveAll(source) return -1, -1, err } + if checks&CheckUpdateModTime != 0 && err == nil { + fs.Chtimes(target, time.Now(), time.Now(), false) //nolint:errcheck + } return -1, -1, err } diff --git a/internal/vfs/s3fs.go b/internal/vfs/s3fs.go index e6d9a609..ee56bc24 100644 --- a/internal/vfs/s3fs.go +++ b/internal/vfs/s3fs.go @@ -334,19 +334,21 @@ func (fs *S3Fs) Create(name string, flag, checks int) (File, PipeWriter, func(), } // Rename renames (moves) source to target. -func (fs *S3Fs) Rename(source, target string) (int, int64, error) { +func (fs *S3Fs) Rename(source, target string, checks int) (int, int64, error) { if source == target { return -1, -1, nil } - _, err := fs.Stat(path.Dir(target)) - if err != nil { - return -1, -1, err + if checks&CheckParentDir != 0 { + _, err := fs.Stat(path.Dir(target)) + if err != nil { + return -1, -1, err + } } fi, err := fs.Stat(source) if err != nil { return -1, -1, err } - return fs.renameInternal(source, target, fi, 0) + return fs.renameInternal(source, target, fi, 0, checks&CheckUpdateModTime != 0) } // Remove removes the named file or (empty) directory. @@ -700,21 +702,29 @@ func (fs *S3Fs) copyFileInternal(source, target string, srcInfo os.FileInfo) err ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) defer cancelFn() - _, err := fs.svc.CopyObject(ctx, &s3.CopyObjectInput{ + copyObject := &s3.CopyObjectInput{ Bucket: aws.String(fs.config.Bucket), CopySource: aws.String(copySource), Key: aws.String(target), StorageClass: types.StorageClass(fs.config.StorageClass), ACL: types.ObjectCannedACL(fs.config.ACL), ContentType: util.NilIfEmpty(contentType), - Metadata: getMetadata(srcInfo), - }) + } + + metadata := getMetadata(srcInfo) + if len(metadata) > 0 { + copyObject.Metadata = metadata + } + + _, err := fs.svc.CopyObject(ctx, copyObject) metric.S3CopyObjectCompleted(err) return err } -func (fs *S3Fs) renameInternal(source, target string, srcInfo os.FileInfo, recursion int) (int, int64, error) { +func (fs *S3Fs) renameInternal(source, target string, srcInfo os.FileInfo, recursion int, + updateModTime bool, +) (int, int64, error) { var numFiles int var filesSize int64 @@ -732,7 +742,7 @@ func (fs *S3Fs) renameInternal(source, target string, srcInfo os.FileInfo, recur return numFiles, filesSize, err } if renameMode == 1 { - files, size, err := doRecursiveRename(fs, source, target, fs.renameInternal, recursion) + files, size, err := doRecursiveRename(fs, source, target, fs.renameInternal, recursion, updateModTime) numFiles += files filesSize += size if err != nil { diff --git a/internal/vfs/sftpfs.go b/internal/vfs/sftpfs.go index 0a2f4214..670ea175 100644 --- a/internal/vfs/sftpfs.go +++ b/internal/vfs/sftpfs.go @@ -479,7 +479,7 @@ func (fs *SFTPFs) Create(name string, flag, _ int) (File, PipeWriter, func(), er } // Rename renames (moves) source to target. -func (fs *SFTPFs) Rename(source, target string) (int, int64, error) { +func (fs *SFTPFs) Rename(source, target string, checks int) (int, int64, error) { if source == target { return -1, -1, nil } @@ -489,9 +489,15 @@ func (fs *SFTPFs) Rename(source, target string) (int, int64, error) { } if _, ok := client.HasExtension("posix-rename@openssh.com"); ok { err := client.PosixRename(source, target) + if checks&CheckUpdateModTime != 0 && err == nil { + fs.Chtimes(target, time.Now(), time.Now(), false) //nolint:errcheck + } return -1, -1, err } err = client.Rename(source, target) + if checks&CheckUpdateModTime != 0 && err == nil { + fs.Chtimes(target, time.Now(), time.Now(), false) //nolint:errcheck + } return -1, -1, err } diff --git a/internal/vfs/vfs.go b/internal/vfs/vfs.go index 64ad4186..5778c24f 100644 --- a/internal/vfs/vfs.go +++ b/internal/vfs/vfs.go @@ -52,8 +52,9 @@ const ( // Additional checks for files const ( - CheckParentDir = 1 - CheckResume = 2 + CheckParentDir = 1 + CheckResume = 2 + CheckUpdateModTime = 4 ) var ( @@ -121,7 +122,7 @@ type Fs interface { Lstat(name string) (os.FileInfo, error) Open(name string, offset int64) (File, PipeReader, func(), error) Create(name string, flag, checks int) (File, PipeWriter, func(), error) - Rename(source, target string) (int, int64, error) + Rename(source, target string, checks int) (int, int64, error) Remove(name string, isDir bool) error Mkdir(name string) error Symlink(source, target string) error @@ -1084,7 +1085,7 @@ func IsUploadResumeSupported(fs Fs, size int64) bool { } func getLastModified(metadata map[string]string) int64 { - if val, ok := metadata[lastModifiedField]; ok { + if val, ok := metadata[lastModifiedField]; ok && val != "" { lastModified, err := strconv.ParseInt(val, 10, 64) if err == nil { return lastModified @@ -1167,8 +1168,8 @@ func getLocalTempDir() string { } func doRecursiveRename(fs Fs, source, target string, - renameFn func(string, string, os.FileInfo, int) (int, int64, error), - recursion int, + renameFn func(string, string, os.FileInfo, int, bool) (int, int64, error), + recursion int, updateModTime bool, ) (int, int64, error) { var numFiles int var filesSize int64 @@ -1193,7 +1194,7 @@ func doRecursiveRename(fs Fs, source, target string, for _, info := range entries { sourceEntry := fs.Join(source, info.Name()) targetEntry := fs.Join(target, info.Name()) - files, size, err := renameFn(sourceEntry, targetEntry, info, recursion) + files, size, err := renameFn(sourceEntry, targetEntry, info, recursion, updateModTime) if err != nil { if fs.IsNotExist(err) { fsLog(fs, logger.LevelInfo, "skipping rename for %q: %v", sourceEntry, err) diff --git a/internal/webdavd/handler.go b/internal/webdavd/handler.go index e31d1961..829c1173 100644 --- a/internal/webdavd/handler.go +++ b/internal/webdavd/handler.go @@ -254,7 +254,7 @@ func (c *Connection) handleUploadToExistingFile(fs vfs.Fs, resolvedPath, filePat maxWriteSize, _ := c.GetMaxWriteSize(diskQuota, false, fileSize, fs.IsUploadResumeSupported()) if common.Config.IsAtomicUploadEnabled() && fs.IsAtomicUploadSupported() { - _, _, err = fs.Rename(resolvedPath, filePath) + _, _, err = fs.Rename(resolvedPath, filePath, 0) if err != nil { c.Log(logger.LevelError, "error renaming existing file for atomic upload, source: %q, dest: %q, err: %+v", resolvedPath, filePath, err) diff --git a/internal/webdavd/internal_test.go b/internal/webdavd/internal_test.go index 0366c7d2..61e8a31b 100644 --- a/internal/webdavd/internal_test.go +++ b/internal/webdavd/internal_test.go @@ -312,7 +312,7 @@ func (fs *MockOsFs) Remove(name string, _ bool) error { } // Rename renames (moves) source to target -func (fs *MockOsFs) Rename(source, target string) (int, int64, error) { +func (fs *MockOsFs) Rename(source, target string, _ int) (int, int64, error) { err := os.Rename(source, target) return -1, -1, err } diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 413951f7..b66974fb 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -6973,6 +6973,14 @@ components: type: string value: type: string + RenameConfig: + allOf: + - $ref: '#/components/schemas/KeyValue' + - type: object + properties: + update_modtime: + type: boolean + description: 'Update modification time. This setting is not recursive and only applies to storage providers that support changing modification times' HTTPPart: type: object properties: @@ -7105,7 +7113,7 @@ components: renames: type: array items: - $ref: '#/components/schemas/KeyValue' + $ref: '#/components/schemas/RenameConfig' mkdirs: type: array items: diff --git a/static/locales/en/translation.json b/static/locales/en/translation.json index c9534be0..0d686bfc 100644 --- a/static/locales/en/translation.json +++ b/static/locales/en/translation.json @@ -1007,6 +1007,7 @@ "archive_path": "Archive path", "archive_path_help": "Full path, as seen by SFTPGo users, to the zip archive to create. Placeholders are supported. If the specified file already exists, it is overwritten", "placeholders_modal_title": "Supported placeholders", + "update_mod_times": "Update timestamp", "types": { "http": "HTTP", "email": "Email", diff --git a/static/locales/it/translation.json b/static/locales/it/translation.json index 5db589e7..201be86e 100644 --- a/static/locales/it/translation.json +++ b/static/locales/it/translation.json @@ -1007,6 +1007,7 @@ "archive_path": "Percorso dell'archivio", "archive_path_help": "Percorso completo, come visto dagli utenti SFTPGo, dell'archivio zip da creare. I segnaposto sono supportati. Se il file specificato esiste già, verrà sovrascritto", "placeholders_modal_title": "Segnaposto supportati", + "update_mod_times": "Aggiorna timestamp", "types": { "http": "HTTP", "email": "Email", diff --git a/templates/webadmin/eventaction.html b/templates/webadmin/eventaction.html index d7e6d098..a2b77bca 100644 --- a/templates/webadmin/eventaction.html +++ b/templates/webadmin/eventaction.html @@ -653,12 +653,18 @@ explicit grant from the SFTPGo Team (support@sftpgo.com).
-
+
-
+
+
+ +
@@ -677,12 +683,18 @@ explicit grant from the SFTPGo Team (support@sftpgo.com). {{- else}}