beenull/sftpgo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

conditional support for recursive renaming for cloud providers

Browse source 
Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
This commit is contained in:
Nicola Murino 2023-01-06 12:33:50 +01:00
parent f0dedbfabf
commit 8cad436421
No known key found for this signature in database
GPG key ID: 935D2952DEC4EECF
26 changed files with 650 additions and 473 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
docs/full-configuration.md
View file

@ -66,6 +66,7 @@ The configuration file contains the following sections:
- `execute_sync`, list of strings. Actions, defined in the `execute_on` list above, to be performed synchronously. The `pre-*` actions are always executed synchronously while the other ones are asynchronous. Executing an action synchronously means that SFTPGo will not return a result code to the client (which is waiting for it) until your hook have completed its execution. Leave empty to execute only the defined `pre-*` hook synchronously
- `hook`, string. Absolute path to the command to execute or HTTP URL to notify.
- `setstat_mode`, integer. 0 means "normal mode": requests for changing permissions, owner/group and access/modification times are executed. 1 means "ignore mode": requests for changing permissions, owner/group and access/modification times are silently ignored. 2 means "ignore mode if not supported": requests for changing permissions and owner/group are silently ignored for cloud filesystems and executed for local/SFTP filesystem. Requests for changing modification times are always executed for local/SFTP filesystems and are executed for cloud based filesystems if the target is a file and there is a metadata plugin available. A metadata plugin can be found [here](https://github.com/sftpgo/sftpgo-plugin-metadata).
- `rename_mode`, integer. By default (`0`), renaming of non-empty directories is not allowed for cloud storage providers (S3, GCS, Azure Blob). Set to `1` to enable recursive renames for these providers, they may be slow, there is no atomic rename API like for local filesystem, so SFTPGo will recursively list the directory contents and do a rename for each entry (partial renaming and incorrect disk quota updates are possible in error cases). Default `0`.
- `temp_path`, string. Defines the path for temporary files such as those used for atomic uploads or file pipes. If you set this option you must make sure that the defined path exists, is accessible for writing by the user running SFTPGo, and is on the same filesystem as the users home directories otherwise the renaming for atomic uploads will become a copy and therefore may take a long time. The temporary files are not namespaced. The default is generally fine. Leave empty for the default.
- `proxy_protocol`, integer. Support for [HAProxy PROXY protocol](https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt). If you are running SFTPGo behind a proxy server such as HAProxy, AWS ELB or NGINX, you can enable the proxy protocol. It provides a convenient way to safely transport connection information such as a client's address across multiple layers of NAT or TCP proxies to get the real client IP address instead of the proxy IP. Both protocol versions 1 and 2 are supported. If the proxy protocol is enabled in SFTPGo then you have to enable the protocol in your proxy configuration too. For example, for HAProxy, add `send-proxy` or `send-proxy-v2` to each server configuration line. The PROXY protocol is supported for SSH/SFTP and FTP/S. The following modes are supported:
- 0, disabled

56
go.mod
View file

@ -9,22 +9,22 @@ require (
github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962
github.com/alexedwards/argon2id v0.0.0-20211130144151-3585854a6387
github.com/aws/aws-sdk-go-v2 v1.17.3
github.com/aws/aws-sdk-go-v2/config v1.18.7
github.com/aws/aws-sdk-go-v2/credentials v1.13.7
github.com/aws/aws-sdk-go-v2/config v1.18.8
github.com/aws/aws-sdk-go-v2/credentials v1.13.8
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.46
github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.26
github.com/aws/aws-sdk-go-v2/service/s3 v1.29.6
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.17.0
github.com/aws/aws-sdk-go-v2/service/sts v1.17.7
github.com/bmatcuk/doublestar/v4 v4.4.0
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.47
github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.14.0
github.com/aws/aws-sdk-go-v2/service/s3 v1.30.0
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.18.0
github.com/aws/aws-sdk-go-v2/service/sts v1.18.0
github.com/bmatcuk/doublestar/v4 v4.6.0
github.com/cockroachdb/cockroach-go/v2 v2.2.20
github.com/coreos/go-oidc/v3 v3.4.0
github.com/coreos/go-oidc/v3 v3.5.0
github.com/drakkan/webdav v0.0.0-20221101181759-17ed21f9337b
github.com/eikenb/pipeat v0.0.0-20210730190139-06b3e6902001
github.com/fclairamb/ftpserverlib v0.20.1-0.20221012093027-95be4ae0c9a6
github.com/fclairamb/ftpserverlib v0.20.1-0.20230104020606-0b1a04eec221
github.com/fclairamb/go-log v0.4.1
github.com/go-acme/lego/v4 v4.9.1
github.com/go-acme/lego/v4 v4.9.2-0.20230104103215-fd54758bba4c
github.com/go-chi/chi/v5 v5.0.8
github.com/go-chi/jwtauth/v5 v5.1.0
github.com/go-chi/render v1.0.2
@ -45,7 +45,7 @@ require (
github.com/minio/sio v0.3.0
github.com/otiai10/copy v1.9.0
github.com/pires/go-proxyproto v0.6.2
github.com/pkg/sftp v1.13.6-0.20221020054726-e4133ab7e9bd
github.com/pkg/sftp v1.13.6-0.20230104082718-2489717da0f3
github.com/pquerna/otp v1.4.0
github.com/prometheus/client_golang v1.14.0
github.com/robfig/cron/v3 v3.0.1
@ -67,21 +67,21 @@ require (
go.etcd.io/bbolt v1.3.6
go.uber.org/automaxprocs v1.5.1
gocloud.dev v0.27.0
golang.org/x/crypto v0.4.0
golang.org/x/net v0.4.0
golang.org/x/oauth2 v0.3.0
golang.org/x/sys v0.3.0
golang.org/x/term v0.3.0
golang.org/x/crypto v0.5.0
golang.org/x/net v0.5.0
golang.org/x/oauth2 v0.4.0
golang.org/x/sys v0.4.0
golang.org/x/term v0.4.0
golang.org/x/time v0.3.0
google.golang.org/api v0.105.0
google.golang.org/api v0.106.0
gopkg.in/natefinch/lumberjack.v2 v2.0.0
)
require (
cloud.google.com/go v0.107.0 // indirect
cloud.google.com/go/compute v1.14.0 // indirect
cloud.google.com/go/compute v1.15.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v0.9.0 // indirect
cloud.google.com/go/iam v0.10.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect
github.com/ajg/form v1.5.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
@ -93,8 +93,8 @@ require (
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.22 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.21 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.28 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.12.0 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.0 // indirect
github.com/aws/smithy-go v1.13.5 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/boombuler/barcode v1.0.1 // indirect
@ -106,6 +106,7 @@ require (
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-jose/go-jose/v3 v3.0.0 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-test/deep v1.1.0 // indirect
github.com/goccy/go-json v0.10.0 // indirect
@ -156,22 +157,19 @@ require (
github.com/yusufpapurcu/wmi v1.2.2 // indirect
go.opencensus.io v0.24.0 // indirect
golang.org/x/mod v0.7.0 // indirect
golang.org/x/text v0.5.0 // indirect
golang.org/x/tools v0.4.0 // indirect
golang.org/x/text v0.6.0 // indirect
golang.org/x/tools v0.5.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect
google.golang.org/genproto v0.0.0-20230104163317-caabf589fcbf // indirect
google.golang.org/grpc v1.51.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/square/go-jose.v2 v2.6.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
replace (
github.com/fclairamb/ftpserverlib => github.com/drakkan/ftpserverlib v0.0.0-20221203115213-ba73c775a9fd
github.com/jlaffaye/ftp => github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9
github.com/pkg/sftp => github.com/drakkan/sftp v0.0.0-20221225162142-08880975fb1e
golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20221223081523-be6917ff6f72
golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20230106095953-5417b4dfde62
)

106
go.sum
View file

@ -50,8 +50,9 @@ cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6m
cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s=
cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU=
cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U=
cloud.google.com/go/compute v1.14.0 h1:hfm2+FfxVmnRlh6LpB7cg1ZNU+5edAHmW679JePztk0=
cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo=
cloud.google.com/go/compute v1.15.0 h1:PiKE4V948A1BRvhuwA2hOxL8imyvwuRgrOiytC+NlXo=
cloud.google.com/go/compute v1.15.0/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA=
cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
@ -60,8 +61,8 @@ cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqCl
cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY=
cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c=
cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY=
cloud.google.com/go/iam v0.9.0 h1:bK6Or6mxhuL8lnj1i9j0yMo2wE/IeTO2cWlfUrf/TZs=
cloud.google.com/go/iam v0.9.0/go.mod h1:nXAECrMt2qHpF6RZUZseteD6QyanL68reN4OXPw0UWM=
cloud.google.com/go/iam v0.10.0 h1:fpP/gByFs6US1ma53v7VxhvbJpO2Aapng6wabJ99MuI=
cloud.google.com/go/iam v0.10.0/go.mod h1:nXAECrMt2qHpF6RZUZseteD6QyanL68reN4OXPw0UWM=
cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA=
cloud.google.com/go/kms v1.6.0 h1:OWRZzrPmOZUzurjI2FBGtgY2mB1WaJkqhw6oIwSj0Yg=
cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs=
@ -233,17 +234,17 @@ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.3/go.mod h1:gNsR5CaXK
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno=
github.com/aws/aws-sdk-go-v2/config v1.15.15/go.mod h1:A1Lzyy/o21I5/s2FbyX5AevQfSVXpvvIDCoVFD0BC4E=
github.com/aws/aws-sdk-go-v2/config v1.18.7 h1:V94lTcix6jouwmAsgQMAEBozVAGJMFhVj+6/++xfe3E=
github.com/aws/aws-sdk-go-v2/config v1.18.7/go.mod h1:OZYsyHFL5PB9UpyS78NElgKs11qI/B5KJau2XOJDXHA=
github.com/aws/aws-sdk-go-v2/config v1.18.8 h1:lDpy0WM8AHsywOnVrOHaSMfpaiV2igOw8D7svkFkXVA=
github.com/aws/aws-sdk-go-v2/config v1.18.8/go.mod h1:5XCmmyutmzzgkpk/6NYTjeWb6lgo9N170m1j6pQkIBs=
github.com/aws/aws-sdk-go-v2/credentials v1.12.10/go.mod h1:g5eIM5XRs/OzIIK81QMBl+dAuDyoLN0VYaLP+tBqEOk=
github.com/aws/aws-sdk-go-v2/credentials v1.13.7 h1:qUUcNS5Z1092XBFT66IJM7mYkMwgZ8fcC8YDIbEwXck=
github.com/aws/aws-sdk-go-v2/credentials v1.13.7/go.mod h1:AdCcbZXHQCjJh6NaH3pFaw8LUeBFn5+88BZGMVGuBT8=
github.com/aws/aws-sdk-go-v2/credentials v1.13.8 h1:vTrwTvv5qAwjWIGhZDSBH/oQHuIQjGmD232k01FUh6A=
github.com/aws/aws-sdk-go-v2/credentials v1.13.8/go.mod h1:lVa4OHbvgjVot4gmh1uouF1ubgexSCN92P6CJQpT0t8=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.9/go.mod h1:KDCCm4ONIdHtUloDcFvK2+vshZvx4Zmj7UMDfusuz5s=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 h1:j9wi1kQ8b+e0FBVHxCqCGo4kxDU175hoDHcWAi0sauU=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21/go.mod h1:ugwW57Z5Z48bpvUyZuaPy4Kv+vEfJWnIrky7RmkBvJg=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.21/go.mod h1:iIYPrQ2rYfZiB/iADYlhj9HHZ9TTi6PqKQPAqygohbE=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.46 h1:OCX1pQ4pcqhsDV7B92HzdLWjHWOQsILvjLinpaUWhcc=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.46/go.mod h1:MxCBOcyNXGJRvfpPiH+L6n/BF9zbowthGSUZdDvQF/c=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.47 h1:E884ndKWVGt8IhtUuGhXbEsmaCvdAAkTTUDu7uAok1g=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.47/go.mod h1:KybsEsmXLO0u75FyS3F0sY4OQ97syDe8z+ISq8oEczA=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.15/go.mod h1:pWrr2OoHlT7M/Pd2y4HV3gJyPb3qj5qMmnPkKSNPYK4=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 h1:I3cakv2Uy1vNmmhRQmFptYDxOvBnwCdNwyw63N0RaRU=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27/go.mod h1:a1/UpzeyBBerajpnP5nGZa9mGzsBn5cOKxm6NWQsvoI=
@ -269,25 +270,25 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.9/go.mod h1:Rc5+wn2
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.21 h1:vY5siRXvW5TrOKm2qKEf9tliBfdLxdfy0i02LOcmqUo=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.21/go.mod h1:WZvNXT1XuH8dnJM0HvOlvk+RNn7NbAPvA/ACO0QarSc=
github.com/aws/aws-sdk-go-v2/service/kms v1.18.1/go.mod h1:4PZMUkc9rXHWGVB5J9vKaZy3D7Nai79ORworQ3ASMiM=
github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.26 h1:NlA5om7Um+ohI/S0inBd55Vsp84BKhBrAgw2IVqEb30=
github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.26/go.mod h1:DSuypbY6jb7WZSxrLuCgd7ouB5uRQ+Hg5wbt0GmgRcc=
github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.14.0 h1:zhGJVqFAHNmnYFGfPXqUgG+yHkmlsDb5R56B6rCNuRw=
github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.14.0/go.mod h1:DSuypbY6jb7WZSxrLuCgd7ouB5uRQ+Hg5wbt0GmgRcc=
github.com/aws/aws-sdk-go-v2/service/s3 v1.27.2/go.mod h1:u+566cosFI+d+motIz3USXEh6sN8Nq4GrNXSg2RXVMo=
github.com/aws/aws-sdk-go-v2/service/s3 v1.29.6 h1:W8pLcSn6Uy0eXgDBUUl8M8Kxv7JCoP68ZKTD04OXLEA=
github.com/aws/aws-sdk-go-v2/service/s3 v1.29.6/go.mod h1:L2l2/q76teehcW7YEsgsDjqdsDTERJeX3nOMIFlgGUE=
github.com/aws/aws-sdk-go-v2/service/s3 v1.30.0 h1:wddsyuESfviaiXk3w9N6/4iRwTg/a3gktjODY6jYQBo=
github.com/aws/aws-sdk-go-v2/service/s3 v1.30.0/go.mod h1:L2l2/q76teehcW7YEsgsDjqdsDTERJeX3nOMIFlgGUE=
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.14/go.mod h1:xakbH8KMsQQKqzX87uyyzTHshc/0/Df8bsTneTS5pFU=
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.17.0 h1:6W6BLZcXytRJsVvc2gGwxKE4wbMSlWqdxZivBP/E+ys=
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.17.0/go.mod h1:jAeo/PdIJZuDSwsvxJS94G4d6h8tStj7WXVuKwLHWU8=
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.18.0 h1:UQDiRZyaHQGPXIuCYqKsz/wIVZknCiZdRmPW8buD/xc=
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.18.0/go.mod h1:jAeo/PdIJZuDSwsvxJS94G4d6h8tStj7WXVuKwLHWU8=
github.com/aws/aws-sdk-go-v2/service/sns v1.17.10/go.mod h1:uITsRNVMeCB3MkWpXxXw0eDz8pW4TYLzj+eyQtbhSxM=
github.com/aws/aws-sdk-go-v2/service/sqs v1.19.1/go.mod h1:A94o564Gj+Yn+7QO1eLFeI7UVv3riy/YBFOfICVqFvU=
github.com/aws/aws-sdk-go-v2/service/ssm v1.27.6/go.mod h1:fiFzQgj4xNOg4/wqmAiPvzgDMXPD+cUEplX/CYn+0j0=
github.com/aws/aws-sdk-go-v2/service/sso v1.11.13/go.mod h1:d7ptRksDDgvXaUvxyHZ9SYh+iMDymm94JbVcgvSYSzU=
github.com/aws/aws-sdk-go-v2/service/sso v1.11.28 h1:gItLq3zBYyRDPmqAClgzTH8PBjDQGeyptYGHIwtYYNA=
github.com/aws/aws-sdk-go-v2/service/sso v1.11.28/go.mod h1:wo/B7uUm/7zw/dWhBJ4FXuw1sySU5lyIhVg1Bu2yL9A=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11 h1:KCacyVSs/wlcPGx37hcbT3IGYO8P8Jx+TgSDhAXtQMY=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11/go.mod h1:TZSH7xLO7+phDtViY/KUp9WGCJMQkLJ/VpgkTFd5gh8=
github.com/aws/aws-sdk-go-v2/service/sso v1.12.0 h1:/2gzjhQowRLarkkBOGPXSRnb8sQ2RVsjdG1C/UliK/c=
github.com/aws/aws-sdk-go-v2/service/sso v1.12.0/go.mod h1:wo/B7uUm/7zw/dWhBJ4FXuw1sySU5lyIhVg1Bu2yL9A=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.0 h1:Jfly6mRxk2ZOSlbCvZfKNS7TukSx1mIzhSsqZ/IGSZI=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.0/go.mod h1:TZSH7xLO7+phDtViY/KUp9WGCJMQkLJ/VpgkTFd5gh8=
github.com/aws/aws-sdk-go-v2/service/sts v1.16.10/go.mod h1:cftkHYN6tCDNfkSasAmclSfl4l7cySoay8vz7p/ce0E=
github.com/aws/aws-sdk-go-v2/service/sts v1.17.7 h1:9Mtq1KM6nD8/+HStvWcvYnixJ5N85DX+P+OY3kI3W2k=
github.com/aws/aws-sdk-go-v2/service/sts v1.17.7/go.mod h1:+lGbb3+1ugwKrNTWcf2RT05Xmp543B06zDFTwiTLp7I=
github.com/aws/aws-sdk-go-v2/service/sts v1.18.0 h1:kOO++CYo50RcTFISESluhWEi5Prhg+gaSs4whWabiZU=
github.com/aws/aws-sdk-go-v2/service/sts v1.18.0/go.mod h1:+lGbb3+1ugwKrNTWcf2RT05Xmp543B06zDFTwiTLp7I=
github.com/aws/smithy-go v1.12.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
@ -304,8 +305,8 @@ github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edY
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/bmatcuk/doublestar/v4 v4.4.0 h1:LmAwNwhjEbYtyVLzjcP/XeVw4nhuScHGkF/XWXnvIic=
github.com/bmatcuk/doublestar/v4 v4.4.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/bmatcuk/doublestar/v4 v4.6.0 h1:HTuxyug8GyFbRkrffIpzNCSK4luc0TY3wzXvzIZhEXc=
github.com/bmatcuk/doublestar/v4 v4.6.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
github.com/boombuler/barcode v1.0.1 h1:NDBbPmhS+EqABEs5Kg3n/5ZNjy73Pz7SIV+KCeqyXcs=
@ -472,8 +473,8 @@ github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmeka
github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
github.com/coreos/go-oidc/v3 v3.4.0 h1:xz7elHb/LDwm/ERpwHd+5nb7wFHL32rsr6bBOgaeu6g=
github.com/coreos/go-oidc/v3 v3.4.0/go.mod h1:eHUXhZtXPQLgEaDrOVTgwbgmz1xGOkJNye6h3zkD2Pw=
github.com/coreos/go-oidc/v3 v3.5.0 h1:VxKtbccHZxs8juq7RdJntSqtXFtde9YpNpGn0yqgEHw=
github.com/coreos/go-oidc/v3 v3.5.0/go.mod h1:ecXRtV4romGPeO6ieExAsUK9cb/3fp9hXNz1tlv8PIM=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@ -540,14 +541,10 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/drakkan/crypto v0.0.0-20221223081523-be6917ff6f72 h1:Ivant8yrd81A5y3tQOS7vqwL9QaOdlGonHNOfRR3rsQ=
github.com/drakkan/crypto v0.0.0-20221223081523-be6917ff6f72/go.mod h1:cy6DFZ6nHFw1bTHZksT/gYKmdxPdzr7Rw7xcJFSayo4=
github.com/drakkan/crypto v0.0.0-20230106095953-5417b4dfde62 h1:1Bk+GbTbF1PBu0idZumIiT7dQ2dW8UeswfJGWbxO4D8=
github.com/drakkan/crypto v0.0.0-20230106095953-5417b4dfde62/go.mod h1:eekSq7nI5pP2ZldL4867reOp0VL9TOfTaZa0DydSYk4=
github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9 h1:LPH1dEblAOO/LoG7yHPMtBLXhQmjaga91/DDjWk9jWA=
github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU=
github.com/drakkan/ftpserverlib v0.0.0-20221203115213-ba73c775a9fd h1:wu/ys+33GwD9PyRO8QDCUpI2WBZtwFiDk8QkFPW8rhQ=
github.com/drakkan/ftpserverlib v0.0.0-20221203115213-ba73c775a9fd/go.mod h1:FHiqwx5L+7z3o7EXRtT6asSd1uO4yTqEljqFU9L+zVA=
github.com/drakkan/sftp v0.0.0-20221225162142-08880975fb1e h1:Eeg6op40DlnZOarl7OWX9t1wdjkhUHT2kPlSkSHOvLA=
github.com/drakkan/sftp v0.0.0-20221225162142-08880975fb1e/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk=
github.com/drakkan/webdav v0.0.0-20221101181759-17ed21f9337b h1:B9z7XyDoVxLO4yEvnXgdvZ+0Uw9NA1qdD4KTSGmKcoQ=
github.com/drakkan/webdav v0.0.0-20221101181759-17ed21f9337b/go.mod h1:8opebuqUyBXrvl7Vo/S1Zzl9U0G1X2Ceud440eVuhUE=
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@ -584,6 +581,8 @@ github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL
github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
github.com/fclairamb/ftpserverlib v0.20.1-0.20230104020606-0b1a04eec221 h1:oIEBdcX1yNS5F+rk0xaDXMkwu9cT6+YSBEih45Wptec=
github.com/fclairamb/ftpserverlib v0.20.1-0.20230104020606-0b1a04eec221/go.mod h1:2PS2QXGtruTtfUszbKGOuuWhDiK5u/GD9DK2DdAW+S8=
github.com/fclairamb/go-log v0.4.1 h1:rLtdSG9x2pK41AIAnE8WYpl05xBJfw1ZyYxZaXFcBsM=
github.com/fclairamb/go-log v0.4.1/go.mod h1:sw1KvnkZ4wKCYkvy4SL3qVZcJSWFP8Ure4pM3z+KNn4=
github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
@ -611,8 +610,8 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U=
github.com/go-acme/lego/v4 v4.9.1 h1:n9Z5MQwANeGSQKlVE3bEh9SDvAySK9oVYOKCGCESqQE=
github.com/go-acme/lego/v4 v4.9.1/go.mod h1:g3JRUyWS3L/VObpp4bCxzJftKyf/Wba8QrSSnoiqjg4=
github.com/go-acme/lego/v4 v4.9.2-0.20230104103215-fd54758bba4c h1:PDd4Q867Ia2D68T+KglkyxMDoIUEp3sNYVXuN3TXjAE=
github.com/go-acme/lego/v4 v4.9.2-0.20230104103215-fd54758bba4c/go.mod h1:qib35rauo2OW1BzAI0qUfR3xw/JIIuaO0ZA83QIsw0s=
github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0=
github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
github.com/go-chi/jwtauth/v5 v5.1.0 h1:wJyf2YZ/ohPvNJBwPOzZaQbyzwgMZZceE1m8FOzXLeA=
@ -623,6 +622,8 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo=
github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o=
@ -1070,7 +1071,6 @@ github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e
github.com/klauspost/compress v1.15.14 h1:i7WCKDToww0wA+9qrUZ1xOjp218vfFo3nTU6UHp+gOc=
github.com/klauspost/compress v1.15.14/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.2/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU=
github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
github.com/kolo/xmlrpc v0.0.0-20201022064351-38db28db192b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM=
@ -1342,6 +1342,10 @@ github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
github.com/pkg/sftp v1.13.6-0.20230104082718-2489717da0f3 h1:eKBJ919kpjpfHltsNthMO6ZQ/XQy76cHHbuz2bOmMSA=
github.com/pkg/sftp v1.13.6-0.20230104082718-2489717da0f3/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
@ -1818,9 +1822,10 @@ golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220802222814-0bcc04d9c69b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU=
golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -1846,9 +1851,9 @@ golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7Lm
golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
golang.org/x/oauth2 v0.0.0-20220628200809-02e64fa58f26/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
golang.org/x/oauth2 v0.3.0 h1:6l90koy8/LaBLmLu8jpHeHexzMwEita0zFfYlggy2F8=
golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk=
golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M=
golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -2011,14 +2016,16 @@ golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI=
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg=
golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -2028,8 +2035,9 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@ -2128,8 +2136,8 @@ golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.4.0 h1:7mTAgkunk3fr4GAloyyCasadO6h9zSsQZbwvcaIciV4=
golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
golang.org/x/tools v0.5.0 h1:+bSpV5HIeWkuvgaMfI3UmKRThoTA5ODJTUd8T17NO+4=
golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k=
golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -2189,8 +2197,8 @@ google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6F
google.golang.org/api v0.86.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
google.golang.org/api v0.91.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
google.golang.org/api v0.105.0 h1:t6P9Jj+6XTn4U9I2wycQai6Q/Kz7iOT+QzjJ3G2V4x8=
google.golang.org/api v0.105.0/go.mod h1:qh7eD5FJks5+BcE+cjBIm6Gz8vioK7EHvnlniqXBnqI=
google.golang.org/api v0.106.0 h1:ffmW0faWCwKkpbbtvlY/K/8fUl+JKvNS5CVzRoyfCv8=
google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@ -2301,8 +2309,8 @@ google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljW
google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
google.golang.org/genproto v0.0.0-20220802133213-ce4fa296bf78/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc=
google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef h1:uQ2vjV/sHTsWSqdKeLqmwitzgvjMl7o4IdtHwUDXSJY=
google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
google.golang.org/genproto v0.0.0-20230104163317-caabf589fcbf h1:/JqRexUvugu6JURQ0O7RfV1EnvgrOxUV4tSjuAv0Sr0=
google.golang.org/genproto v0.0.0-20230104163317-caabf589fcbf/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
@ -2389,8 +2397,6 @@ gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI=
gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
gopkg.in/telebot.v3 v3.0.0/go.mod h1:7rExV8/0mDDNu9epSrDm/8j22KLaActH1Tbee6YjzWg=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=

6
internal/common/common.go
View file

@ -220,6 +220,7 @@ func Initialize(c Configuration, isShared int) error {
vfs.SetTempPath(c.TempPath)
dataprovider.SetTempPath(c.TempPath)
vfs.SetAllowSelfConnections(c.AllowSelfConnections)
vfs.SetRenameMode(c.RenameMode)
dataprovider.SetAllowSelfConnections(c.AllowSelfConnections)
transfersChecker = getTransfersChecker(isShared)
return nil
@ -529,6 +530,11 @@ type Configuration struct {
// silently ignored for cloud based filesystem such as S3, GCS, Azure Blob. Requests for changing
// modification times are ignored for cloud based filesystem if they are not supported.
SetstatMode int `json:"setstat_mode" mapstructure:"setstat_mode"`
// RenameMode defines how to handle directory renames. By default, renaming of non-empty directories
// is not allowed for cloud storage providers (S3, GCS, Azure Blob). Set to 1 to enable recursive
// renames for these providers, they may be slow, there is no atomic rename API like for local
// filesystem, so SFTPGo will recursively list the directory contents and do a rename for each entry
RenameMode int `json:"rename_mode" mapstructure:"rename_mode"`
// TempPath defines the path for temporary files such as those used for atomic uploads or file pipes.
// If you set this option you must make sure that the defined path exists, is accessible for writing
// by the user running SFTPGo, and is on the same filesystem as the users home directories otherwise

95
internal/common/connection.go
View file

@ -470,7 +470,7 @@ func (c *BaseConnection) RemoveDir(virtualPath string) error {
if fs.IsNotExist(err) && fs.HasVirtualFolders() {
return nil
}
c.Log(logger.LevelError, "failed to remove a dir %#v: stat error: %+v", fsPath, err)
c.Log(logger.LevelError, "failed to remove a dir %q: stat error: %+v", fsPath, err)
return c.GetFsError(fs, err)
}
if !fi.IsDir() || fi.Mode()&os.ModeSymlink != 0 {
@ -671,6 +671,10 @@ func (c *BaseConnection) Copy(virtualSourcePath, virtualTargetPath string) error
if err := c.CheckParentDirs(path.Dir(destPath)); err != nil {
return err
}
done := make(chan bool)
defer close(done)
go keepConnectionAlive(c, done, 2*time.Minute)
return c.doRecursiveCopy(virtualSourcePath, destPath, srcInfo, createTargetDir)
}
@ -728,12 +732,17 @@ func (c *BaseConnection) renameInternal(virtualSourcePath, virtualTargetPath str
if checkParentDestination {
c.CheckParentDirs(path.Dir(virtualTargetPath)) //nolint:errcheck
}
if err := fsDst.Rename(fsSourcePath, fsTargetPath); err != nil {
done := make(chan bool)
defer close(done)
go keepConnectionAlive(c, done, 2*time.Minute)
files, size, err := fsDst.Rename(fsSourcePath, fsTargetPath)
if err != nil {
c.Log(logger.LevelError, "failed to rename %q -> %q: %+v", fsSourcePath, fsTargetPath, err)
return c.GetFsError(fsSrc, err)
}
vfs.SetPathPermissions(fsDst, fsTargetPath, c.User.GetUID(), c.User.GetGID())
c.updateQuotaAfterRename(fsDst, virtualSourcePath, virtualTargetPath, fsTargetPath, initialSize) //nolint:errcheck
c.updateQuotaAfterRename(fsDst, virtualSourcePath, virtualTargetPath, fsTargetPath, initialSize, files, size) //nolint:errcheck
logger.CommandLog(renameLogSender, fsSourcePath, fsTargetPath, c.User.Username, "", c.ID, c.protocol, -1, -1,
"", "", "", -1, c.localAddr, c.remoteAddr)
ExecuteActionNotification(c, operationRename, fsSourcePath, virtualSourcePath, fsTargetPath, //nolint:errcheck
@ -1008,7 +1017,7 @@ func (c *BaseConnection) checkRecursiveRenameDirPermissions(fsSrc, fsDst vfs.Fs,
if !c.User.HasPermissionsInside(virtualSourcePath) &&
!c.User.HasPermissionsInside(virtualTargetPath) {
if !c.isRenamePermitted(fsSrc, fsDst, sourcePath, targetPath, virtualSourcePath, virtualTargetPath, fi) {
c.Log(logger.LevelInfo, "rename %#v -> %#v is not allowed, virtual destination path: %#v",
c.Log(logger.LevelInfo, "rename %q -> %q is not allowed, virtual destination path: %q",
sourcePath, targetPath, virtualTargetPath)
return c.GetPermissionDeniedError()
}
@ -1024,7 +1033,7 @@ func (c *BaseConnection) checkRecursiveRenameDirPermissions(fsSrc, fsDst vfs.Fs,
if err != nil {
return c.GetFsError(fsSrc, err)
}
if walkedPath != sourcePath && vfs.HasImplicitAtomicUploads(fsSrc) {
if walkedPath != sourcePath && vfs.HasImplicitAtomicUploads(fsSrc) && Config.RenameMode == 0 {
c.Log(logger.LevelInfo, "cannot rename non empty directory %q on this filesystem", virtualSourcePath)
return c.GetOpUnsupportedError()
}
@ -1083,6 +1092,11 @@ func (c *BaseConnection) checkFolderRename(fsSrc, fsDst vfs.Fs, fsSourcePath, fs
virtualSourcePath)
return c.GetOpUnsupportedError()
}
if c.User.HasVirtualFoldersInside(virtualTargetPath) {
c.Log(logger.LevelDebug, "renaming the folder %q is not supported, the target %q has virtual folders inside it",
virtualSourcePath, virtualTargetPath)
return c.GetOpUnsupportedError()
}
if err := c.checkRecursiveRenameDirPermissions(fsSrc, fsDst, fsSourcePath, fsTargetPath,
virtualSourcePath, virtualTargetPath, fi); err != nil {
c.Log(logger.LevelDebug, "error checking recursive permissions before renaming %q: %+v", fsSourcePath, err)
@ -1118,7 +1132,7 @@ func (c *BaseConnection) isRenamePermitted(fsSrc, fsDst vfs.Fs, fsSourcePath, fs
isSrcAllowed, _ := c.User.IsFileAllowed(virtualSourcePath)
isDstAllowed, _ := c.User.IsFileAllowed(virtualTargetPath)
if !isSrcAllowed || !isDstAllowed {
c.Log(logger.LevelDebug, "renaming source: %#v to target: %#v not allowed", virtualSourcePath,
c.Log(logger.LevelDebug, "renaming source: %q to target: %q not allowed", virtualSourcePath,
virtualTargetPath)
return false
}
@ -1147,7 +1161,15 @@ func (c *BaseConnection) hasSpaceForRename(fs vfs.Fs, virtualSourcePath, virtual
// rename between user root dir and a virtual folder included in user quota
return true
}
if errDst != nil && sourceFolder.IsIncludedInUserQuota() {
// rename between a virtual folder included in user quota and the user root dir
return true
}
quotaResult, _ := c.HasSpace(true, false, virtualTargetPath)
if quotaResult.HasSpace && quotaResult.QuotaSize == 0 && quotaResult.QuotaFiles == 0 {
// no quota restrictions
return true
}
return c.hasSpaceForCrossRename(fs, quotaResult, initialSize, fsSourcePath)
}
@ -1159,7 +1181,7 @@ func (c *BaseConnection) hasSpaceForCrossRename(fs vfs.Fs, quotaResult vfs.Quota
}
fi, err := fs.Lstat(sourcePath)
if err != nil {
c.Log(logger.LevelError, "cross rename denied, stat error for path %#v: %v", sourcePath, err)
c.Log(logger.LevelError, "cross rename denied, stat error for path %q: %v", sourcePath, err)
return false
}
var sizeDiff int64
@ -1174,7 +1196,7 @@ func (c *BaseConnection) hasSpaceForCrossRename(fs vfs.Fs, quotaResult vfs.Quota
} else if fi.IsDir() {
filesDiff, sizeDiff, err = fs.GetDirSize(sourcePath)
if err != nil {
c.Log(logger.LevelError, "cross rename denied, error getting size for directory %#v: %v", sourcePath, err)
c.Log(logger.LevelError, "cross rename denied, error getting size for directory %q: %v", sourcePath, err)
return false
}
}
@ -1183,14 +1205,14 @@ func (c *BaseConnection) hasSpaceForCrossRename(fs vfs.Fs, quotaResult vfs.Quota
if quotaResult.QuotaSize == 0 {
return true
}
c.Log(logger.LevelDebug, "cross rename overwrite, source %#v, used size %v, size to add %v",
c.Log(logger.LevelDebug, "cross rename overwrite, source %q, used size %d, size to add %d",
sourcePath, quotaResult.UsedSize, sizeDiff)
quotaResult.UsedSize += sizeDiff
return quotaResult.GetRemainingSize() >= 0
}
if quotaResult.QuotaFiles > 0 {
remainingFiles := quotaResult.GetRemainingFiles()
c.Log(logger.LevelDebug, "cross rename, source %#v remaining file %v to add %v", sourcePath,
c.Log(logger.LevelDebug, "cross rename, source %q remaining file %d to add %d", sourcePath,
remainingFiles, filesDiff)
if remainingFiles < filesDiff {
return false
@ -1198,7 +1220,7 @@ func (c *BaseConnection) hasSpaceForCrossRename(fs vfs.Fs, quotaResult vfs.Quota
}
if quotaResult.QuotaSize > 0 {
remainingSize := quotaResult.GetRemainingSize()
c.Log(logger.LevelDebug, "cross rename, source %#v remaining size %v to add %v", sourcePath,
c.Log(logger.LevelDebug, "cross rename, source %q remaining size %d to add %d", sourcePath,
remainingSize, sizeDiff)
if remainingSize < sizeDiff {
return false
@ -1329,7 +1351,7 @@ func (c *BaseConnection) HasSpace(checkFiles, getUsage bool, requestPath string)
result.AllowedSize = result.QuotaSize - result.UsedSize
if (checkFiles && result.QuotaFiles > 0 && result.UsedFiles >= result.QuotaFiles) ||
(result.QuotaSize > 0 && result.UsedSize >= result.QuotaSize) {
c.Log(logger.LevelDebug, "quota exceed for user %#v, request path %#v, num files: %v/%v, size: %v/%v check files: %v",
c.Log(logger.LevelDebug, "quota exceed for user %q, request path %q, num files: %d/%d, size: %d/%d check files: %t",
c.User.Username, requestPath, result.UsedFiles, result.QuotaFiles, result.UsedSize, result.QuotaSize, checkFiles)
result.HasSpace = false
return result, transferQuota
@ -1430,7 +1452,9 @@ func (c *BaseConnection) updateQuotaMoveToVFolder(dstFolder *vfs.VirtualFolder,
}
}
func (c *BaseConnection) updateQuotaAfterRename(fs vfs.Fs, virtualSourcePath, virtualTargetPath, targetPath string, initialSize int64) error {
func (c *BaseConnection) updateQuotaAfterRename(fs vfs.Fs, virtualSourcePath, virtualTargetPath, targetPath string,
initialSize int64, numFiles int, filesSize int64,
) error {
if dataprovider.GetQuotaTracking() == 0 {
return nil
}
@ -1451,22 +1475,27 @@ func (c *BaseConnection) updateQuotaAfterRename(fs vfs.Fs, virtualSourcePath, vi
return nil
}
filesSize := int64(0)
numFiles := 1
if fi, err := fs.Stat(targetPath); err == nil {
if fi.Mode().IsDir() {
numFiles, filesSize, err = fs.GetDirSize(targetPath)
if err != nil {
c.Log(logger.LevelError, "failed to update quota after rename, error scanning moved folder %#v: %v",
targetPath, err)
return err
if filesSize == -1 {
// fs.Rename didn't return the affected files/sizes, we need to calculate them
numFiles = 1
if fi, err := fs.Stat(targetPath); err == nil {
if fi.Mode().IsDir() {
numFiles, filesSize, err = fs.GetDirSize(targetPath)
if err != nil {
c.Log(logger.LevelError, "failed to update quota after rename, error scanning moved folder %q: %+v",
targetPath, err)
return err
}
} else {
filesSize = fi.Size()
}
} else {
filesSize = fi.Size()
c.Log(logger.LevelError, "failed to update quota after renaming, file %q stat error: %+v", targetPath, err)
return err
}
c.Log(logger.LevelDebug, "calculated renamed files: %d, size: %d bytes", numFiles, filesSize)
} else {
c.Log(logger.LevelError, "failed to update quota after rename, file %#v stat error: %+v", targetPath, err)
return err
c.Log(logger.LevelDebug, "returned renamed files: %d, size: %d bytes", numFiles, filesSize)
}
if errSrc == nil && errDst == nil {
c.updateQuotaMoveBetweenVFolders(&sourceFolder, &dstFolder, initialSize, filesSize, numFiles)
@ -1660,3 +1689,19 @@ func (c *BaseConnection) GetFsAndResolvedPath(virtualPath string) (vfs.Fs, strin
return fs, fsPath, nil
}
func keepConnectionAlive(c *BaseConnection, done chan bool, interval time.Duration) {
ticker := time.NewTicker(interval)
defer func() {
ticker.Stop()
}()
for {
select {
case <-done:
return
case <-ticker.C:
c.UpdateLastActivity()
}
}
}

33
internal/common/connection_test.go
View file

@ -280,6 +280,13 @@ func TestRenamePerms(t *testing.T) {
func TestRenameNestedFolders(t *testing.T) {
u := dataprovider.User{}
u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{
BaseVirtualFolder: vfs.BaseVirtualFolder{
Name: "vfolder",
MappedPath: filepath.Join(os.TempDir(), "f"),
},
VirtualPath: "/vdirs/f",
})
conn := NewBaseConnection("", ProtocolSFTP, "", "", u)
err := conn.checkFolderRename(nil, nil, filepath.Clean(os.TempDir()), filepath.Join(os.TempDir(), "subdir"), "/src", "/dst", nil)
assert.Error(t, err)
@ -287,6 +294,8 @@ func TestRenameNestedFolders(t *testing.T) {
assert.Error(t, err)
err = conn.checkFolderRename(nil, nil, "", "", "/src/sub", "/src", nil)
assert.Error(t, err)
err = conn.checkFolderRename(nil, nil, filepath.Join(os.TempDir(), "src"), filepath.Join(os.TempDir(), "vdirs"), "/src", "/vdirs", nil)
assert.Error(t, err)
}
func TestUpdateQuotaAfterRename(t *testing.T) {
@ -331,7 +340,7 @@ func TestUpdateQuotaAfterRename(t *testing.T) {
assert.NoError(t, err)
err = os.Chmod(testDirPath, 0001)
assert.NoError(t, err)
err = c.updateQuotaAfterRename(fs, request.Filepath, request.Target, testDirPath, 0)
err = c.updateQuotaAfterRename(fs, request.Filepath, request.Target, testDirPath, 0, -1, -1)
assert.Error(t, err)
err = os.Chmod(testDirPath, os.ModePerm)
assert.NoError(t, err)
@ -339,23 +348,25 @@ func TestUpdateQuotaAfterRename(t *testing.T) {
testFile1 := "/testfile1"
request.Target = testFile1
request.Filepath = path.Join("/vdir", "file")
err = c.updateQuotaAfterRename(fs, request.Filepath, request.Target, filepath.Join(mappedPath, "file"), 0)
err = c.updateQuotaAfterRename(fs, request.Filepath, request.Target, filepath.Join(mappedPath, "file"), 0, -1, -1)
assert.Error(t, err)
err = os.WriteFile(filepath.Join(mappedPath, "file"), []byte("test content"), os.ModePerm)
assert.NoError(t, err)
request.Filepath = testFile1
request.Target = path.Join("/vdir", "file")
err = c.updateQuotaAfterRename(fs, request.Filepath, request.Target, filepath.Join(mappedPath, "file"), 12)
err = c.updateQuotaAfterRename(fs, request.Filepath, request.Target, filepath.Join(mappedPath, "file"), 12, -1, -1)
assert.NoError(t, err)
err = os.WriteFile(filepath.Join(user.GetHomeDir(), "testfile1"), []byte("test content"), os.ModePerm)
assert.NoError(t, err)
request.Target = testFile1
request.Filepath = path.Join("/vdir", "file")
err = c.updateQuotaAfterRename(fs, request.Filepath, request.Target, filepath.Join(mappedPath, "file"), 12)
err = c.updateQuotaAfterRename(fs, request.Filepath, request.Target, filepath.Join(mappedPath, "file"), 12, -1, -1)
assert.NoError(t, err)
request.Target = path.Join("/vdir1", "file")
request.Filepath = path.Join("/vdir", "file")
err = c.updateQuotaAfterRename(fs, request.Filepath, request.Target, filepath.Join(mappedPath, "file"), 12)
err = c.updateQuotaAfterRename(fs, request.Filepath, request.Target, filepath.Join(mappedPath, "file"), 12, -1, -1)
assert.NoError(t, err)
err = c.updateQuotaAfterRename(fs, request.Filepath, request.Target, filepath.Join(mappedPath, "file"), 12, 1, 100)
assert.NoError(t, err)
err = os.RemoveAll(mappedPath)
@ -604,3 +615,15 @@ func TestErrorResolvePath(t *testing.T) {
err = os.RemoveAll(filepath.Dir(sourceFile))
assert.NoError(t, err)
}
func TestConnectionKeepAlive(t *testing.T) {
conn := NewBaseConnection("", ProtocolWebDAV, "", "", dataprovider.User{})
lastActivity := conn.GetLastActivity()
done := make(chan bool)
go func() {
time.Sleep(200 * time.Millisecond)
close(done)
}()
keepConnectionAlive(conn, done, 50*time.Millisecond)
assert.Greater(t, conn.GetLastActivity(), lastActivity)
}

2
internal/common/transfer.go
View file

@ -372,7 +372,7 @@ func (t *BaseTransfer) Close() error {
t.File.Name(), err)
} else if t.transferType == TransferUpload && t.effectiveFsPath != t.fsPath {
if t.ErrTransfer == nil || Config.UploadMode == UploadModeAtomicWithResume {
err = t.Fs.Rename(t.effectiveFsPath, t.fsPath)
_, _, err = t.Fs.Rename(t.effectiveFsPath, t.fsPath)
t.Connection.Log(logger.LevelDebug, "atomic upload completed, rename: %#v -> %#v, error: %v",
t.effectiveFsPath, t.fsPath, err)
// the file must be removed if it is uploaded to a path outside the home dir and cannot be renamed

2
internal/config/config.go
View file

@ -201,6 +201,7 @@ func Init() {
Hook: "",
},
SetstatMode: 0,
RenameMode: 0,
TempPath: "",
ProxyProtocol: 0,
ProxyAllowed: []string{},
@ -1922,6 +1923,7 @@ func setViperDefaults() {
viper.SetDefault("common.actions.execute_sync", globalConf.Common.Actions.ExecuteSync)
viper.SetDefault("common.actions.hook", globalConf.Common.Actions.Hook)
viper.SetDefault("common.setstat_mode", globalConf.Common.SetstatMode)
viper.SetDefault("common.rename_mode", globalConf.Common.RenameMode)
viper.SetDefault("common.temp_path", globalConf.Common.TempPath)
viper.SetDefault("common.proxy_protocol", globalConf.Common.ProxyProtocol)
viper.SetDefault("common.proxy_allowed", globalConf.Common.ProxyAllowed)

2
internal/ftpd/handler.go
View file

@ -455,7 +455,7 @@ func (c *Connection) handleFTPUploadToExistingFile(fs vfs.Fs, flags int, resolve
}
if common.Config.IsAtomicUploadEnabled() && fs.IsAtomicUploadSupported() {
err = fs.Rename(resolvedPath, filePath)
_, _, err = fs.Rename(resolvedPath, filePath)
if err != nil {
c.Log(logger.LevelError, "error renaming existing file for atomic upload, source: %#v, dest: %#v, err: %+v",
resolvedPath, filePath, err)

7
internal/ftpd/internal_test.go
View file

@ -387,11 +387,12 @@ func (fs MockOsFs) Remove(name string, isDir bool) error {
}
// Rename renames (moves) source to target
func (fs MockOsFs) Rename(source, target string) error {
func (fs MockOsFs) Rename(source, target string) (int, int64, error) {
if fs.err != nil {
return fs.err
return -1, -1, fs.err
}
return os.Rename(source, target)
err := os.Rename(source, target)
return -1, -1, err
}
func newMockOsFs(err, statErr error, atomicUpload bool, connectionID, rootDir string) vfs.Fs {

2
internal/httpd/handler.go
View file

@ -176,7 +176,7 @@ func (c *Connection) getFileWriter(name string) (io.WriteCloser, error) {
}
if common.Config.IsAtomicUploadEnabled() && fs.IsAtomicUploadSupported() {
err = fs.Rename(p, filePath)
_, _, err = fs.Rename(p, filePath)
if err != nil {
c.Log(logger.LevelError, "error renaming existing file for atomic upload, source: %#v, dest: %#v, err: %+v",
p, filePath, err)

2
internal/sftpd/handler.go
View file

@ -455,7 +455,7 @@ func (c *Connection) handleSFTPUploadToExistingFile(fs vfs.Fs, pflags sftp.FileO
}
if common.Config.IsAtomicUploadEnabled() && fs.IsAtomicUploadSupported() {
err = fs.Rename(resolvedPath, filePath)
_, _, err = fs.Rename(resolvedPath, filePath)
if err != nil {
c.Log(logger.LevelError, "error renaming existing file for atomic upload, source: %#v, dest: %#v, err: %+v",
resolvedPath, filePath, err)

7
internal/sftpd/internal_test.go
View file

@ -138,11 +138,12 @@ func (fs MockOsFs) Remove(name string, isDir bool) error {
}
// Rename renames (moves) source to target
func (fs MockOsFs) Rename(source, target string) error {
func (fs MockOsFs) Rename(source, target string) (int, int64, error) {
if fs.err != nil {
return fs.err
return -1, -1, fs.err
}
return os.Rename(source, target)
err := os.Rename(source, target)
return -1, -1, err
}
func newMockOsFs(err, statErr error, atomicUpload bool, connectionID, rootDir string) vfs.Fs {

2
internal/sftpd/scp.go
View file

@ -335,7 +335,7 @@ func (c *scpCommand) handleUpload(uploadFilePath string, sizeToRead int64) error
}
if common.Config.IsAtomicUploadEnabled() && fs.IsAtomicUploadSupported() {
err = fs.Rename(p, filePath)
_, _, err = fs.Rename(p, filePath)
if err != nil {
c.connection.Log(logger.LevelError, "error renaming existing file for atomic upload, source: %#v, dest: %#v, err: %v",
p, filePath, err)

2
internal/sftpd/sftpd_test.go
View file

@ -5166,7 +5166,7 @@ func TestVirtualFoldersQuotaLimit(t *testing.T) {
err = client.Rename(path.Join(vdirPath1, testFileName+".rename"), path.Join(vdirPath2, testFileName))
assert.Error(t, err)
err = client.Rename(path.Join(vdirPath1, testFileName+".rename"), testFileName)
assert.Error(t, err)
assert.NoError(t, err)
}
_, err = httpdtest.RemoveUser(user, http.StatusOK)
assert.NoError(t, err)

251
internal/vfs/azblobfs.go
View file

@ -273,68 +273,15 @@ func (fs *AzureBlobFs) Create(name string, flag int) (File, *PipeWriter, func(),
}
// Rename renames (moves) source to target.
// We don't support renaming non empty directories since we should
// rename all the contents too and this could take long time: think
// about directories with thousands of files, for each file we should
// execute a StartCopyFromURL call.
func (fs *AzureBlobFs) Rename(source, target string) error {
func (fs *AzureBlobFs) Rename(source, target string) (int, int64, error) {
if source == target {
return nil
return -1, -1, nil
}
fi, err := fs.Stat(source)
if err != nil {
return err
return -1, -1, err
}
if fi.IsDir() {
hasContents, err := fs.hasContents(source)
if err != nil {
return err
}
if hasContents {
return fmt.Errorf("cannot rename non empty directory: %#v", source)
}
if err := fs.mkdirInternal(target); err != nil {
return err
}
} else {
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout))
defer cancelFn()
srcBlob := fs.containerClient.NewBlockBlobClient(source)
dstBlob := fs.containerClient.NewBlockBlobClient(target)
resp, err := dstBlob.StartCopyFromURL(ctx, srcBlob.URL(), fs.getCopyOptions())
if err != nil {
metric.AZCopyObjectCompleted(err)
return err
}
copyStatus := blob.CopyStatusType(util.GetStringFromPointer((*string)(resp.CopyStatus)))
nErrors := 0
for copyStatus == blob.CopyStatusTypePending {
// Poll until the copy is complete.
time.Sleep(500 * time.Millisecond)
resp, err := dstBlob.GetProperties(ctx, &blob.GetPropertiesOptions{})
if err != nil {
// A GetProperties failure may be transient, so allow a couple
// of them before giving up.
nErrors++
if ctx.Err() != nil || nErrors == 3 {
metric.AZCopyObjectCompleted(err)
return err
}
} else {
copyStatus = blob.CopyStatusType(util.GetStringFromPointer((*string)(resp.CopyStatus)))
}
}
if copyStatus != blob.CopyStatusTypeSuccess {
err := fmt.Errorf("copy failed with status: %s", copyStatus)
metric.AZCopyObjectCompleted(err)
return err
}
metric.AZCopyObjectCompleted(nil)
fs.preserveModificationTime(source, target, fi)
}
return fs.Remove(source, fi.IsDir())
return fs.renameInternal(source, target, fi)
}
// Remove removes the named file or (empty) directory.
@ -575,44 +522,7 @@ func (fs *AzureBlobFs) CheckRootPath(username string, uid int, gid int) bool {
// ScanRootDirContents returns the number of files contained in the bucket,
// and their size
func (fs *AzureBlobFs) ScanRootDirContents() (int, int64, error) {
numFiles := 0
size := int64(0)
pager := fs.containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
Include: container.ListBlobsInclude{
Metadata: true,
},
Prefix: &fs.config.KeyPrefix,
})
for pager.More() {
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
defer cancelFn()
resp, err := pager.NextPage(ctx)
if err != nil {
metric.AZListObjectsCompleted(err)
return numFiles, size, err
}
for _, blobItem := range resp.ListBlobsFlatSegmentResponse.Segment.BlobItems {
if blobItem.Properties != nil {
contentType := util.GetStringFromPointer(blobItem.Properties.ContentType)
isDir := checkDirectoryMarkers(contentType, blobItem.Metadata)
blobSize := util.GetIntFromPointer(blobItem.Properties.ContentLength)
if isDir && blobSize == 0 {
continue
}
numFiles++
size += blobSize
if numFiles%1000 == 0 {
fsLog(fs, logger.LevelDebug, "root dir scan in progress, files: %d, size: %d", numFiles, size)
}
}
}
}
metric.AZListObjectsCompleted(nil)
return numFiles, size, nil
return fs.GetDirSize(fs.config.KeyPrefix)
}
func (fs *AzureBlobFs) getFileNamesInPrefix(fsPrefix string) (map[string]bool, error) {
@ -663,8 +573,46 @@ func (fs *AzureBlobFs) CheckMetadata() error {
// GetDirSize returns the number of files and the size for a folder
// including any subfolders
func (*AzureBlobFs) GetDirSize(dirname string) (int, int64, error) {
return 0, 0, ErrVfsUnsupported
func (fs *AzureBlobFs) GetDirSize(dirname string) (int, int64, error) {
numFiles := 0
size := int64(0)
prefix := fs.getPrefix(dirname)
pager := fs.containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
Include: container.ListBlobsInclude{
Metadata: true,
},
Prefix: &prefix,
})
for pager.More() {
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
defer cancelFn()
resp, err := pager.NextPage(ctx)
if err != nil {
metric.AZListObjectsCompleted(err)
return numFiles, size, err
}
for _, blobItem := range resp.ListBlobsFlatSegmentResponse.Segment.BlobItems {
if blobItem.Properties != nil {
contentType := util.GetStringFromPointer(blobItem.Properties.ContentType)
isDir := checkDirectoryMarkers(contentType, blobItem.Metadata)
blobSize := util.GetIntFromPointer(blobItem.Properties.ContentLength)
if isDir && blobSize == 0 {
continue
}
numFiles++
size += blobSize
if numFiles%1000 == 0 {
fsLog(fs, logger.LevelDebug, "dirname %q scan in progress, files: %d, size: %d", dirname, numFiles, size)
}
}
}
}
metric.AZListObjectsCompleted(nil)
return numFiles, size, nil
}
// GetAtomicUploadPath returns the path to use for an atomic upload.
@ -838,6 +786,102 @@ func (fs *AzureBlobFs) setConfigDefaults() {
}
}
func (fs *AzureBlobFs) copyFileInternal(source, target string) error {
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout))
defer cancelFn()
srcBlob := fs.containerClient.NewBlockBlobClient(source)
dstBlob := fs.containerClient.NewBlockBlobClient(target)
resp, err := dstBlob.StartCopyFromURL(ctx, srcBlob.URL(), fs.getCopyOptions())
if err != nil {
metric.AZCopyObjectCompleted(err)
return err
}
copyStatus := blob.CopyStatusType(util.GetStringFromPointer((*string)(resp.CopyStatus)))
nErrors := 0
for copyStatus == blob.CopyStatusTypePending {
// Poll until the copy is complete.
time.Sleep(500 * time.Millisecond)
resp, err := dstBlob.GetProperties(ctx, &blob.GetPropertiesOptions{})
if err != nil {
// A GetProperties failure may be transient, so allow a couple
// of them before giving up.
nErrors++
if ctx.Err() != nil || nErrors == 3 {
metric.AZCopyObjectCompleted(err)
return err
}
} else {
copyStatus = blob.CopyStatusType(util.GetStringFromPointer((*string)(resp.CopyStatus)))
}
}
if copyStatus != blob.CopyStatusTypeSuccess {
err := fmt.Errorf("copy failed with status: %s", copyStatus)
metric.AZCopyObjectCompleted(err)
return err
}
metric.AZCopyObjectCompleted(nil)
return nil
}
func (fs *AzureBlobFs) renameInternal(source, target string, fi os.FileInfo) (int, int64, error) {
var numFiles int
var filesSize int64
if fi.IsDir() {
if renameMode == 0 {
hasContents, err := fs.hasContents(source)
if err != nil {
return numFiles, filesSize, err
}
if hasContents {
return numFiles, filesSize, fmt.Errorf("cannot rename non empty directory: %q", source)
}
}
if err := fs.mkdirInternal(target); err != nil {
return numFiles, filesSize, err
}
if renameMode == 1 {
entries, err := fs.ReadDir(source)
if err != nil {
return numFiles, filesSize, err
}
for _, info := range entries {
sourceEntry := fs.Join(source, info.Name())
targetEntry := fs.Join(target, info.Name())
files, size, err := fs.renameInternal(sourceEntry, targetEntry, info)
if err != nil {
return numFiles, filesSize, err
}
numFiles += files
filesSize += size
}
}
} else {
if err := fs.copyFileInternal(source, target); err != nil {
return numFiles, filesSize, err
}
numFiles++
filesSize += fi.Size()
if plugin.Handler.HasMetadater() {
if !fi.IsDir() {
err := plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(target),
util.GetTimeAsMsSinceEpoch(fi.ModTime()))
if err != nil {
fsLog(fs, logger.LevelWarn, "unable to preserve modification time after renaming %q -> %q: %+v",
source, target, err)
}
}
}
}
err := fs.Remove(source, fi.IsDir())
if fs.IsNotExist(err) {
err = nil
}
return numFiles, filesSize, err
}
func (fs *AzureBlobFs) mkdirInternal(name string) error {
_, w, _, err := fs.Create(name, -1)
if err != nil {
@ -1104,19 +1148,6 @@ func (*AzureBlobFs) readFill(r io.Reader, buf []byte) (n int, err error) {
return n, err
}
func (fs *AzureBlobFs) preserveModificationTime(source, target string, fi os.FileInfo) {
if plugin.Handler.HasMetadater() {
if !fi.IsDir() {
err := plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(target),
util.GetTimeAsMsSinceEpoch(fi.ModTime()))
if err != nil {
fsLog(fs, logger.LevelWarn, "unable to preserve modification time after renaming %#v -> %#v: %+v",
source, target, err)
}
}
}
}
func (fs *AzureBlobFs) getCopyOptions() *blob.StartCopyFromURLOptions {
copyOptions := &blob.StartCopyFromURLOptions{}
if fs.config.AccessTier != "" {

270
internal/vfs/gcsfs.go
View file

@ -120,8 +120,7 @@ func (fs *GCSFs) Stat(name string) (os.FileInfo, error) {
if fs.config.KeyPrefix == name+"/" {
return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, 0, time.Unix(0, 0), false))
}
_, info, err := fs.getObjectStat(name)
return info, err
return fs.getObjectStat(name)
}
// Lstat returns a FileInfo describing the named file
@ -224,71 +223,15 @@ func (fs *GCSFs) Create(name string, flag int) (File, *PipeWriter, func(), error
}
// Rename renames (moves) source to target.
// We don't support renaming non empty directories since we should
// rename all the contents too and this could take long time: think
// about directories with thousands of files, for each file we should
// execute a CopyObject call.
func (fs *GCSFs) Rename(source, target string) error {
func (fs *GCSFs) Rename(source, target string) (int, int64, error) {
if source == target {
return nil
return -1, -1, nil
}
realSourceName, fi, err := fs.getObjectStat(source)
fi, err := fs.getObjectStat(source)
if err != nil {
return err
return -1, -1, err
}
if fi.IsDir() {
hasContents, err := fs.hasContents(source)
if err != nil {
return err
}
if hasContents {
return fmt.Errorf("cannot rename non empty directory: %#v", source)
}
if err := fs.mkdirInternal(target); err != nil {
return err
}
} else {
src := fs.svc.Bucket(fs.config.Bucket).Object(realSourceName)
dst := fs.svc.Bucket(fs.config.Bucket).Object(target)
attrs, statErr := fs.headObject(target)
if statErr == nil {
dst = dst.If(storage.Conditions{GenerationMatch: attrs.Generation})
} else if fs.IsNotExist(statErr) {
dst = dst.If(storage.Conditions{DoesNotExist: true})
} else {
fsLog(fs, logger.LevelWarn, "unable to set precondition for rename, target %q, stat err: %v",
target, statErr)
}
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout))
defer cancelFn()
copier := dst.CopierFrom(src)
if fs.config.StorageClass != "" {
copier.StorageClass = fs.config.StorageClass
}
if fs.config.ACL != "" {
copier.PredefinedACL = fs.config.ACL
}
contentType := mime.TypeByExtension(path.Ext(source))
if contentType != "" {
copier.ContentType = contentType
}
_, err = copier.Run(ctx)
metric.GCSCopyObjectCompleted(err)
if err != nil {
return err
}
if plugin.Handler.HasMetadater() {
err = plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(target),
util.GetTimeAsMsSinceEpoch(fi.ModTime()))
if err != nil {
fsLog(fs, logger.LevelWarn, "unable to preserve modification time after renaming %#v -> %#v: %+v",
source, target, err)
}
}
}
return fs.Remove(source, fi.IsDir())
return fs.renameInternal(source, target, fi)
}
// Remove removes the named file or (empty) directory.
@ -526,51 +469,7 @@ func (fs *GCSFs) CheckRootPath(username string, uid int, gid int) bool {
// ScanRootDirContents returns the number of files contained in the bucket,
// and their size
func (fs *GCSFs) ScanRootDirContents() (int, int64, error) {
numFiles := 0
size := int64(0)
query := &storage.Query{Prefix: fs.config.KeyPrefix}
err := query.SetAttrSelection(gcsDefaultFieldsSelection)
if err != nil {
return numFiles, size, err
}
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout))
defer cancelFn()
bkt := fs.svc.Bucket(fs.config.Bucket)
it := bkt.Objects(ctx, query)
pager := iterator.NewPager(it, defaultGCSPageSize, "")
for {
var objects []*storage.ObjectAttrs
pageToken, err := pager.NextPage(&objects)
if err != nil {
metric.GCSListObjectsCompleted(err)
return numFiles, size, err
}
for _, attrs := range objects {
if !attrs.Deleted.IsZero() {
continue
}
isDir := strings.HasSuffix(attrs.Name, "/") || attrs.ContentType == dirMimeType
if isDir && attrs.Size == 0 {
continue
}
numFiles++
size += attrs.Size
if numFiles%1000 == 0 {
fsLog(fs, logger.LevelDebug, "root dir scan in progress, files: %d, size: %d", numFiles, size)
}
}
objects = nil
if pageToken == "" {
break
}
}
metric.GCSListObjectsCompleted(nil)
return numFiles, size, err
return fs.GetDirSize(fs.config.KeyPrefix)
}
func (fs *GCSFs) getFileNamesInPrefix(fsPrefix string) (map[string]bool, error) {
@ -636,8 +535,54 @@ func (fs *GCSFs) CheckMetadata() error {
// GetDirSize returns the number of files and the size for a folder
// including any subfolders
func (*GCSFs) GetDirSize(dirname string) (int, int64, error) {
return 0, 0, ErrVfsUnsupported
func (fs *GCSFs) GetDirSize(dirname string) (int, int64, error) {
prefix := fs.getPrefix(dirname)
numFiles := 0
size := int64(0)
query := &storage.Query{Prefix: prefix}
err := query.SetAttrSelection(gcsDefaultFieldsSelection)
if err != nil {
return numFiles, size, err
}
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout))
defer cancelFn()
bkt := fs.svc.Bucket(fs.config.Bucket)
it := bkt.Objects(ctx, query)
pager := iterator.NewPager(it, defaultGCSPageSize, "")
for {
var objects []*storage.ObjectAttrs
pageToken, err := pager.NextPage(&objects)
if err != nil {
metric.GCSListObjectsCompleted(err)
return numFiles, size, err
}
for _, attrs := range objects {
if !attrs.Deleted.IsZero() {
continue
}
isDir := strings.HasSuffix(attrs.Name, "/") || attrs.ContentType == dirMimeType
if isDir && attrs.Size == 0 {
continue
}
numFiles++
size += attrs.Size
if numFiles%1000 == 0 {
fsLog(fs, logger.LevelDebug, "dirname %q scan in progress, files: %d, size: %d", dirname, numFiles, size)
}
}
objects = nil
if pageToken == "" {
break
}
}
metric.GCSListObjectsCompleted(nil)
return numFiles, size, err
}
// GetAtomicUploadPath returns the path to use for an atomic upload.
@ -755,35 +700,118 @@ func (fs *GCSFs) resolve(name, prefix, contentType string) (string, bool) {
}
// getObjectStat returns the stat result and the real object name as first value
func (fs *GCSFs) getObjectStat(name string) (string, os.FileInfo, error) {
func (fs *GCSFs) getObjectStat(name string) (os.FileInfo, error) {
attrs, err := fs.headObject(name)
var info os.FileInfo
if err == nil {
objSize := attrs.Size
objectModTime := attrs.Updated
isDir := attrs.ContentType == dirMimeType || strings.HasSuffix(attrs.Name, "/")
info, err = updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, isDir, objSize, objectModTime, false))
return name, info, err
return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, isDir, objSize, objectModTime, false))
}
if !fs.IsNotExist(err) {
return "", nil, err
return nil, err
}
// now check if this is a prefix (virtual directory)
hasContents, err := fs.hasContents(name)
if err != nil {
return "", nil, err
return nil, err
}
if hasContents {
info, err = updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, 0, time.Unix(0, 0), false))
return name, info, err
return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, 0, time.Unix(0, 0), false))
}
// finally check if this is an object with a trailing /
attrs, err = fs.headObject(name + "/")
if err != nil {
return "", nil, err
return nil, err
}
info, err = updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, attrs.Size, attrs.Updated, false))
return name + "/", info, err
return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, attrs.Size, attrs.Updated, false))
}
func (fs *GCSFs) copyFileInternal(source, target string) error {
src := fs.svc.Bucket(fs.config.Bucket).Object(source)
dst := fs.svc.Bucket(fs.config.Bucket).Object(target)
attrs, statErr := fs.headObject(target)
if statErr == nil {
dst = dst.If(storage.Conditions{GenerationMatch: attrs.Generation})
} else if fs.IsNotExist(statErr) {
dst = dst.If(storage.Conditions{DoesNotExist: true})
} else {
fsLog(fs, logger.LevelWarn, "unable to set precondition for rename, target %q, stat err: %v",
target, statErr)
}
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout))
defer cancelFn()
copier := dst.CopierFrom(src)
if fs.config.StorageClass != "" {
copier.StorageClass = fs.config.StorageClass
}
if fs.config.ACL != "" {
copier.PredefinedACL = fs.config.ACL
}
contentType := mime.TypeByExtension(path.Ext(source))
if contentType != "" {
copier.ContentType = contentType
}
_, err := copier.Run(ctx)
metric.GCSCopyObjectCompleted(err)
return err
}
func (fs *GCSFs) renameInternal(source, target string, fi os.FileInfo) (int, int64, error) {
var numFiles int
var filesSize int64
if fi.IsDir() {
if renameMode == 0 {
hasContents, err := fs.hasContents(source)
if err != nil {
return numFiles, filesSize, err
}
if hasContents {
return numFiles, filesSize, fmt.Errorf("cannot rename non empty directory: %q", source)
}
}
if err := fs.mkdirInternal(target); err != nil {
return numFiles, filesSize, err
}
if renameMode == 1 {
entries, err := fs.ReadDir(source)
if err != nil {
return numFiles, filesSize, err
}
for _, info := range entries {
sourceEntry := fs.Join(source, info.Name())
targetEntry := fs.Join(target, info.Name())
files, size, err := fs.renameInternal(sourceEntry, targetEntry, info)
if err != nil {
return numFiles, filesSize, err
}
numFiles += files
filesSize += size
}
}
} else {
if err := fs.copyFileInternal(source, target); err != nil {
return numFiles, filesSize, err
}
numFiles++
filesSize += fi.Size()
if plugin.Handler.HasMetadater() {
err := plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(target),
util.GetTimeAsMsSinceEpoch(fi.ModTime()))
if err != nil {
fsLog(fs, logger.LevelWarn, "unable to preserve modification time after renaming %q -> %q: %+v",
source, target, err)
}
}
}
err := fs.Remove(source, fi.IsDir())
if fs.IsNotExist(err) {
err = nil
}
return numFiles, filesSize, err
}
func (fs *GCSFs) mkdirInternal(name string) error {

8
internal/vfs/httpfs.go
View file

@ -368,9 +368,9 @@ func (fs *HTTPFs) Create(name string, flag int) (File, *PipeWriter, func(), erro
}
// Rename renames (moves) source to target.
func (fs *HTTPFs) Rename(source, target string) error {
func (fs *HTTPFs) Rename(source, target string) (int, int64, error) {
if source == target {
return nil
return -1, -1, nil
}
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
defer cancelFn()
@ -378,10 +378,10 @@ func (fs *HTTPFs) Rename(source, target string) error {
queryString := fmt.Sprintf("?target=%s", url.QueryEscape(target))
resp, err := fs.sendHTTPRequest(ctx, http.MethodPatch, "rename", source, queryString, "", nil)
if err != nil {
return err
return -1, -1, err
}
defer resp.Body.Close()
return nil
return -1, -1, nil
}
// Remove removes the named file or (empty) directory.

13
internal/vfs/osfs.go
View file

@ -116,13 +116,13 @@ func (*OsFs) Create(name string, flag int) (File, *PipeWriter, func(), error) {
}
// Rename renames (moves) source to target
func (fs *OsFs) Rename(source, target string) error {
func (fs *OsFs) Rename(source, target string) (int, int64, error) {
if source == target {
return nil
return -1, -1, nil
}
err := os.Rename(source, target)
if err != nil && isCrossDeviceError(err) {
fsLog(fs, logger.LevelError, "cross device error detected while renaming %#v -> %#v. Trying a copy and remove, this could take a long time",
fsLog(fs, logger.LevelError, "cross device error detected while renaming %q -> %q. Trying a copy and remove, this could take a long time",
source, target)
err = fscopy.Copy(source, target, fscopy.Options{
OnSymlink: func(src string) fscopy.SymlinkAction {
@ -131,11 +131,12 @@ func (fs *OsFs) Rename(source, target string) error {
})
if err != nil {
fsLog(fs, logger.LevelError, "cross device copy error: %v", err)
return err
return -1, -1, err
}
return os.RemoveAll(source)
err = os.RemoveAll(source)
return -1, -1, err
}
return err
return -1, -1, err
}
// Remove removes the named file or (empty) directory.

218
internal/vfs/s3fs.go
View file

@ -286,74 +286,15 @@ func (fs *S3Fs) Create(name string, flag int) (File, *PipeWriter, func(), error)
}
// Rename renames (moves) source to target.
// We don't support renaming non empty directories since we should
// rename all the contents too and this could take long time: think
// about directories with thousands of files, for each file we should
// execute a CopyObject call.
func (fs *S3Fs) Rename(source, target string) error {
func (fs *S3Fs) Rename(source, target string) (int, int64, error) {
if source == target {
return nil
return -1, -1, nil
}
fi, err := fs.Stat(source)
if err != nil {
return err
return -1, -1, err
}
if fi.IsDir() {
hasContents, err := fs.hasContents(source)
if err != nil {
return err
}
if hasContents {
return fmt.Errorf("cannot rename non empty directory: %q", source)
}
if err := fs.mkdirInternal(target); err != nil {
return err
}
} else {
contentType := mime.TypeByExtension(path.Ext(source))
copySource := pathEscape(fs.Join(fs.config.Bucket, source))
if fi.Size() > 500*1024*1024 {
fsLog(fs, logger.LevelDebug, "renaming file %q with size %d using multipart copy",
source, fi.Size())
err = fs.doMultipartCopy(copySource, target, contentType, fi.Size())
} else {
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
defer cancelFn()
_, err = fs.svc.CopyObject(ctx, &s3.CopyObjectInput{
Bucket: aws.String(fs.config.Bucket),
CopySource: aws.String(copySource),
Key: aws.String(target),
StorageClass: types.StorageClass(fs.config.StorageClass),
ACL: types.ObjectCannedACL(fs.config.ACL),
ContentType: util.NilIfEmpty(contentType),
})
}
if err != nil {
metric.S3CopyObjectCompleted(err)
return err
}
waiter := s3.NewObjectExistsWaiter(fs.svc)
err = waiter.Wait(context.Background(), &s3.HeadObjectInput{
Bucket: aws.String(fs.config.Bucket),
Key: aws.String(target),
}, 10*time.Second)
metric.S3CopyObjectCompleted(err)
if err != nil {
return err
}
if plugin.Handler.HasMetadater() {
err = plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(target),
util.GetTimeAsMsSinceEpoch(fi.ModTime()))
if err != nil {
fsLog(fs, logger.LevelWarn, "unable to preserve modification time after renaming %#v -> %#v: %+v",
source, target, err)
}
}
}
return fs.Remove(source, fi.IsDir())
return fs.renameInternal(source, target, fi)
}
// Remove removes the named file or (empty) directory.
@ -568,38 +509,7 @@ func (fs *S3Fs) CheckRootPath(username string, uid int, gid int) bool {
// ScanRootDirContents returns the number of files contained in the bucket,
// and their size
func (fs *S3Fs) ScanRootDirContents() (int, int64, error) {
numFiles := 0
size := int64(0)
paginator := s3.NewListObjectsV2Paginator(fs.svc, &s3.ListObjectsV2Input{
Bucket: aws.String(fs.config.Bucket),
Prefix: aws.String(fs.config.KeyPrefix),
})
for paginator.HasMorePages() {
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
defer cancelFn()
page, err := paginator.NextPage(ctx)
if err != nil {
metric.S3ListObjectsCompleted(err)
return numFiles, size, err
}
for _, fileObject := range page.Contents {
isDir := strings.HasSuffix(util.GetStringFromPointer(fileObject.Key), "/")
if isDir && fileObject.Size == 0 {
continue
}
numFiles++
size += fileObject.Size
if numFiles%1000 == 0 {
fsLog(fs, logger.LevelDebug, "root dir scan in progress, files: %d, size: %d", numFiles, size)
}
}
}
metric.S3ListObjectsCompleted(nil)
return numFiles, size, nil
return fs.GetDirSize(fs.config.KeyPrefix)
}
func (fs *S3Fs) getFileNamesInPrefix(fsPrefix string) (map[string]bool, error) {
@ -647,8 +557,40 @@ func (fs *S3Fs) CheckMetadata() error {
// GetDirSize returns the number of files and the size for a folder
// including any subfolders
func (*S3Fs) GetDirSize(dirname string) (int, int64, error) {
return 0, 0, ErrVfsUnsupported
func (fs *S3Fs) GetDirSize(dirname string) (int, int64, error) {
prefix := fs.getPrefix(dirname)
numFiles := 0
size := int64(0)
paginator := s3.NewListObjectsV2Paginator(fs.svc, &s3.ListObjectsV2Input{
Bucket: aws.String(fs.config.Bucket),
Prefix: aws.String(prefix),
})
for paginator.HasMorePages() {
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
defer cancelFn()
page, err := paginator.NextPage(ctx)
if err != nil {
metric.S3ListObjectsCompleted(err)
return numFiles, size, err
}
for _, fileObject := range page.Contents {
isDir := strings.HasSuffix(util.GetStringFromPointer(fileObject.Key), "/")
if isDir && fileObject.Size == 0 {
continue
}
numFiles++
size += fileObject.Size
if numFiles%1000 == 0 {
fsLog(fs, logger.LevelDebug, "dirname %q scan in progress, files: %d, size: %d", dirname, numFiles, size)
}
}
}
metric.S3ListObjectsCompleted(nil)
return numFiles, size, nil
}
// GetAtomicUploadPath returns the path to use for an atomic upload.
@ -770,6 +712,88 @@ func (fs *S3Fs) setConfigDefaults() {
}
}
func (fs *S3Fs) copyFileInternal(source, target string, fileSize int64) error {
contentType := mime.TypeByExtension(path.Ext(source))
copySource := pathEscape(fs.Join(fs.config.Bucket, source))
if fileSize > 500*1024*1024 {
fsLog(fs, logger.LevelDebug, "renaming file %q with size %d using multipart copy",
source, fileSize)
err := fs.doMultipartCopy(copySource, target, contentType, fileSize)
metric.S3CopyObjectCompleted(err)
return err
}
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
defer cancelFn()
_, err := fs.svc.CopyObject(ctx, &s3.CopyObjectInput{
Bucket: aws.String(fs.config.Bucket),
CopySource: aws.String(copySource),
Key: aws.String(target),
StorageClass: types.StorageClass(fs.config.StorageClass),
ACL: types.ObjectCannedACL(fs.config.ACL),
ContentType: util.NilIfEmpty(contentType),
})
metric.S3CopyObjectCompleted(err)
return err
}
func (fs *S3Fs) renameInternal(source, target string, fi os.FileInfo) (int, int64, error) {
var numFiles int
var filesSize int64
if fi.IsDir() {
if renameMode == 0 {
hasContents, err := fs.hasContents(source)
if err != nil {
return numFiles, filesSize, err
}
if hasContents {
return numFiles, filesSize, fmt.Errorf("cannot rename non empty directory: %q", source)
}
}
if err := fs.mkdirInternal(target); err != nil {
return numFiles, filesSize, err
}
if renameMode == 1 {
entries, err := fs.ReadDir(source)
if err != nil {
return numFiles, filesSize, err
}
for _, info := range entries {
sourceEntry := fs.Join(source, info.Name())
targetEntry := fs.Join(target, info.Name())
files, size, err := fs.renameInternal(sourceEntry, targetEntry, info)
if err != nil {
return numFiles, filesSize, err
}
numFiles += files
filesSize += size
}
}
} else {
if err := fs.copyFileInternal(source, target, fi.Size()); err != nil {
return numFiles, filesSize, err
}
numFiles++
filesSize += fi.Size()
if plugin.Handler.HasMetadater() {
err := plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(target),
util.GetTimeAsMsSinceEpoch(fi.ModTime()))
if err != nil {
fsLog(fs, logger.LevelWarn, "unable to preserve modification time after renaming %q -> %q: %+v",
source, target, err)
}
}
}
err := fs.Remove(source, fi.IsDir())
if fs.IsNotExist(err) {
err = nil
}
return numFiles, filesSize, err
}
func (fs *S3Fs) mkdirInternal(name string) error {
if !strings.HasSuffix(name, "/") {
name += "/"

12
internal/vfs/sftpfs.go
View file

@ -427,18 +427,20 @@ func (fs *SFTPFs) Create(name string, flag int) (File, *PipeWriter, func(), erro
}
// Rename renames (moves) source to target.
func (fs *SFTPFs) Rename(source, target string) error {
func (fs *SFTPFs) Rename(source, target string) (int, int64, error) {
if source == target {
return nil
return -1, -1, nil
}
client, err := fs.conn.getClient()
if err != nil {
return err
return -1, -1, err
}
if _, ok := client.HasExtension("posix-rename@openssh.com"); ok {
return client.PosixRename(source, target)
err := client.PosixRename(source, target)
return -1, -1, err
}
return client.Rename(source, target)
err = client.Rename(source, target)
return -1, -1, err
}
// Remove removes the named file or (empty) directory.

14
internal/vfs/vfs.go
View file

@ -54,6 +54,7 @@ var (
tempPath string
sftpFingerprints []string
allowSelfConnections int
renameMode int
)
// SetAllowSelfConnections sets the desired behaviour for self connections
@ -76,6 +77,11 @@ func SetSFTPFingerprints(fp []string) {
sftpFingerprints = fp
}
// SetRenameMode sets the rename mode
func SetRenameMode(val int) {
renameMode = val
}
// Fs defines the interface for filesystem backends
type Fs interface {
Name() string
@ -84,7 +90,7 @@ type Fs interface {
Lstat(name string) (os.FileInfo, error)
Open(name string, offset int64) (File, *pipeat.PipeReaderAt, func(), error)
Create(name string, flag int) (File, *PipeWriter, func(), error)
Rename(source, target string) error
Rename(source, target string) (int, int64, error)
Remove(name string, isDir bool) error
Mkdir(name string) error
Symlink(source, target string) error
@ -873,7 +879,7 @@ func fsMetadataCheck(fs fsMetadataChecker, storageID, keyPrefix string) error {
}
if keyPrefix != "" {
if !strings.HasPrefix(fsPrefix, "/"+keyPrefix) {
fsLog(fs, logger.LevelDebug, "skip metadata check for folder %#v outside prefix %#v",
fsLog(fs, logger.LevelDebug, "skip metadata check for folder %q outside prefix %q",
folder, keyPrefix)
continue
}
@ -898,9 +904,9 @@ func fsMetadataCheck(fs fsMetadataChecker, storageID, keyPrefix string) error {
if _, ok := fileNames[k]; !ok {
filePath := ensureAbsPath(path.Join(folder, k))
if err = plugin.Handler.RemoveMetadata(storageID, filePath); err != nil {
fsLog(fs, logger.LevelError, "unable to remove metadata for missing file %#v: %v", filePath, err)
fsLog(fs, logger.LevelError, "unable to remove metadata for missing file %q: %v", filePath, err)
} else {
fsLog(fs, logger.LevelDebug, "metadata removed for missing file %#v", filePath)
fsLog(fs, logger.LevelDebug, "metadata removed for missing file %q", filePath)
}
}
}

2
internal/webdavd/handler.go
View file

@ -254,7 +254,7 @@ func (c *Connection) handleUploadToExistingFile(fs vfs.Fs, resolvedPath, filePat
maxWriteSize, _ := c.GetMaxWriteSize(diskQuota, false, fileSize, fs.IsUploadResumeSupported())
if common.Config.IsAtomicUploadEnabled() && fs.IsAtomicUploadSupported() {
err = fs.Rename(resolvedPath, filePath)
_, _, err = fs.Rename(resolvedPath, filePath)
if err != nil {
c.Log(logger.LevelError, "error renaming existing file for atomic upload, source: %#v, dest: %#v, err: %+v",
resolvedPath, filePath, err)

5
internal/webdavd/internal_test.go
View file

@ -311,8 +311,9 @@ func (fs *MockOsFs) Remove(name string, isDir bool) error {
}
// Rename renames (moves) source to target
func (fs *MockOsFs) Rename(source, target string) error {
return os.Rename(source, target)
func (fs *MockOsFs) Rename(source, target string) (int, int64, error) {
err := os.Rename(source, target)
return -1, -1, err
}
// GetMimeType returns the content type

1
sftpgo.json
View file

@ -8,6 +8,7 @@
"hook": ""
},
"setstat_mode": 0,
"rename_mode": 0,
"temp_path": "",
"proxy_protocol": 0,
"proxy_allowed": [],

4
templates/webadmin/users.html
View file

@ -318,8 +318,8 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
if (type !== 'display') {
return data;
}
if (row[12] !== ""){
var formattedDate = dateFn(row[12], type);
if (row[13] !== ""){
var formattedDate = dateFn(row[13], type);
data = `${data}. Updated at: ${formattedDate}`;
}
let ellipsisFn = $.fn.dataTable.render.ellipsis(70, true);