diff --git a/docs/full-configuration.md b/docs/full-configuration.md index ef693242..4fffa5a7 100644 --- a/docs/full-configuration.md +++ b/docs/full-configuration.md @@ -66,6 +66,7 @@ The configuration file contains the following sections: - `execute_sync`, list of strings. Actions, defined in the `execute_on` list above, to be performed synchronously. The `pre-*` actions are always executed synchronously while the other ones are asynchronous. Executing an action synchronously means that SFTPGo will not return a result code to the client (which is waiting for it) until your hook have completed its execution. Leave empty to execute only the defined `pre-*` hook synchronously - `hook`, string. Absolute path to the command to execute or HTTP URL to notify. - `setstat_mode`, integer. 0 means "normal mode": requests for changing permissions, owner/group and access/modification times are executed. 1 means "ignore mode": requests for changing permissions, owner/group and access/modification times are silently ignored. 2 means "ignore mode if not supported": requests for changing permissions and owner/group are silently ignored for cloud filesystems and executed for local/SFTP filesystem. Requests for changing modification times are always executed for local/SFTP filesystems and are executed for cloud based filesystems if the target is a file and there is a metadata plugin available. A metadata plugin can be found [here](https://github.com/sftpgo/sftpgo-plugin-metadata). + - `rename_mode`, integer. By default (`0`), renaming of non-empty directories is not allowed for cloud storage providers (S3, GCS, Azure Blob). Set to `1` to enable recursive renames for these providers, they may be slow, there is no atomic rename API like for local filesystem, so SFTPGo will recursively list the directory contents and do a rename for each entry (partial renaming and incorrect disk quota updates are possible in error cases). Default `0`. - `temp_path`, string. Defines the path for temporary files such as those used for atomic uploads or file pipes. If you set this option you must make sure that the defined path exists, is accessible for writing by the user running SFTPGo, and is on the same filesystem as the users home directories otherwise the renaming for atomic uploads will become a copy and therefore may take a long time. The temporary files are not namespaced. The default is generally fine. Leave empty for the default. - `proxy_protocol`, integer. Support for [HAProxy PROXY protocol](https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt). If you are running SFTPGo behind a proxy server such as HAProxy, AWS ELB or NGINX, you can enable the proxy protocol. It provides a convenient way to safely transport connection information such as a client's address across multiple layers of NAT or TCP proxies to get the real client IP address instead of the proxy IP. Both protocol versions 1 and 2 are supported. If the proxy protocol is enabled in SFTPGo then you have to enable the protocol in your proxy configuration too. For example, for HAProxy, add `send-proxy` or `send-proxy-v2` to each server configuration line. The PROXY protocol is supported for SSH/SFTP and FTP/S. The following modes are supported: - 0, disabled diff --git a/go.mod b/go.mod index af66bfaa..2afd8287 100644 --- a/go.mod +++ b/go.mod @@ -9,22 +9,22 @@ require ( github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 github.com/alexedwards/argon2id v0.0.0-20211130144151-3585854a6387 github.com/aws/aws-sdk-go-v2 v1.17.3 - github.com/aws/aws-sdk-go-v2/config v1.18.7 - github.com/aws/aws-sdk-go-v2/credentials v1.13.7 + github.com/aws/aws-sdk-go-v2/config v1.18.8 + github.com/aws/aws-sdk-go-v2/credentials v1.13.8 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.46 - github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.26 - github.com/aws/aws-sdk-go-v2/service/s3 v1.29.6 - github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.17.0 - github.com/aws/aws-sdk-go-v2/service/sts v1.17.7 - github.com/bmatcuk/doublestar/v4 v4.4.0 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.47 + github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.14.0 + github.com/aws/aws-sdk-go-v2/service/s3 v1.30.0 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.18.0 + github.com/aws/aws-sdk-go-v2/service/sts v1.18.0 + github.com/bmatcuk/doublestar/v4 v4.6.0 github.com/cockroachdb/cockroach-go/v2 v2.2.20 - github.com/coreos/go-oidc/v3 v3.4.0 + github.com/coreos/go-oidc/v3 v3.5.0 github.com/drakkan/webdav v0.0.0-20221101181759-17ed21f9337b github.com/eikenb/pipeat v0.0.0-20210730190139-06b3e6902001 - github.com/fclairamb/ftpserverlib v0.20.1-0.20221012093027-95be4ae0c9a6 + github.com/fclairamb/ftpserverlib v0.20.1-0.20230104020606-0b1a04eec221 github.com/fclairamb/go-log v0.4.1 - github.com/go-acme/lego/v4 v4.9.1 + github.com/go-acme/lego/v4 v4.9.2-0.20230104103215-fd54758bba4c github.com/go-chi/chi/v5 v5.0.8 github.com/go-chi/jwtauth/v5 v5.1.0 github.com/go-chi/render v1.0.2 @@ -45,7 +45,7 @@ require ( github.com/minio/sio v0.3.0 github.com/otiai10/copy v1.9.0 github.com/pires/go-proxyproto v0.6.2 - github.com/pkg/sftp v1.13.6-0.20221020054726-e4133ab7e9bd + github.com/pkg/sftp v1.13.6-0.20230104082718-2489717da0f3 github.com/pquerna/otp v1.4.0 github.com/prometheus/client_golang v1.14.0 github.com/robfig/cron/v3 v3.0.1 @@ -67,21 +67,21 @@ require ( go.etcd.io/bbolt v1.3.6 go.uber.org/automaxprocs v1.5.1 gocloud.dev v0.27.0 - golang.org/x/crypto v0.4.0 - golang.org/x/net v0.4.0 - golang.org/x/oauth2 v0.3.0 - golang.org/x/sys v0.3.0 - golang.org/x/term v0.3.0 + golang.org/x/crypto v0.5.0 + golang.org/x/net v0.5.0 + golang.org/x/oauth2 v0.4.0 + golang.org/x/sys v0.4.0 + golang.org/x/term v0.4.0 golang.org/x/time v0.3.0 - google.golang.org/api v0.105.0 + google.golang.org/api v0.106.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) require ( cloud.google.com/go v0.107.0 // indirect - cloud.google.com/go/compute v1.14.0 // indirect + cloud.google.com/go/compute v1.15.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.9.0 // indirect + cloud.google.com/go/iam v0.10.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/ajg/form v1.5.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect @@ -93,8 +93,8 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.22 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.21 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.11.28 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.0 // indirect github.com/aws/smithy-go v1.13.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/boombuler/barcode v1.0.1 // indirect @@ -106,6 +106,7 @@ require ( github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect github.com/fatih/color v1.13.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-jose/go-jose/v3 v3.0.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-test/deep v1.1.0 // indirect github.com/goccy/go-json v0.10.0 // indirect @@ -156,22 +157,19 @@ require ( github.com/yusufpapurcu/wmi v1.2.2 // indirect go.opencensus.io v0.24.0 // indirect golang.org/x/mod v0.7.0 // indirect - golang.org/x/text v0.5.0 // indirect - golang.org/x/tools v0.4.0 // indirect + golang.org/x/text v0.6.0 // indirect + golang.org/x/tools v0.5.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect + google.golang.org/genproto v0.0.0-20230104163317-caabf589fcbf // indirect google.golang.org/grpc v1.51.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) replace ( - github.com/fclairamb/ftpserverlib => github.com/drakkan/ftpserverlib v0.0.0-20221203115213-ba73c775a9fd github.com/jlaffaye/ftp => github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9 - github.com/pkg/sftp => github.com/drakkan/sftp v0.0.0-20221225162142-08880975fb1e - golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20221223081523-be6917ff6f72 + golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20230106095953-5417b4dfde62 ) diff --git a/go.sum b/go.sum index 321176a6..75f4e3b2 100644 --- a/go.sum +++ b/go.sum @@ -50,8 +50,9 @@ cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6m cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= -cloud.google.com/go/compute v1.14.0 h1:hfm2+FfxVmnRlh6LpB7cg1ZNU+5edAHmW679JePztk0= -cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= +cloud.google.com/go/compute v1.15.0 h1:PiKE4V948A1BRvhuwA2hOxL8imyvwuRgrOiytC+NlXo= +cloud.google.com/go/compute v1.15.0/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= +cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= @@ -60,8 +61,8 @@ cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqCl cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= -cloud.google.com/go/iam v0.9.0 h1:bK6Or6mxhuL8lnj1i9j0yMo2wE/IeTO2cWlfUrf/TZs= -cloud.google.com/go/iam v0.9.0/go.mod h1:nXAECrMt2qHpF6RZUZseteD6QyanL68reN4OXPw0UWM= +cloud.google.com/go/iam v0.10.0 h1:fpP/gByFs6US1ma53v7VxhvbJpO2Aapng6wabJ99MuI= +cloud.google.com/go/iam v0.10.0/go.mod h1:nXAECrMt2qHpF6RZUZseteD6QyanL68reN4OXPw0UWM= cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= cloud.google.com/go/kms v1.6.0 h1:OWRZzrPmOZUzurjI2FBGtgY2mB1WaJkqhw6oIwSj0Yg= cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= @@ -233,17 +234,17 @@ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.3/go.mod h1:gNsR5CaXK github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= github.com/aws/aws-sdk-go-v2/config v1.15.15/go.mod h1:A1Lzyy/o21I5/s2FbyX5AevQfSVXpvvIDCoVFD0BC4E= -github.com/aws/aws-sdk-go-v2/config v1.18.7 h1:V94lTcix6jouwmAsgQMAEBozVAGJMFhVj+6/++xfe3E= -github.com/aws/aws-sdk-go-v2/config v1.18.7/go.mod h1:OZYsyHFL5PB9UpyS78NElgKs11qI/B5KJau2XOJDXHA= +github.com/aws/aws-sdk-go-v2/config v1.18.8 h1:lDpy0WM8AHsywOnVrOHaSMfpaiV2igOw8D7svkFkXVA= +github.com/aws/aws-sdk-go-v2/config v1.18.8/go.mod h1:5XCmmyutmzzgkpk/6NYTjeWb6lgo9N170m1j6pQkIBs= github.com/aws/aws-sdk-go-v2/credentials v1.12.10/go.mod h1:g5eIM5XRs/OzIIK81QMBl+dAuDyoLN0VYaLP+tBqEOk= -github.com/aws/aws-sdk-go-v2/credentials v1.13.7 h1:qUUcNS5Z1092XBFT66IJM7mYkMwgZ8fcC8YDIbEwXck= -github.com/aws/aws-sdk-go-v2/credentials v1.13.7/go.mod h1:AdCcbZXHQCjJh6NaH3pFaw8LUeBFn5+88BZGMVGuBT8= +github.com/aws/aws-sdk-go-v2/credentials v1.13.8 h1:vTrwTvv5qAwjWIGhZDSBH/oQHuIQjGmD232k01FUh6A= +github.com/aws/aws-sdk-go-v2/credentials v1.13.8/go.mod h1:lVa4OHbvgjVot4gmh1uouF1ubgexSCN92P6CJQpT0t8= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.9/go.mod h1:KDCCm4ONIdHtUloDcFvK2+vshZvx4Zmj7UMDfusuz5s= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 h1:j9wi1kQ8b+e0FBVHxCqCGo4kxDU175hoDHcWAi0sauU= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21/go.mod h1:ugwW57Z5Z48bpvUyZuaPy4Kv+vEfJWnIrky7RmkBvJg= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.21/go.mod h1:iIYPrQ2rYfZiB/iADYlhj9HHZ9TTi6PqKQPAqygohbE= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.46 h1:OCX1pQ4pcqhsDV7B92HzdLWjHWOQsILvjLinpaUWhcc= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.46/go.mod h1:MxCBOcyNXGJRvfpPiH+L6n/BF9zbowthGSUZdDvQF/c= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.47 h1:E884ndKWVGt8IhtUuGhXbEsmaCvdAAkTTUDu7uAok1g= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.47/go.mod h1:KybsEsmXLO0u75FyS3F0sY4OQ97syDe8z+ISq8oEczA= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.15/go.mod h1:pWrr2OoHlT7M/Pd2y4HV3gJyPb3qj5qMmnPkKSNPYK4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 h1:I3cakv2Uy1vNmmhRQmFptYDxOvBnwCdNwyw63N0RaRU= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27/go.mod h1:a1/UpzeyBBerajpnP5nGZa9mGzsBn5cOKxm6NWQsvoI= @@ -269,25 +270,25 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.9/go.mod h1:Rc5+wn2 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.21 h1:vY5siRXvW5TrOKm2qKEf9tliBfdLxdfy0i02LOcmqUo= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.21/go.mod h1:WZvNXT1XuH8dnJM0HvOlvk+RNn7NbAPvA/ACO0QarSc= github.com/aws/aws-sdk-go-v2/service/kms v1.18.1/go.mod h1:4PZMUkc9rXHWGVB5J9vKaZy3D7Nai79ORworQ3ASMiM= -github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.26 h1:NlA5om7Um+ohI/S0inBd55Vsp84BKhBrAgw2IVqEb30= -github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.26/go.mod h1:DSuypbY6jb7WZSxrLuCgd7ouB5uRQ+Hg5wbt0GmgRcc= +github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.14.0 h1:zhGJVqFAHNmnYFGfPXqUgG+yHkmlsDb5R56B6rCNuRw= +github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.14.0/go.mod h1:DSuypbY6jb7WZSxrLuCgd7ouB5uRQ+Hg5wbt0GmgRcc= github.com/aws/aws-sdk-go-v2/service/s3 v1.27.2/go.mod h1:u+566cosFI+d+motIz3USXEh6sN8Nq4GrNXSg2RXVMo= -github.com/aws/aws-sdk-go-v2/service/s3 v1.29.6 h1:W8pLcSn6Uy0eXgDBUUl8M8Kxv7JCoP68ZKTD04OXLEA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.29.6/go.mod h1:L2l2/q76teehcW7YEsgsDjqdsDTERJeX3nOMIFlgGUE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.30.0 h1:wddsyuESfviaiXk3w9N6/4iRwTg/a3gktjODY6jYQBo= +github.com/aws/aws-sdk-go-v2/service/s3 v1.30.0/go.mod h1:L2l2/q76teehcW7YEsgsDjqdsDTERJeX3nOMIFlgGUE= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.14/go.mod h1:xakbH8KMsQQKqzX87uyyzTHshc/0/Df8bsTneTS5pFU= -github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.17.0 h1:6W6BLZcXytRJsVvc2gGwxKE4wbMSlWqdxZivBP/E+ys= -github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.17.0/go.mod h1:jAeo/PdIJZuDSwsvxJS94G4d6h8tStj7WXVuKwLHWU8= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.18.0 h1:UQDiRZyaHQGPXIuCYqKsz/wIVZknCiZdRmPW8buD/xc= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.18.0/go.mod h1:jAeo/PdIJZuDSwsvxJS94G4d6h8tStj7WXVuKwLHWU8= github.com/aws/aws-sdk-go-v2/service/sns v1.17.10/go.mod h1:uITsRNVMeCB3MkWpXxXw0eDz8pW4TYLzj+eyQtbhSxM= github.com/aws/aws-sdk-go-v2/service/sqs v1.19.1/go.mod h1:A94o564Gj+Yn+7QO1eLFeI7UVv3riy/YBFOfICVqFvU= github.com/aws/aws-sdk-go-v2/service/ssm v1.27.6/go.mod h1:fiFzQgj4xNOg4/wqmAiPvzgDMXPD+cUEplX/CYn+0j0= github.com/aws/aws-sdk-go-v2/service/sso v1.11.13/go.mod h1:d7ptRksDDgvXaUvxyHZ9SYh+iMDymm94JbVcgvSYSzU= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.28 h1:gItLq3zBYyRDPmqAClgzTH8PBjDQGeyptYGHIwtYYNA= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.28/go.mod h1:wo/B7uUm/7zw/dWhBJ4FXuw1sySU5lyIhVg1Bu2yL9A= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11 h1:KCacyVSs/wlcPGx37hcbT3IGYO8P8Jx+TgSDhAXtQMY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11/go.mod h1:TZSH7xLO7+phDtViY/KUp9WGCJMQkLJ/VpgkTFd5gh8= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.0 h1:/2gzjhQowRLarkkBOGPXSRnb8sQ2RVsjdG1C/UliK/c= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.0/go.mod h1:wo/B7uUm/7zw/dWhBJ4FXuw1sySU5lyIhVg1Bu2yL9A= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.0 h1:Jfly6mRxk2ZOSlbCvZfKNS7TukSx1mIzhSsqZ/IGSZI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.0/go.mod h1:TZSH7xLO7+phDtViY/KUp9WGCJMQkLJ/VpgkTFd5gh8= github.com/aws/aws-sdk-go-v2/service/sts v1.16.10/go.mod h1:cftkHYN6tCDNfkSasAmclSfl4l7cySoay8vz7p/ce0E= -github.com/aws/aws-sdk-go-v2/service/sts v1.17.7 h1:9Mtq1KM6nD8/+HStvWcvYnixJ5N85DX+P+OY3kI3W2k= -github.com/aws/aws-sdk-go-v2/service/sts v1.17.7/go.mod h1:+lGbb3+1ugwKrNTWcf2RT05Xmp543B06zDFTwiTLp7I= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.0 h1:kOO++CYo50RcTFISESluhWEi5Prhg+gaSs4whWabiZU= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.0/go.mod h1:+lGbb3+1ugwKrNTWcf2RT05Xmp543B06zDFTwiTLp7I= github.com/aws/smithy-go v1.12.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= @@ -304,8 +305,8 @@ github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edY github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/bmatcuk/doublestar/v4 v4.4.0 h1:LmAwNwhjEbYtyVLzjcP/XeVw4nhuScHGkF/XWXnvIic= -github.com/bmatcuk/doublestar/v4 v4.4.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/bmatcuk/doublestar/v4 v4.6.0 h1:HTuxyug8GyFbRkrffIpzNCSK4luc0TY3wzXvzIZhEXc= +github.com/bmatcuk/doublestar/v4 v4.6.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1 h1:NDBbPmhS+EqABEs5Kg3n/5ZNjy73Pz7SIV+KCeqyXcs= @@ -472,8 +473,8 @@ github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmeka github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-oidc/v3 v3.4.0 h1:xz7elHb/LDwm/ERpwHd+5nb7wFHL32rsr6bBOgaeu6g= -github.com/coreos/go-oidc/v3 v3.4.0/go.mod h1:eHUXhZtXPQLgEaDrOVTgwbgmz1xGOkJNye6h3zkD2Pw= +github.com/coreos/go-oidc/v3 v3.5.0 h1:VxKtbccHZxs8juq7RdJntSqtXFtde9YpNpGn0yqgEHw= +github.com/coreos/go-oidc/v3 v3.5.0/go.mod h1:ecXRtV4romGPeO6ieExAsUK9cb/3fp9hXNz1tlv8PIM= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -540,14 +541,10 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/drakkan/crypto v0.0.0-20221223081523-be6917ff6f72 h1:Ivant8yrd81A5y3tQOS7vqwL9QaOdlGonHNOfRR3rsQ= -github.com/drakkan/crypto v0.0.0-20221223081523-be6917ff6f72/go.mod h1:cy6DFZ6nHFw1bTHZksT/gYKmdxPdzr7Rw7xcJFSayo4= +github.com/drakkan/crypto v0.0.0-20230106095953-5417b4dfde62 h1:1Bk+GbTbF1PBu0idZumIiT7dQ2dW8UeswfJGWbxO4D8= +github.com/drakkan/crypto v0.0.0-20230106095953-5417b4dfde62/go.mod h1:eekSq7nI5pP2ZldL4867reOp0VL9TOfTaZa0DydSYk4= github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9 h1:LPH1dEblAOO/LoG7yHPMtBLXhQmjaga91/DDjWk9jWA= github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU= -github.com/drakkan/ftpserverlib v0.0.0-20221203115213-ba73c775a9fd h1:wu/ys+33GwD9PyRO8QDCUpI2WBZtwFiDk8QkFPW8rhQ= -github.com/drakkan/ftpserverlib v0.0.0-20221203115213-ba73c775a9fd/go.mod h1:FHiqwx5L+7z3o7EXRtT6asSd1uO4yTqEljqFU9L+zVA= -github.com/drakkan/sftp v0.0.0-20221225162142-08880975fb1e h1:Eeg6op40DlnZOarl7OWX9t1wdjkhUHT2kPlSkSHOvLA= -github.com/drakkan/sftp v0.0.0-20221225162142-08880975fb1e/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= github.com/drakkan/webdav v0.0.0-20221101181759-17ed21f9337b h1:B9z7XyDoVxLO4yEvnXgdvZ+0Uw9NA1qdD4KTSGmKcoQ= github.com/drakkan/webdav v0.0.0-20221101181759-17ed21f9337b/go.mod h1:8opebuqUyBXrvl7Vo/S1Zzl9U0G1X2Ceud440eVuhUE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -584,6 +581,8 @@ github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fclairamb/ftpserverlib v0.20.1-0.20230104020606-0b1a04eec221 h1:oIEBdcX1yNS5F+rk0xaDXMkwu9cT6+YSBEih45Wptec= +github.com/fclairamb/ftpserverlib v0.20.1-0.20230104020606-0b1a04eec221/go.mod h1:2PS2QXGtruTtfUszbKGOuuWhDiK5u/GD9DK2DdAW+S8= github.com/fclairamb/go-log v0.4.1 h1:rLtdSG9x2pK41AIAnE8WYpl05xBJfw1ZyYxZaXFcBsM= github.com/fclairamb/go-log v0.4.1/go.mod h1:sw1KvnkZ4wKCYkvy4SL3qVZcJSWFP8Ure4pM3z+KNn4= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= @@ -611,8 +610,8 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= -github.com/go-acme/lego/v4 v4.9.1 h1:n9Z5MQwANeGSQKlVE3bEh9SDvAySK9oVYOKCGCESqQE= -github.com/go-acme/lego/v4 v4.9.1/go.mod h1:g3JRUyWS3L/VObpp4bCxzJftKyf/Wba8QrSSnoiqjg4= +github.com/go-acme/lego/v4 v4.9.2-0.20230104103215-fd54758bba4c h1:PDd4Q867Ia2D68T+KglkyxMDoIUEp3sNYVXuN3TXjAE= +github.com/go-acme/lego/v4 v4.9.2-0.20230104103215-fd54758bba4c/go.mod h1:qib35rauo2OW1BzAI0qUfR3xw/JIIuaO0ZA83QIsw0s= github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0= github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-chi/jwtauth/v5 v5.1.0 h1:wJyf2YZ/ohPvNJBwPOzZaQbyzwgMZZceE1m8FOzXLeA= @@ -623,6 +622,8 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo= +github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= @@ -1070,7 +1071,6 @@ github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.15.14 h1:i7WCKDToww0wA+9qrUZ1xOjp218vfFo3nTU6UHp+gOc= github.com/klauspost/compress v1.15.14/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.2/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/kolo/xmlrpc v0.0.0-20201022064351-38db28db192b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM= @@ -1342,6 +1342,10 @@ github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pkg/sftp v1.13.6-0.20230104082718-2489717da0f3 h1:eKBJ919kpjpfHltsNthMO6ZQ/XQy76cHHbuz2bOmMSA= +github.com/pkg/sftp v1.13.6-0.20230104082718-2489717da0f3/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= @@ -1818,9 +1822,10 @@ golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220802222814-0bcc04d9c69b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1846,9 +1851,9 @@ golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7Lm golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= golang.org/x/oauth2 v0.0.0-20220628200809-02e64fa58f26/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= -golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= -golang.org/x/oauth2 v0.3.0 h1:6l90koy8/LaBLmLu8jpHeHexzMwEita0zFfYlggy2F8= golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= +golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= +golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -2011,14 +2016,16 @@ golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2028,8 +2035,9 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2128,8 +2136,8 @@ golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.4.0 h1:7mTAgkunk3fr4GAloyyCasadO6h9zSsQZbwvcaIciV4= -golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= +golang.org/x/tools v0.5.0 h1:+bSpV5HIeWkuvgaMfI3UmKRThoTA5ODJTUd8T17NO+4= +golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2189,8 +2197,8 @@ google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6F google.golang.org/api v0.86.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= google.golang.org/api v0.91.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= -google.golang.org/api v0.105.0 h1:t6P9Jj+6XTn4U9I2wycQai6Q/Kz7iOT+QzjJ3G2V4x8= -google.golang.org/api v0.105.0/go.mod h1:qh7eD5FJks5+BcE+cjBIm6Gz8vioK7EHvnlniqXBnqI= +google.golang.org/api v0.106.0 h1:ffmW0faWCwKkpbbtvlY/K/8fUl+JKvNS5CVzRoyfCv8= +google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -2301,8 +2309,8 @@ google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljW google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220802133213-ce4fa296bf78/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= -google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef h1:uQ2vjV/sHTsWSqdKeLqmwitzgvjMl7o4IdtHwUDXSJY= -google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230104163317-caabf589fcbf h1:/JqRexUvugu6JURQ0O7RfV1EnvgrOxUV4tSjuAv0Sr0= +google.golang.org/genproto v0.0.0-20230104163317-caabf589fcbf/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -2389,8 +2397,6 @@ gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= -gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/telebot.v3 v3.0.0/go.mod h1:7rExV8/0mDDNu9epSrDm/8j22KLaActH1Tbee6YjzWg= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= diff --git a/internal/common/common.go b/internal/common/common.go index 1b804326..0714baa5 100644 --- a/internal/common/common.go +++ b/internal/common/common.go @@ -220,6 +220,7 @@ func Initialize(c Configuration, isShared int) error { vfs.SetTempPath(c.TempPath) dataprovider.SetTempPath(c.TempPath) vfs.SetAllowSelfConnections(c.AllowSelfConnections) + vfs.SetRenameMode(c.RenameMode) dataprovider.SetAllowSelfConnections(c.AllowSelfConnections) transfersChecker = getTransfersChecker(isShared) return nil @@ -529,6 +530,11 @@ type Configuration struct { // silently ignored for cloud based filesystem such as S3, GCS, Azure Blob. Requests for changing // modification times are ignored for cloud based filesystem if they are not supported. SetstatMode int `json:"setstat_mode" mapstructure:"setstat_mode"` + // RenameMode defines how to handle directory renames. By default, renaming of non-empty directories + // is not allowed for cloud storage providers (S3, GCS, Azure Blob). Set to 1 to enable recursive + // renames for these providers, they may be slow, there is no atomic rename API like for local + // filesystem, so SFTPGo will recursively list the directory contents and do a rename for each entry + RenameMode int `json:"rename_mode" mapstructure:"rename_mode"` // TempPath defines the path for temporary files such as those used for atomic uploads or file pipes. // If you set this option you must make sure that the defined path exists, is accessible for writing // by the user running SFTPGo, and is on the same filesystem as the users home directories otherwise diff --git a/internal/common/connection.go b/internal/common/connection.go index 01dd4dd4..60f96b14 100644 --- a/internal/common/connection.go +++ b/internal/common/connection.go @@ -470,7 +470,7 @@ func (c *BaseConnection) RemoveDir(virtualPath string) error { if fs.IsNotExist(err) && fs.HasVirtualFolders() { return nil } - c.Log(logger.LevelError, "failed to remove a dir %#v: stat error: %+v", fsPath, err) + c.Log(logger.LevelError, "failed to remove a dir %q: stat error: %+v", fsPath, err) return c.GetFsError(fs, err) } if !fi.IsDir() || fi.Mode()&os.ModeSymlink != 0 { @@ -671,6 +671,10 @@ func (c *BaseConnection) Copy(virtualSourcePath, virtualTargetPath string) error if err := c.CheckParentDirs(path.Dir(destPath)); err != nil { return err } + done := make(chan bool) + defer close(done) + go keepConnectionAlive(c, done, 2*time.Minute) + return c.doRecursiveCopy(virtualSourcePath, destPath, srcInfo, createTargetDir) } @@ -728,12 +732,17 @@ func (c *BaseConnection) renameInternal(virtualSourcePath, virtualTargetPath str if checkParentDestination { c.CheckParentDirs(path.Dir(virtualTargetPath)) //nolint:errcheck } - if err := fsDst.Rename(fsSourcePath, fsTargetPath); err != nil { + done := make(chan bool) + defer close(done) + go keepConnectionAlive(c, done, 2*time.Minute) + + files, size, err := fsDst.Rename(fsSourcePath, fsTargetPath) + if err != nil { c.Log(logger.LevelError, "failed to rename %q -> %q: %+v", fsSourcePath, fsTargetPath, err) return c.GetFsError(fsSrc, err) } vfs.SetPathPermissions(fsDst, fsTargetPath, c.User.GetUID(), c.User.GetGID()) - c.updateQuotaAfterRename(fsDst, virtualSourcePath, virtualTargetPath, fsTargetPath, initialSize) //nolint:errcheck + c.updateQuotaAfterRename(fsDst, virtualSourcePath, virtualTargetPath, fsTargetPath, initialSize, files, size) //nolint:errcheck logger.CommandLog(renameLogSender, fsSourcePath, fsTargetPath, c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "", -1, c.localAddr, c.remoteAddr) ExecuteActionNotification(c, operationRename, fsSourcePath, virtualSourcePath, fsTargetPath, //nolint:errcheck @@ -1008,7 +1017,7 @@ func (c *BaseConnection) checkRecursiveRenameDirPermissions(fsSrc, fsDst vfs.Fs, if !c.User.HasPermissionsInside(virtualSourcePath) && !c.User.HasPermissionsInside(virtualTargetPath) { if !c.isRenamePermitted(fsSrc, fsDst, sourcePath, targetPath, virtualSourcePath, virtualTargetPath, fi) { - c.Log(logger.LevelInfo, "rename %#v -> %#v is not allowed, virtual destination path: %#v", + c.Log(logger.LevelInfo, "rename %q -> %q is not allowed, virtual destination path: %q", sourcePath, targetPath, virtualTargetPath) return c.GetPermissionDeniedError() } @@ -1024,7 +1033,7 @@ func (c *BaseConnection) checkRecursiveRenameDirPermissions(fsSrc, fsDst vfs.Fs, if err != nil { return c.GetFsError(fsSrc, err) } - if walkedPath != sourcePath && vfs.HasImplicitAtomicUploads(fsSrc) { + if walkedPath != sourcePath && vfs.HasImplicitAtomicUploads(fsSrc) && Config.RenameMode == 0 { c.Log(logger.LevelInfo, "cannot rename non empty directory %q on this filesystem", virtualSourcePath) return c.GetOpUnsupportedError() } @@ -1083,6 +1092,11 @@ func (c *BaseConnection) checkFolderRename(fsSrc, fsDst vfs.Fs, fsSourcePath, fs virtualSourcePath) return c.GetOpUnsupportedError() } + if c.User.HasVirtualFoldersInside(virtualTargetPath) { + c.Log(logger.LevelDebug, "renaming the folder %q is not supported, the target %q has virtual folders inside it", + virtualSourcePath, virtualTargetPath) + return c.GetOpUnsupportedError() + } if err := c.checkRecursiveRenameDirPermissions(fsSrc, fsDst, fsSourcePath, fsTargetPath, virtualSourcePath, virtualTargetPath, fi); err != nil { c.Log(logger.LevelDebug, "error checking recursive permissions before renaming %q: %+v", fsSourcePath, err) @@ -1118,7 +1132,7 @@ func (c *BaseConnection) isRenamePermitted(fsSrc, fsDst vfs.Fs, fsSourcePath, fs isSrcAllowed, _ := c.User.IsFileAllowed(virtualSourcePath) isDstAllowed, _ := c.User.IsFileAllowed(virtualTargetPath) if !isSrcAllowed || !isDstAllowed { - c.Log(logger.LevelDebug, "renaming source: %#v to target: %#v not allowed", virtualSourcePath, + c.Log(logger.LevelDebug, "renaming source: %q to target: %q not allowed", virtualSourcePath, virtualTargetPath) return false } @@ -1147,7 +1161,15 @@ func (c *BaseConnection) hasSpaceForRename(fs vfs.Fs, virtualSourcePath, virtual // rename between user root dir and a virtual folder included in user quota return true } + if errDst != nil && sourceFolder.IsIncludedInUserQuota() { + // rename between a virtual folder included in user quota and the user root dir + return true + } quotaResult, _ := c.HasSpace(true, false, virtualTargetPath) + if quotaResult.HasSpace && quotaResult.QuotaSize == 0 && quotaResult.QuotaFiles == 0 { + // no quota restrictions + return true + } return c.hasSpaceForCrossRename(fs, quotaResult, initialSize, fsSourcePath) } @@ -1159,7 +1181,7 @@ func (c *BaseConnection) hasSpaceForCrossRename(fs vfs.Fs, quotaResult vfs.Quota } fi, err := fs.Lstat(sourcePath) if err != nil { - c.Log(logger.LevelError, "cross rename denied, stat error for path %#v: %v", sourcePath, err) + c.Log(logger.LevelError, "cross rename denied, stat error for path %q: %v", sourcePath, err) return false } var sizeDiff int64 @@ -1174,7 +1196,7 @@ func (c *BaseConnection) hasSpaceForCrossRename(fs vfs.Fs, quotaResult vfs.Quota } else if fi.IsDir() { filesDiff, sizeDiff, err = fs.GetDirSize(sourcePath) if err != nil { - c.Log(logger.LevelError, "cross rename denied, error getting size for directory %#v: %v", sourcePath, err) + c.Log(logger.LevelError, "cross rename denied, error getting size for directory %q: %v", sourcePath, err) return false } } @@ -1183,14 +1205,14 @@ func (c *BaseConnection) hasSpaceForCrossRename(fs vfs.Fs, quotaResult vfs.Quota if quotaResult.QuotaSize == 0 { return true } - c.Log(logger.LevelDebug, "cross rename overwrite, source %#v, used size %v, size to add %v", + c.Log(logger.LevelDebug, "cross rename overwrite, source %q, used size %d, size to add %d", sourcePath, quotaResult.UsedSize, sizeDiff) quotaResult.UsedSize += sizeDiff return quotaResult.GetRemainingSize() >= 0 } if quotaResult.QuotaFiles > 0 { remainingFiles := quotaResult.GetRemainingFiles() - c.Log(logger.LevelDebug, "cross rename, source %#v remaining file %v to add %v", sourcePath, + c.Log(logger.LevelDebug, "cross rename, source %q remaining file %d to add %d", sourcePath, remainingFiles, filesDiff) if remainingFiles < filesDiff { return false @@ -1198,7 +1220,7 @@ func (c *BaseConnection) hasSpaceForCrossRename(fs vfs.Fs, quotaResult vfs.Quota } if quotaResult.QuotaSize > 0 { remainingSize := quotaResult.GetRemainingSize() - c.Log(logger.LevelDebug, "cross rename, source %#v remaining size %v to add %v", sourcePath, + c.Log(logger.LevelDebug, "cross rename, source %q remaining size %d to add %d", sourcePath, remainingSize, sizeDiff) if remainingSize < sizeDiff { return false @@ -1329,7 +1351,7 @@ func (c *BaseConnection) HasSpace(checkFiles, getUsage bool, requestPath string) result.AllowedSize = result.QuotaSize - result.UsedSize if (checkFiles && result.QuotaFiles > 0 && result.UsedFiles >= result.QuotaFiles) || (result.QuotaSize > 0 && result.UsedSize >= result.QuotaSize) { - c.Log(logger.LevelDebug, "quota exceed for user %#v, request path %#v, num files: %v/%v, size: %v/%v check files: %v", + c.Log(logger.LevelDebug, "quota exceed for user %q, request path %q, num files: %d/%d, size: %d/%d check files: %t", c.User.Username, requestPath, result.UsedFiles, result.QuotaFiles, result.UsedSize, result.QuotaSize, checkFiles) result.HasSpace = false return result, transferQuota @@ -1430,7 +1452,9 @@ func (c *BaseConnection) updateQuotaMoveToVFolder(dstFolder *vfs.VirtualFolder, } } -func (c *BaseConnection) updateQuotaAfterRename(fs vfs.Fs, virtualSourcePath, virtualTargetPath, targetPath string, initialSize int64) error { +func (c *BaseConnection) updateQuotaAfterRename(fs vfs.Fs, virtualSourcePath, virtualTargetPath, targetPath string, + initialSize int64, numFiles int, filesSize int64, +) error { if dataprovider.GetQuotaTracking() == 0 { return nil } @@ -1451,22 +1475,27 @@ func (c *BaseConnection) updateQuotaAfterRename(fs vfs.Fs, virtualSourcePath, vi return nil } - filesSize := int64(0) - numFiles := 1 - if fi, err := fs.Stat(targetPath); err == nil { - if fi.Mode().IsDir() { - numFiles, filesSize, err = fs.GetDirSize(targetPath) - if err != nil { - c.Log(logger.LevelError, "failed to update quota after rename, error scanning moved folder %#v: %v", - targetPath, err) - return err + if filesSize == -1 { + // fs.Rename didn't return the affected files/sizes, we need to calculate them + numFiles = 1 + if fi, err := fs.Stat(targetPath); err == nil { + if fi.Mode().IsDir() { + numFiles, filesSize, err = fs.GetDirSize(targetPath) + if err != nil { + c.Log(logger.LevelError, "failed to update quota after rename, error scanning moved folder %q: %+v", + targetPath, err) + return err + } + } else { + filesSize = fi.Size() } } else { - filesSize = fi.Size() + c.Log(logger.LevelError, "failed to update quota after renaming, file %q stat error: %+v", targetPath, err) + return err } + c.Log(logger.LevelDebug, "calculated renamed files: %d, size: %d bytes", numFiles, filesSize) } else { - c.Log(logger.LevelError, "failed to update quota after rename, file %#v stat error: %+v", targetPath, err) - return err + c.Log(logger.LevelDebug, "returned renamed files: %d, size: %d bytes", numFiles, filesSize) } if errSrc == nil && errDst == nil { c.updateQuotaMoveBetweenVFolders(&sourceFolder, &dstFolder, initialSize, filesSize, numFiles) @@ -1660,3 +1689,19 @@ func (c *BaseConnection) GetFsAndResolvedPath(virtualPath string) (vfs.Fs, strin return fs, fsPath, nil } + +func keepConnectionAlive(c *BaseConnection, done chan bool, interval time.Duration) { + ticker := time.NewTicker(interval) + defer func() { + ticker.Stop() + }() + + for { + select { + case <-done: + return + case <-ticker.C: + c.UpdateLastActivity() + } + } +} diff --git a/internal/common/connection_test.go b/internal/common/connection_test.go index a42562dc..e183c7f7 100644 --- a/internal/common/connection_test.go +++ b/internal/common/connection_test.go @@ -280,6 +280,13 @@ func TestRenamePerms(t *testing.T) { func TestRenameNestedFolders(t *testing.T) { u := dataprovider.User{} + u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{ + BaseVirtualFolder: vfs.BaseVirtualFolder{ + Name: "vfolder", + MappedPath: filepath.Join(os.TempDir(), "f"), + }, + VirtualPath: "/vdirs/f", + }) conn := NewBaseConnection("", ProtocolSFTP, "", "", u) err := conn.checkFolderRename(nil, nil, filepath.Clean(os.TempDir()), filepath.Join(os.TempDir(), "subdir"), "/src", "/dst", nil) assert.Error(t, err) @@ -287,6 +294,8 @@ func TestRenameNestedFolders(t *testing.T) { assert.Error(t, err) err = conn.checkFolderRename(nil, nil, "", "", "/src/sub", "/src", nil) assert.Error(t, err) + err = conn.checkFolderRename(nil, nil, filepath.Join(os.TempDir(), "src"), filepath.Join(os.TempDir(), "vdirs"), "/src", "/vdirs", nil) + assert.Error(t, err) } func TestUpdateQuotaAfterRename(t *testing.T) { @@ -331,7 +340,7 @@ func TestUpdateQuotaAfterRename(t *testing.T) { assert.NoError(t, err) err = os.Chmod(testDirPath, 0001) assert.NoError(t, err) - err = c.updateQuotaAfterRename(fs, request.Filepath, request.Target, testDirPath, 0) + err = c.updateQuotaAfterRename(fs, request.Filepath, request.Target, testDirPath, 0, -1, -1) assert.Error(t, err) err = os.Chmod(testDirPath, os.ModePerm) assert.NoError(t, err) @@ -339,23 +348,25 @@ func TestUpdateQuotaAfterRename(t *testing.T) { testFile1 := "/testfile1" request.Target = testFile1 request.Filepath = path.Join("/vdir", "file") - err = c.updateQuotaAfterRename(fs, request.Filepath, request.Target, filepath.Join(mappedPath, "file"), 0) + err = c.updateQuotaAfterRename(fs, request.Filepath, request.Target, filepath.Join(mappedPath, "file"), 0, -1, -1) assert.Error(t, err) err = os.WriteFile(filepath.Join(mappedPath, "file"), []byte("test content"), os.ModePerm) assert.NoError(t, err) request.Filepath = testFile1 request.Target = path.Join("/vdir", "file") - err = c.updateQuotaAfterRename(fs, request.Filepath, request.Target, filepath.Join(mappedPath, "file"), 12) + err = c.updateQuotaAfterRename(fs, request.Filepath, request.Target, filepath.Join(mappedPath, "file"), 12, -1, -1) assert.NoError(t, err) err = os.WriteFile(filepath.Join(user.GetHomeDir(), "testfile1"), []byte("test content"), os.ModePerm) assert.NoError(t, err) request.Target = testFile1 request.Filepath = path.Join("/vdir", "file") - err = c.updateQuotaAfterRename(fs, request.Filepath, request.Target, filepath.Join(mappedPath, "file"), 12) + err = c.updateQuotaAfterRename(fs, request.Filepath, request.Target, filepath.Join(mappedPath, "file"), 12, -1, -1) assert.NoError(t, err) request.Target = path.Join("/vdir1", "file") request.Filepath = path.Join("/vdir", "file") - err = c.updateQuotaAfterRename(fs, request.Filepath, request.Target, filepath.Join(mappedPath, "file"), 12) + err = c.updateQuotaAfterRename(fs, request.Filepath, request.Target, filepath.Join(mappedPath, "file"), 12, -1, -1) + assert.NoError(t, err) + err = c.updateQuotaAfterRename(fs, request.Filepath, request.Target, filepath.Join(mappedPath, "file"), 12, 1, 100) assert.NoError(t, err) err = os.RemoveAll(mappedPath) @@ -604,3 +615,15 @@ func TestErrorResolvePath(t *testing.T) { err = os.RemoveAll(filepath.Dir(sourceFile)) assert.NoError(t, err) } + +func TestConnectionKeepAlive(t *testing.T) { + conn := NewBaseConnection("", ProtocolWebDAV, "", "", dataprovider.User{}) + lastActivity := conn.GetLastActivity() + done := make(chan bool) + go func() { + time.Sleep(200 * time.Millisecond) + close(done) + }() + keepConnectionAlive(conn, done, 50*time.Millisecond) + assert.Greater(t, conn.GetLastActivity(), lastActivity) +} diff --git a/internal/common/transfer.go b/internal/common/transfer.go index ef207f07..5ea544ed 100644 --- a/internal/common/transfer.go +++ b/internal/common/transfer.go @@ -372,7 +372,7 @@ func (t *BaseTransfer) Close() error { t.File.Name(), err) } else if t.transferType == TransferUpload && t.effectiveFsPath != t.fsPath { if t.ErrTransfer == nil || Config.UploadMode == UploadModeAtomicWithResume { - err = t.Fs.Rename(t.effectiveFsPath, t.fsPath) + _, _, err = t.Fs.Rename(t.effectiveFsPath, t.fsPath) t.Connection.Log(logger.LevelDebug, "atomic upload completed, rename: %#v -> %#v, error: %v", t.effectiveFsPath, t.fsPath, err) // the file must be removed if it is uploaded to a path outside the home dir and cannot be renamed diff --git a/internal/config/config.go b/internal/config/config.go index 5caf8c5e..f4bf34ea 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -201,6 +201,7 @@ func Init() { Hook: "", }, SetstatMode: 0, + RenameMode: 0, TempPath: "", ProxyProtocol: 0, ProxyAllowed: []string{}, @@ -1922,6 +1923,7 @@ func setViperDefaults() { viper.SetDefault("common.actions.execute_sync", globalConf.Common.Actions.ExecuteSync) viper.SetDefault("common.actions.hook", globalConf.Common.Actions.Hook) viper.SetDefault("common.setstat_mode", globalConf.Common.SetstatMode) + viper.SetDefault("common.rename_mode", globalConf.Common.RenameMode) viper.SetDefault("common.temp_path", globalConf.Common.TempPath) viper.SetDefault("common.proxy_protocol", globalConf.Common.ProxyProtocol) viper.SetDefault("common.proxy_allowed", globalConf.Common.ProxyAllowed) diff --git a/internal/ftpd/handler.go b/internal/ftpd/handler.go index bf0c55f8..da4338bb 100644 --- a/internal/ftpd/handler.go +++ b/internal/ftpd/handler.go @@ -455,7 +455,7 @@ func (c *Connection) handleFTPUploadToExistingFile(fs vfs.Fs, flags int, resolve } if common.Config.IsAtomicUploadEnabled() && fs.IsAtomicUploadSupported() { - err = fs.Rename(resolvedPath, filePath) + _, _, err = fs.Rename(resolvedPath, filePath) if err != nil { c.Log(logger.LevelError, "error renaming existing file for atomic upload, source: %#v, dest: %#v, err: %+v", resolvedPath, filePath, err) diff --git a/internal/ftpd/internal_test.go b/internal/ftpd/internal_test.go index 936da39b..e96aeefd 100644 --- a/internal/ftpd/internal_test.go +++ b/internal/ftpd/internal_test.go @@ -387,11 +387,12 @@ func (fs MockOsFs) Remove(name string, isDir bool) error { } // Rename renames (moves) source to target -func (fs MockOsFs) Rename(source, target string) error { +func (fs MockOsFs) Rename(source, target string) (int, int64, error) { if fs.err != nil { - return fs.err + return -1, -1, fs.err } - return os.Rename(source, target) + err := os.Rename(source, target) + return -1, -1, err } func newMockOsFs(err, statErr error, atomicUpload bool, connectionID, rootDir string) vfs.Fs { diff --git a/internal/httpd/handler.go b/internal/httpd/handler.go index 63e3eb58..b9c26898 100644 --- a/internal/httpd/handler.go +++ b/internal/httpd/handler.go @@ -176,7 +176,7 @@ func (c *Connection) getFileWriter(name string) (io.WriteCloser, error) { } if common.Config.IsAtomicUploadEnabled() && fs.IsAtomicUploadSupported() { - err = fs.Rename(p, filePath) + _, _, err = fs.Rename(p, filePath) if err != nil { c.Log(logger.LevelError, "error renaming existing file for atomic upload, source: %#v, dest: %#v, err: %+v", p, filePath, err) diff --git a/internal/sftpd/handler.go b/internal/sftpd/handler.go index d5ce9129..4d915cbc 100644 --- a/internal/sftpd/handler.go +++ b/internal/sftpd/handler.go @@ -455,7 +455,7 @@ func (c *Connection) handleSFTPUploadToExistingFile(fs vfs.Fs, pflags sftp.FileO } if common.Config.IsAtomicUploadEnabled() && fs.IsAtomicUploadSupported() { - err = fs.Rename(resolvedPath, filePath) + _, _, err = fs.Rename(resolvedPath, filePath) if err != nil { c.Log(logger.LevelError, "error renaming existing file for atomic upload, source: %#v, dest: %#v, err: %+v", resolvedPath, filePath, err) diff --git a/internal/sftpd/internal_test.go b/internal/sftpd/internal_test.go index 3a8fdee0..8276c143 100644 --- a/internal/sftpd/internal_test.go +++ b/internal/sftpd/internal_test.go @@ -138,11 +138,12 @@ func (fs MockOsFs) Remove(name string, isDir bool) error { } // Rename renames (moves) source to target -func (fs MockOsFs) Rename(source, target string) error { +func (fs MockOsFs) Rename(source, target string) (int, int64, error) { if fs.err != nil { - return fs.err + return -1, -1, fs.err } - return os.Rename(source, target) + err := os.Rename(source, target) + return -1, -1, err } func newMockOsFs(err, statErr error, atomicUpload bool, connectionID, rootDir string) vfs.Fs { diff --git a/internal/sftpd/scp.go b/internal/sftpd/scp.go index 11e33751..cd241e73 100644 --- a/internal/sftpd/scp.go +++ b/internal/sftpd/scp.go @@ -335,7 +335,7 @@ func (c *scpCommand) handleUpload(uploadFilePath string, sizeToRead int64) error } if common.Config.IsAtomicUploadEnabled() && fs.IsAtomicUploadSupported() { - err = fs.Rename(p, filePath) + _, _, err = fs.Rename(p, filePath) if err != nil { c.connection.Log(logger.LevelError, "error renaming existing file for atomic upload, source: %#v, dest: %#v, err: %v", p, filePath, err) diff --git a/internal/sftpd/sftpd_test.go b/internal/sftpd/sftpd_test.go index ffde9482..a5d4e5d5 100644 --- a/internal/sftpd/sftpd_test.go +++ b/internal/sftpd/sftpd_test.go @@ -5166,7 +5166,7 @@ func TestVirtualFoldersQuotaLimit(t *testing.T) { err = client.Rename(path.Join(vdirPath1, testFileName+".rename"), path.Join(vdirPath2, testFileName)) assert.Error(t, err) err = client.Rename(path.Join(vdirPath1, testFileName+".rename"), testFileName) - assert.Error(t, err) + assert.NoError(t, err) } _, err = httpdtest.RemoveUser(user, http.StatusOK) assert.NoError(t, err) diff --git a/internal/vfs/azblobfs.go b/internal/vfs/azblobfs.go index 9262847a..02491f66 100644 --- a/internal/vfs/azblobfs.go +++ b/internal/vfs/azblobfs.go @@ -273,68 +273,15 @@ func (fs *AzureBlobFs) Create(name string, flag int) (File, *PipeWriter, func(), } // Rename renames (moves) source to target. -// We don't support renaming non empty directories since we should -// rename all the contents too and this could take long time: think -// about directories with thousands of files, for each file we should -// execute a StartCopyFromURL call. -func (fs *AzureBlobFs) Rename(source, target string) error { +func (fs *AzureBlobFs) Rename(source, target string) (int, int64, error) { if source == target { - return nil + return -1, -1, nil } fi, err := fs.Stat(source) if err != nil { - return err + return -1, -1, err } - if fi.IsDir() { - hasContents, err := fs.hasContents(source) - if err != nil { - return err - } - if hasContents { - return fmt.Errorf("cannot rename non empty directory: %#v", source) - } - if err := fs.mkdirInternal(target); err != nil { - return err - } - } else { - ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout)) - defer cancelFn() - - srcBlob := fs.containerClient.NewBlockBlobClient(source) - dstBlob := fs.containerClient.NewBlockBlobClient(target) - resp, err := dstBlob.StartCopyFromURL(ctx, srcBlob.URL(), fs.getCopyOptions()) - if err != nil { - metric.AZCopyObjectCompleted(err) - return err - } - copyStatus := blob.CopyStatusType(util.GetStringFromPointer((*string)(resp.CopyStatus))) - nErrors := 0 - for copyStatus == blob.CopyStatusTypePending { - // Poll until the copy is complete. - time.Sleep(500 * time.Millisecond) - resp, err := dstBlob.GetProperties(ctx, &blob.GetPropertiesOptions{}) - if err != nil { - // A GetProperties failure may be transient, so allow a couple - // of them before giving up. - nErrors++ - if ctx.Err() != nil || nErrors == 3 { - metric.AZCopyObjectCompleted(err) - return err - } - } else { - copyStatus = blob.CopyStatusType(util.GetStringFromPointer((*string)(resp.CopyStatus))) - } - } - if copyStatus != blob.CopyStatusTypeSuccess { - err := fmt.Errorf("copy failed with status: %s", copyStatus) - metric.AZCopyObjectCompleted(err) - return err - } - - metric.AZCopyObjectCompleted(nil) - fs.preserveModificationTime(source, target, fi) - } - return fs.Remove(source, fi.IsDir()) + return fs.renameInternal(source, target, fi) } // Remove removes the named file or (empty) directory. @@ -575,44 +522,7 @@ func (fs *AzureBlobFs) CheckRootPath(username string, uid int, gid int) bool { // ScanRootDirContents returns the number of files contained in the bucket, // and their size func (fs *AzureBlobFs) ScanRootDirContents() (int, int64, error) { - numFiles := 0 - size := int64(0) - - pager := fs.containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{ - Include: container.ListBlobsInclude{ - Metadata: true, - }, - Prefix: &fs.config.KeyPrefix, - }) - - for pager.More() { - ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) - defer cancelFn() - - resp, err := pager.NextPage(ctx) - if err != nil { - metric.AZListObjectsCompleted(err) - return numFiles, size, err - } - for _, blobItem := range resp.ListBlobsFlatSegmentResponse.Segment.BlobItems { - if blobItem.Properties != nil { - contentType := util.GetStringFromPointer(blobItem.Properties.ContentType) - isDir := checkDirectoryMarkers(contentType, blobItem.Metadata) - blobSize := util.GetIntFromPointer(blobItem.Properties.ContentLength) - if isDir && blobSize == 0 { - continue - } - numFiles++ - size += blobSize - if numFiles%1000 == 0 { - fsLog(fs, logger.LevelDebug, "root dir scan in progress, files: %d, size: %d", numFiles, size) - } - } - } - } - metric.AZListObjectsCompleted(nil) - - return numFiles, size, nil + return fs.GetDirSize(fs.config.KeyPrefix) } func (fs *AzureBlobFs) getFileNamesInPrefix(fsPrefix string) (map[string]bool, error) { @@ -663,8 +573,46 @@ func (fs *AzureBlobFs) CheckMetadata() error { // GetDirSize returns the number of files and the size for a folder // including any subfolders -func (*AzureBlobFs) GetDirSize(dirname string) (int, int64, error) { - return 0, 0, ErrVfsUnsupported +func (fs *AzureBlobFs) GetDirSize(dirname string) (int, int64, error) { + numFiles := 0 + size := int64(0) + prefix := fs.getPrefix(dirname) + + pager := fs.containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{ + Include: container.ListBlobsInclude{ + Metadata: true, + }, + Prefix: &prefix, + }) + + for pager.More() { + ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) + defer cancelFn() + + resp, err := pager.NextPage(ctx) + if err != nil { + metric.AZListObjectsCompleted(err) + return numFiles, size, err + } + for _, blobItem := range resp.ListBlobsFlatSegmentResponse.Segment.BlobItems { + if blobItem.Properties != nil { + contentType := util.GetStringFromPointer(blobItem.Properties.ContentType) + isDir := checkDirectoryMarkers(contentType, blobItem.Metadata) + blobSize := util.GetIntFromPointer(blobItem.Properties.ContentLength) + if isDir && blobSize == 0 { + continue + } + numFiles++ + size += blobSize + if numFiles%1000 == 0 { + fsLog(fs, logger.LevelDebug, "dirname %q scan in progress, files: %d, size: %d", dirname, numFiles, size) + } + } + } + } + metric.AZListObjectsCompleted(nil) + + return numFiles, size, nil } // GetAtomicUploadPath returns the path to use for an atomic upload. @@ -838,6 +786,102 @@ func (fs *AzureBlobFs) setConfigDefaults() { } } +func (fs *AzureBlobFs) copyFileInternal(source, target string) error { + ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout)) + defer cancelFn() + + srcBlob := fs.containerClient.NewBlockBlobClient(source) + dstBlob := fs.containerClient.NewBlockBlobClient(target) + resp, err := dstBlob.StartCopyFromURL(ctx, srcBlob.URL(), fs.getCopyOptions()) + if err != nil { + metric.AZCopyObjectCompleted(err) + return err + } + copyStatus := blob.CopyStatusType(util.GetStringFromPointer((*string)(resp.CopyStatus))) + nErrors := 0 + for copyStatus == blob.CopyStatusTypePending { + // Poll until the copy is complete. + time.Sleep(500 * time.Millisecond) + resp, err := dstBlob.GetProperties(ctx, &blob.GetPropertiesOptions{}) + if err != nil { + // A GetProperties failure may be transient, so allow a couple + // of them before giving up. + nErrors++ + if ctx.Err() != nil || nErrors == 3 { + metric.AZCopyObjectCompleted(err) + return err + } + } else { + copyStatus = blob.CopyStatusType(util.GetStringFromPointer((*string)(resp.CopyStatus))) + } + } + if copyStatus != blob.CopyStatusTypeSuccess { + err := fmt.Errorf("copy failed with status: %s", copyStatus) + metric.AZCopyObjectCompleted(err) + return err + } + + metric.AZCopyObjectCompleted(nil) + return nil +} + +func (fs *AzureBlobFs) renameInternal(source, target string, fi os.FileInfo) (int, int64, error) { + var numFiles int + var filesSize int64 + + if fi.IsDir() { + if renameMode == 0 { + hasContents, err := fs.hasContents(source) + if err != nil { + return numFiles, filesSize, err + } + if hasContents { + return numFiles, filesSize, fmt.Errorf("cannot rename non empty directory: %q", source) + } + } + if err := fs.mkdirInternal(target); err != nil { + return numFiles, filesSize, err + } + if renameMode == 1 { + entries, err := fs.ReadDir(source) + if err != nil { + return numFiles, filesSize, err + } + for _, info := range entries { + sourceEntry := fs.Join(source, info.Name()) + targetEntry := fs.Join(target, info.Name()) + files, size, err := fs.renameInternal(sourceEntry, targetEntry, info) + if err != nil { + return numFiles, filesSize, err + } + numFiles += files + filesSize += size + } + } + } else { + if err := fs.copyFileInternal(source, target); err != nil { + return numFiles, filesSize, err + } + numFiles++ + filesSize += fi.Size() + if plugin.Handler.HasMetadater() { + if !fi.IsDir() { + err := plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(target), + util.GetTimeAsMsSinceEpoch(fi.ModTime())) + if err != nil { + fsLog(fs, logger.LevelWarn, "unable to preserve modification time after renaming %q -> %q: %+v", + source, target, err) + } + } + } + } + err := fs.Remove(source, fi.IsDir()) + if fs.IsNotExist(err) { + err = nil + } + return numFiles, filesSize, err +} + func (fs *AzureBlobFs) mkdirInternal(name string) error { _, w, _, err := fs.Create(name, -1) if err != nil { @@ -1104,19 +1148,6 @@ func (*AzureBlobFs) readFill(r io.Reader, buf []byte) (n int, err error) { return n, err } -func (fs *AzureBlobFs) preserveModificationTime(source, target string, fi os.FileInfo) { - if plugin.Handler.HasMetadater() { - if !fi.IsDir() { - err := plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(target), - util.GetTimeAsMsSinceEpoch(fi.ModTime())) - if err != nil { - fsLog(fs, logger.LevelWarn, "unable to preserve modification time after renaming %#v -> %#v: %+v", - source, target, err) - } - } - } -} - func (fs *AzureBlobFs) getCopyOptions() *blob.StartCopyFromURLOptions { copyOptions := &blob.StartCopyFromURLOptions{} if fs.config.AccessTier != "" { diff --git a/internal/vfs/gcsfs.go b/internal/vfs/gcsfs.go index a0a1129e..7b5c39d1 100644 --- a/internal/vfs/gcsfs.go +++ b/internal/vfs/gcsfs.go @@ -120,8 +120,7 @@ func (fs *GCSFs) Stat(name string) (os.FileInfo, error) { if fs.config.KeyPrefix == name+"/" { return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, 0, time.Unix(0, 0), false)) } - _, info, err := fs.getObjectStat(name) - return info, err + return fs.getObjectStat(name) } // Lstat returns a FileInfo describing the named file @@ -224,71 +223,15 @@ func (fs *GCSFs) Create(name string, flag int) (File, *PipeWriter, func(), error } // Rename renames (moves) source to target. -// We don't support renaming non empty directories since we should -// rename all the contents too and this could take long time: think -// about directories with thousands of files, for each file we should -// execute a CopyObject call. -func (fs *GCSFs) Rename(source, target string) error { +func (fs *GCSFs) Rename(source, target string) (int, int64, error) { if source == target { - return nil + return -1, -1, nil } - realSourceName, fi, err := fs.getObjectStat(source) + fi, err := fs.getObjectStat(source) if err != nil { - return err + return -1, -1, err } - if fi.IsDir() { - hasContents, err := fs.hasContents(source) - if err != nil { - return err - } - if hasContents { - return fmt.Errorf("cannot rename non empty directory: %#v", source) - } - if err := fs.mkdirInternal(target); err != nil { - return err - } - } else { - src := fs.svc.Bucket(fs.config.Bucket).Object(realSourceName) - dst := fs.svc.Bucket(fs.config.Bucket).Object(target) - attrs, statErr := fs.headObject(target) - if statErr == nil { - dst = dst.If(storage.Conditions{GenerationMatch: attrs.Generation}) - } else if fs.IsNotExist(statErr) { - dst = dst.If(storage.Conditions{DoesNotExist: true}) - } else { - fsLog(fs, logger.LevelWarn, "unable to set precondition for rename, target %q, stat err: %v", - target, statErr) - } - - ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout)) - defer cancelFn() - - copier := dst.CopierFrom(src) - if fs.config.StorageClass != "" { - copier.StorageClass = fs.config.StorageClass - } - if fs.config.ACL != "" { - copier.PredefinedACL = fs.config.ACL - } - contentType := mime.TypeByExtension(path.Ext(source)) - if contentType != "" { - copier.ContentType = contentType - } - _, err = copier.Run(ctx) - metric.GCSCopyObjectCompleted(err) - if err != nil { - return err - } - if plugin.Handler.HasMetadater() { - err = plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(target), - util.GetTimeAsMsSinceEpoch(fi.ModTime())) - if err != nil { - fsLog(fs, logger.LevelWarn, "unable to preserve modification time after renaming %#v -> %#v: %+v", - source, target, err) - } - } - } - return fs.Remove(source, fi.IsDir()) + return fs.renameInternal(source, target, fi) } // Remove removes the named file or (empty) directory. @@ -526,51 +469,7 @@ func (fs *GCSFs) CheckRootPath(username string, uid int, gid int) bool { // ScanRootDirContents returns the number of files contained in the bucket, // and their size func (fs *GCSFs) ScanRootDirContents() (int, int64, error) { - numFiles := 0 - size := int64(0) - query := &storage.Query{Prefix: fs.config.KeyPrefix} - err := query.SetAttrSelection(gcsDefaultFieldsSelection) - if err != nil { - return numFiles, size, err - } - ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout)) - defer cancelFn() - - bkt := fs.svc.Bucket(fs.config.Bucket) - it := bkt.Objects(ctx, query) - pager := iterator.NewPager(it, defaultGCSPageSize, "") - - for { - var objects []*storage.ObjectAttrs - pageToken, err := pager.NextPage(&objects) - if err != nil { - metric.GCSListObjectsCompleted(err) - return numFiles, size, err - } - - for _, attrs := range objects { - if !attrs.Deleted.IsZero() { - continue - } - isDir := strings.HasSuffix(attrs.Name, "/") || attrs.ContentType == dirMimeType - if isDir && attrs.Size == 0 { - continue - } - numFiles++ - size += attrs.Size - if numFiles%1000 == 0 { - fsLog(fs, logger.LevelDebug, "root dir scan in progress, files: %d, size: %d", numFiles, size) - } - } - - objects = nil - if pageToken == "" { - break - } - } - - metric.GCSListObjectsCompleted(nil) - return numFiles, size, err + return fs.GetDirSize(fs.config.KeyPrefix) } func (fs *GCSFs) getFileNamesInPrefix(fsPrefix string) (map[string]bool, error) { @@ -636,8 +535,54 @@ func (fs *GCSFs) CheckMetadata() error { // GetDirSize returns the number of files and the size for a folder // including any subfolders -func (*GCSFs) GetDirSize(dirname string) (int, int64, error) { - return 0, 0, ErrVfsUnsupported +func (fs *GCSFs) GetDirSize(dirname string) (int, int64, error) { + prefix := fs.getPrefix(dirname) + numFiles := 0 + size := int64(0) + + query := &storage.Query{Prefix: prefix} + err := query.SetAttrSelection(gcsDefaultFieldsSelection) + if err != nil { + return numFiles, size, err + } + ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout)) + defer cancelFn() + + bkt := fs.svc.Bucket(fs.config.Bucket) + it := bkt.Objects(ctx, query) + pager := iterator.NewPager(it, defaultGCSPageSize, "") + + for { + var objects []*storage.ObjectAttrs + pageToken, err := pager.NextPage(&objects) + if err != nil { + metric.GCSListObjectsCompleted(err) + return numFiles, size, err + } + + for _, attrs := range objects { + if !attrs.Deleted.IsZero() { + continue + } + isDir := strings.HasSuffix(attrs.Name, "/") || attrs.ContentType == dirMimeType + if isDir && attrs.Size == 0 { + continue + } + numFiles++ + size += attrs.Size + if numFiles%1000 == 0 { + fsLog(fs, logger.LevelDebug, "dirname %q scan in progress, files: %d, size: %d", dirname, numFiles, size) + } + } + + objects = nil + if pageToken == "" { + break + } + } + + metric.GCSListObjectsCompleted(nil) + return numFiles, size, err } // GetAtomicUploadPath returns the path to use for an atomic upload. @@ -755,35 +700,118 @@ func (fs *GCSFs) resolve(name, prefix, contentType string) (string, bool) { } // getObjectStat returns the stat result and the real object name as first value -func (fs *GCSFs) getObjectStat(name string) (string, os.FileInfo, error) { +func (fs *GCSFs) getObjectStat(name string) (os.FileInfo, error) { attrs, err := fs.headObject(name) - var info os.FileInfo if err == nil { objSize := attrs.Size objectModTime := attrs.Updated isDir := attrs.ContentType == dirMimeType || strings.HasSuffix(attrs.Name, "/") - info, err = updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, isDir, objSize, objectModTime, false)) - return name, info, err + return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, isDir, objSize, objectModTime, false)) } if !fs.IsNotExist(err) { - return "", nil, err + return nil, err } // now check if this is a prefix (virtual directory) hasContents, err := fs.hasContents(name) if err != nil { - return "", nil, err + return nil, err } if hasContents { - info, err = updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, 0, time.Unix(0, 0), false)) - return name, info, err + return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, 0, time.Unix(0, 0), false)) } // finally check if this is an object with a trailing / attrs, err = fs.headObject(name + "/") if err != nil { - return "", nil, err + return nil, err } - info, err = updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, attrs.Size, attrs.Updated, false)) - return name + "/", info, err + return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, attrs.Size, attrs.Updated, false)) +} + +func (fs *GCSFs) copyFileInternal(source, target string) error { + src := fs.svc.Bucket(fs.config.Bucket).Object(source) + dst := fs.svc.Bucket(fs.config.Bucket).Object(target) + attrs, statErr := fs.headObject(target) + if statErr == nil { + dst = dst.If(storage.Conditions{GenerationMatch: attrs.Generation}) + } else if fs.IsNotExist(statErr) { + dst = dst.If(storage.Conditions{DoesNotExist: true}) + } else { + fsLog(fs, logger.LevelWarn, "unable to set precondition for rename, target %q, stat err: %v", + target, statErr) + } + + ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout)) + defer cancelFn() + + copier := dst.CopierFrom(src) + if fs.config.StorageClass != "" { + copier.StorageClass = fs.config.StorageClass + } + if fs.config.ACL != "" { + copier.PredefinedACL = fs.config.ACL + } + contentType := mime.TypeByExtension(path.Ext(source)) + if contentType != "" { + copier.ContentType = contentType + } + _, err := copier.Run(ctx) + metric.GCSCopyObjectCompleted(err) + return err +} + +func (fs *GCSFs) renameInternal(source, target string, fi os.FileInfo) (int, int64, error) { + var numFiles int + var filesSize int64 + + if fi.IsDir() { + if renameMode == 0 { + hasContents, err := fs.hasContents(source) + if err != nil { + return numFiles, filesSize, err + } + if hasContents { + return numFiles, filesSize, fmt.Errorf("cannot rename non empty directory: %q", source) + } + } + if err := fs.mkdirInternal(target); err != nil { + return numFiles, filesSize, err + } + if renameMode == 1 { + entries, err := fs.ReadDir(source) + if err != nil { + return numFiles, filesSize, err + } + for _, info := range entries { + sourceEntry := fs.Join(source, info.Name()) + targetEntry := fs.Join(target, info.Name()) + files, size, err := fs.renameInternal(sourceEntry, targetEntry, info) + if err != nil { + return numFiles, filesSize, err + } + numFiles += files + filesSize += size + } + } + } else { + if err := fs.copyFileInternal(source, target); err != nil { + return numFiles, filesSize, err + } + numFiles++ + filesSize += fi.Size() + if plugin.Handler.HasMetadater() { + err := plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(target), + util.GetTimeAsMsSinceEpoch(fi.ModTime())) + if err != nil { + fsLog(fs, logger.LevelWarn, "unable to preserve modification time after renaming %q -> %q: %+v", + source, target, err) + } + } + } + err := fs.Remove(source, fi.IsDir()) + if fs.IsNotExist(err) { + err = nil + } + return numFiles, filesSize, err } func (fs *GCSFs) mkdirInternal(name string) error { diff --git a/internal/vfs/httpfs.go b/internal/vfs/httpfs.go index ea287d06..4195b054 100644 --- a/internal/vfs/httpfs.go +++ b/internal/vfs/httpfs.go @@ -368,9 +368,9 @@ func (fs *HTTPFs) Create(name string, flag int) (File, *PipeWriter, func(), erro } // Rename renames (moves) source to target. -func (fs *HTTPFs) Rename(source, target string) error { +func (fs *HTTPFs) Rename(source, target string) (int, int64, error) { if source == target { - return nil + return -1, -1, nil } ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) defer cancelFn() @@ -378,10 +378,10 @@ func (fs *HTTPFs) Rename(source, target string) error { queryString := fmt.Sprintf("?target=%s", url.QueryEscape(target)) resp, err := fs.sendHTTPRequest(ctx, http.MethodPatch, "rename", source, queryString, "", nil) if err != nil { - return err + return -1, -1, err } defer resp.Body.Close() - return nil + return -1, -1, nil } // Remove removes the named file or (empty) directory. diff --git a/internal/vfs/osfs.go b/internal/vfs/osfs.go index bb1b8155..b536eee9 100644 --- a/internal/vfs/osfs.go +++ b/internal/vfs/osfs.go @@ -116,13 +116,13 @@ func (*OsFs) Create(name string, flag int) (File, *PipeWriter, func(), error) { } // Rename renames (moves) source to target -func (fs *OsFs) Rename(source, target string) error { +func (fs *OsFs) Rename(source, target string) (int, int64, error) { if source == target { - return nil + return -1, -1, nil } err := os.Rename(source, target) if err != nil && isCrossDeviceError(err) { - fsLog(fs, logger.LevelError, "cross device error detected while renaming %#v -> %#v. Trying a copy and remove, this could take a long time", + fsLog(fs, logger.LevelError, "cross device error detected while renaming %q -> %q. Trying a copy and remove, this could take a long time", source, target) err = fscopy.Copy(source, target, fscopy.Options{ OnSymlink: func(src string) fscopy.SymlinkAction { @@ -131,11 +131,12 @@ func (fs *OsFs) Rename(source, target string) error { }) if err != nil { fsLog(fs, logger.LevelError, "cross device copy error: %v", err) - return err + return -1, -1, err } - return os.RemoveAll(source) + err = os.RemoveAll(source) + return -1, -1, err } - return err + return -1, -1, err } // Remove removes the named file or (empty) directory. diff --git a/internal/vfs/s3fs.go b/internal/vfs/s3fs.go index 94103516..27326018 100644 --- a/internal/vfs/s3fs.go +++ b/internal/vfs/s3fs.go @@ -286,74 +286,15 @@ func (fs *S3Fs) Create(name string, flag int) (File, *PipeWriter, func(), error) } // Rename renames (moves) source to target. -// We don't support renaming non empty directories since we should -// rename all the contents too and this could take long time: think -// about directories with thousands of files, for each file we should -// execute a CopyObject call. -func (fs *S3Fs) Rename(source, target string) error { +func (fs *S3Fs) Rename(source, target string) (int, int64, error) { if source == target { - return nil + return -1, -1, nil } fi, err := fs.Stat(source) if err != nil { - return err + return -1, -1, err } - if fi.IsDir() { - hasContents, err := fs.hasContents(source) - if err != nil { - return err - } - if hasContents { - return fmt.Errorf("cannot rename non empty directory: %q", source) - } - if err := fs.mkdirInternal(target); err != nil { - return err - } - } else { - contentType := mime.TypeByExtension(path.Ext(source)) - copySource := pathEscape(fs.Join(fs.config.Bucket, source)) - - if fi.Size() > 500*1024*1024 { - fsLog(fs, logger.LevelDebug, "renaming file %q with size %d using multipart copy", - source, fi.Size()) - err = fs.doMultipartCopy(copySource, target, contentType, fi.Size()) - } else { - ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) - defer cancelFn() - - _, err = fs.svc.CopyObject(ctx, &s3.CopyObjectInput{ - Bucket: aws.String(fs.config.Bucket), - CopySource: aws.String(copySource), - Key: aws.String(target), - StorageClass: types.StorageClass(fs.config.StorageClass), - ACL: types.ObjectCannedACL(fs.config.ACL), - ContentType: util.NilIfEmpty(contentType), - }) - } - if err != nil { - metric.S3CopyObjectCompleted(err) - return err - } - - waiter := s3.NewObjectExistsWaiter(fs.svc) - err = waiter.Wait(context.Background(), &s3.HeadObjectInput{ - Bucket: aws.String(fs.config.Bucket), - Key: aws.String(target), - }, 10*time.Second) - metric.S3CopyObjectCompleted(err) - if err != nil { - return err - } - if plugin.Handler.HasMetadater() { - err = plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(target), - util.GetTimeAsMsSinceEpoch(fi.ModTime())) - if err != nil { - fsLog(fs, logger.LevelWarn, "unable to preserve modification time after renaming %#v -> %#v: %+v", - source, target, err) - } - } - } - return fs.Remove(source, fi.IsDir()) + return fs.renameInternal(source, target, fi) } // Remove removes the named file or (empty) directory. @@ -568,38 +509,7 @@ func (fs *S3Fs) CheckRootPath(username string, uid int, gid int) bool { // ScanRootDirContents returns the number of files contained in the bucket, // and their size func (fs *S3Fs) ScanRootDirContents() (int, int64, error) { - numFiles := 0 - size := int64(0) - - paginator := s3.NewListObjectsV2Paginator(fs.svc, &s3.ListObjectsV2Input{ - Bucket: aws.String(fs.config.Bucket), - Prefix: aws.String(fs.config.KeyPrefix), - }) - - for paginator.HasMorePages() { - ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) - defer cancelFn() - - page, err := paginator.NextPage(ctx) - if err != nil { - metric.S3ListObjectsCompleted(err) - return numFiles, size, err - } - for _, fileObject := range page.Contents { - isDir := strings.HasSuffix(util.GetStringFromPointer(fileObject.Key), "/") - if isDir && fileObject.Size == 0 { - continue - } - numFiles++ - size += fileObject.Size - if numFiles%1000 == 0 { - fsLog(fs, logger.LevelDebug, "root dir scan in progress, files: %d, size: %d", numFiles, size) - } - } - } - - metric.S3ListObjectsCompleted(nil) - return numFiles, size, nil + return fs.GetDirSize(fs.config.KeyPrefix) } func (fs *S3Fs) getFileNamesInPrefix(fsPrefix string) (map[string]bool, error) { @@ -647,8 +557,40 @@ func (fs *S3Fs) CheckMetadata() error { // GetDirSize returns the number of files and the size for a folder // including any subfolders -func (*S3Fs) GetDirSize(dirname string) (int, int64, error) { - return 0, 0, ErrVfsUnsupported +func (fs *S3Fs) GetDirSize(dirname string) (int, int64, error) { + prefix := fs.getPrefix(dirname) + numFiles := 0 + size := int64(0) + + paginator := s3.NewListObjectsV2Paginator(fs.svc, &s3.ListObjectsV2Input{ + Bucket: aws.String(fs.config.Bucket), + Prefix: aws.String(prefix), + }) + + for paginator.HasMorePages() { + ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) + defer cancelFn() + + page, err := paginator.NextPage(ctx) + if err != nil { + metric.S3ListObjectsCompleted(err) + return numFiles, size, err + } + for _, fileObject := range page.Contents { + isDir := strings.HasSuffix(util.GetStringFromPointer(fileObject.Key), "/") + if isDir && fileObject.Size == 0 { + continue + } + numFiles++ + size += fileObject.Size + if numFiles%1000 == 0 { + fsLog(fs, logger.LevelDebug, "dirname %q scan in progress, files: %d, size: %d", dirname, numFiles, size) + } + } + } + + metric.S3ListObjectsCompleted(nil) + return numFiles, size, nil } // GetAtomicUploadPath returns the path to use for an atomic upload. @@ -770,6 +712,88 @@ func (fs *S3Fs) setConfigDefaults() { } } +func (fs *S3Fs) copyFileInternal(source, target string, fileSize int64) error { + contentType := mime.TypeByExtension(path.Ext(source)) + copySource := pathEscape(fs.Join(fs.config.Bucket, source)) + + if fileSize > 500*1024*1024 { + fsLog(fs, logger.LevelDebug, "renaming file %q with size %d using multipart copy", + source, fileSize) + err := fs.doMultipartCopy(copySource, target, contentType, fileSize) + metric.S3CopyObjectCompleted(err) + return err + } + ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) + defer cancelFn() + + _, err := fs.svc.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: aws.String(fs.config.Bucket), + CopySource: aws.String(copySource), + Key: aws.String(target), + StorageClass: types.StorageClass(fs.config.StorageClass), + ACL: types.ObjectCannedACL(fs.config.ACL), + ContentType: util.NilIfEmpty(contentType), + }) + + metric.S3CopyObjectCompleted(err) + return err +} + +func (fs *S3Fs) renameInternal(source, target string, fi os.FileInfo) (int, int64, error) { + var numFiles int + var filesSize int64 + + if fi.IsDir() { + if renameMode == 0 { + hasContents, err := fs.hasContents(source) + if err != nil { + return numFiles, filesSize, err + } + if hasContents { + return numFiles, filesSize, fmt.Errorf("cannot rename non empty directory: %q", source) + } + } + if err := fs.mkdirInternal(target); err != nil { + return numFiles, filesSize, err + } + if renameMode == 1 { + entries, err := fs.ReadDir(source) + if err != nil { + return numFiles, filesSize, err + } + for _, info := range entries { + sourceEntry := fs.Join(source, info.Name()) + targetEntry := fs.Join(target, info.Name()) + files, size, err := fs.renameInternal(sourceEntry, targetEntry, info) + if err != nil { + return numFiles, filesSize, err + } + numFiles += files + filesSize += size + } + } + } else { + if err := fs.copyFileInternal(source, target, fi.Size()); err != nil { + return numFiles, filesSize, err + } + numFiles++ + filesSize += fi.Size() + if plugin.Handler.HasMetadater() { + err := plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(target), + util.GetTimeAsMsSinceEpoch(fi.ModTime())) + if err != nil { + fsLog(fs, logger.LevelWarn, "unable to preserve modification time after renaming %q -> %q: %+v", + source, target, err) + } + } + } + err := fs.Remove(source, fi.IsDir()) + if fs.IsNotExist(err) { + err = nil + } + return numFiles, filesSize, err +} + func (fs *S3Fs) mkdirInternal(name string) error { if !strings.HasSuffix(name, "/") { name += "/" diff --git a/internal/vfs/sftpfs.go b/internal/vfs/sftpfs.go index c3a595c5..36533927 100644 --- a/internal/vfs/sftpfs.go +++ b/internal/vfs/sftpfs.go @@ -427,18 +427,20 @@ func (fs *SFTPFs) Create(name string, flag int) (File, *PipeWriter, func(), erro } // Rename renames (moves) source to target. -func (fs *SFTPFs) Rename(source, target string) error { +func (fs *SFTPFs) Rename(source, target string) (int, int64, error) { if source == target { - return nil + return -1, -1, nil } client, err := fs.conn.getClient() if err != nil { - return err + return -1, -1, err } if _, ok := client.HasExtension("posix-rename@openssh.com"); ok { - return client.PosixRename(source, target) + err := client.PosixRename(source, target) + return -1, -1, err } - return client.Rename(source, target) + err = client.Rename(source, target) + return -1, -1, err } // Remove removes the named file or (empty) directory. diff --git a/internal/vfs/vfs.go b/internal/vfs/vfs.go index c1934b81..e80594be 100644 --- a/internal/vfs/vfs.go +++ b/internal/vfs/vfs.go @@ -54,6 +54,7 @@ var ( tempPath string sftpFingerprints []string allowSelfConnections int + renameMode int ) // SetAllowSelfConnections sets the desired behaviour for self connections @@ -76,6 +77,11 @@ func SetSFTPFingerprints(fp []string) { sftpFingerprints = fp } +// SetRenameMode sets the rename mode +func SetRenameMode(val int) { + renameMode = val +} + // Fs defines the interface for filesystem backends type Fs interface { Name() string @@ -84,7 +90,7 @@ type Fs interface { Lstat(name string) (os.FileInfo, error) Open(name string, offset int64) (File, *pipeat.PipeReaderAt, func(), error) Create(name string, flag int) (File, *PipeWriter, func(), error) - Rename(source, target string) error + Rename(source, target string) (int, int64, error) Remove(name string, isDir bool) error Mkdir(name string) error Symlink(source, target string) error @@ -873,7 +879,7 @@ func fsMetadataCheck(fs fsMetadataChecker, storageID, keyPrefix string) error { } if keyPrefix != "" { if !strings.HasPrefix(fsPrefix, "/"+keyPrefix) { - fsLog(fs, logger.LevelDebug, "skip metadata check for folder %#v outside prefix %#v", + fsLog(fs, logger.LevelDebug, "skip metadata check for folder %q outside prefix %q", folder, keyPrefix) continue } @@ -898,9 +904,9 @@ func fsMetadataCheck(fs fsMetadataChecker, storageID, keyPrefix string) error { if _, ok := fileNames[k]; !ok { filePath := ensureAbsPath(path.Join(folder, k)) if err = plugin.Handler.RemoveMetadata(storageID, filePath); err != nil { - fsLog(fs, logger.LevelError, "unable to remove metadata for missing file %#v: %v", filePath, err) + fsLog(fs, logger.LevelError, "unable to remove metadata for missing file %q: %v", filePath, err) } else { - fsLog(fs, logger.LevelDebug, "metadata removed for missing file %#v", filePath) + fsLog(fs, logger.LevelDebug, "metadata removed for missing file %q", filePath) } } } diff --git a/internal/webdavd/handler.go b/internal/webdavd/handler.go index a7e795f5..e1a0f4e4 100644 --- a/internal/webdavd/handler.go +++ b/internal/webdavd/handler.go @@ -254,7 +254,7 @@ func (c *Connection) handleUploadToExistingFile(fs vfs.Fs, resolvedPath, filePat maxWriteSize, _ := c.GetMaxWriteSize(diskQuota, false, fileSize, fs.IsUploadResumeSupported()) if common.Config.IsAtomicUploadEnabled() && fs.IsAtomicUploadSupported() { - err = fs.Rename(resolvedPath, filePath) + _, _, err = fs.Rename(resolvedPath, filePath) if err != nil { c.Log(logger.LevelError, "error renaming existing file for atomic upload, source: %#v, dest: %#v, err: %+v", resolvedPath, filePath, err) diff --git a/internal/webdavd/internal_test.go b/internal/webdavd/internal_test.go index b56a6d27..ef49401d 100644 --- a/internal/webdavd/internal_test.go +++ b/internal/webdavd/internal_test.go @@ -311,8 +311,9 @@ func (fs *MockOsFs) Remove(name string, isDir bool) error { } // Rename renames (moves) source to target -func (fs *MockOsFs) Rename(source, target string) error { - return os.Rename(source, target) +func (fs *MockOsFs) Rename(source, target string) (int, int64, error) { + err := os.Rename(source, target) + return -1, -1, err } // GetMimeType returns the content type diff --git a/sftpgo.json b/sftpgo.json index 165b3f3a..dde5f911 100644 --- a/sftpgo.json +++ b/sftpgo.json @@ -8,6 +8,7 @@ "hook": "" }, "setstat_mode": 0, + "rename_mode": 0, "temp_path": "", "proxy_protocol": 0, "proxy_allowed": [], diff --git a/templates/webadmin/users.html b/templates/webadmin/users.html index 08f92087..64b6aa08 100644 --- a/templates/webadmin/users.html +++ b/templates/webadmin/users.html @@ -318,8 +318,8 @@ along with this program. If not, see . if (type !== 'display') { return data; } - if (row[12] !== ""){ - var formattedDate = dateFn(row[12], type); + if (row[13] !== ""){ + var formattedDate = dateFn(row[13], type); data = `${data}. Updated at: ${formattedDate}`; } let ellipsisFn = $.fn.dataTable.render.ellipsis(70, true);