S3: add SSE customer key

Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
This commit is contained in:
Nicola Murino 2024-08-15 10:09:06 +02:00
parent d783ffc13f
commit 2fbf608895
No known key found for this signature in database
GPG key ID: 935D2952DEC4EECF
14 changed files with 264 additions and 75 deletions

22
go.mod
View file

@ -47,12 +47,12 @@ require (
github.com/pires/go-proxyproto v0.7.0
github.com/pkg/sftp v1.13.7-0.20240410063531-637088883317
github.com/pquerna/otp v1.4.0
github.com/prometheus/client_golang v1.19.1
github.com/prometheus/client_golang v1.20.0
github.com/robfig/cron/v3 v3.0.1
github.com/rs/cors v1.11.0
github.com/rs/xid v1.5.0
github.com/rs/zerolog v1.33.0
github.com/sftpgo/sdk v0.1.8
github.com/sftpgo/sdk v0.1.9-0.20240815080450-426add0ab063
github.com/shirou/gopsutil/v3 v3.24.5
github.com/spf13/afero v1.11.0
github.com/spf13/cobra v1.8.1
@ -66,20 +66,20 @@ require (
github.com/yl2chen/cidranger v1.0.3-0.20210928021809-d1cb2c52f37a
go.etcd.io/bbolt v1.3.10
go.uber.org/automaxprocs v1.5.3
gocloud.dev v0.38.0
gocloud.dev v0.39.0
golang.org/x/crypto v0.26.0
golang.org/x/net v0.28.0
golang.org/x/oauth2 v0.22.0
golang.org/x/sys v0.24.0
golang.org/x/term v0.23.0
golang.org/x/time v0.6.0
google.golang.org/api v0.191.0
google.golang.org/api v0.192.0
gopkg.in/natefinch/lumberjack.v2 v2.2.1
)
require (
cloud.google.com/go v0.115.0 // indirect
cloud.google.com/go/auth v0.8.0 // indirect
cloud.google.com/go v0.115.1 // indirect
cloud.google.com/go/auth v0.8.1 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
cloud.google.com/go/compute/metadata v0.5.0 // indirect
cloud.google.com/go/iam v1.1.13 // indirect
@ -97,7 +97,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect
github.com/aws/smithy-go v1.20.3 // indirect
github.com/aws/smithy-go v1.20.4 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/boombuler/barcode v1.0.2 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
@ -138,7 +138,7 @@ require (
github.com/magiconair/properties v1.8.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/miekg/dns v1.1.61 // indirect
github.com/miekg/dns v1.1.62 // indirect
github.com/mitchellh/go-testing-interface v1.14.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
@ -173,9 +173,9 @@ require (
golang.org/x/text v0.17.0 // indirect
golang.org/x/tools v0.24.0 // indirect
golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect
google.golang.org/genproto v0.0.0-20240812133136-8ffd90a71988 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240812133136-8ffd90a71988 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240812133136-8ffd90a71988 // indirect
google.golang.org/genproto v0.0.0-20240814211410-ddb44dafa142 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect
google.golang.org/grpc v1.65.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect

52
go.sum
View file

@ -1,18 +1,18 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14=
cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU=
cloud.google.com/go/auth v0.8.0 h1:y8jUJLl/Fg+qNBWxP/Hox2ezJvjkrPb952PC1p0G6A4=
cloud.google.com/go/auth v0.8.0/go.mod h1:qGVp/Y3kDRSDZ5gFD/XPUfYQ9xW1iI7q8RIRoCyBbJc=
cloud.google.com/go v0.115.1 h1:Jo0SM9cQnSkYfp44+v+NQXHpcHqlnRJk2qxh6yvxxxQ=
cloud.google.com/go v0.115.1/go.mod h1:DuujITeaufu3gL68/lOFIirVNJwQeyf5UXyi+Wbgknc=
cloud.google.com/go/auth v0.8.1 h1:QZW9FjC5lZzN864p13YxvAtGUlQ+KgRL+8Sg45Z6vxo=
cloud.google.com/go/auth v0.8.1/go.mod h1:qGVp/Y3kDRSDZ5gFD/XPUfYQ9xW1iI7q8RIRoCyBbJc=
cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY=
cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc=
cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY=
cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY=
cloud.google.com/go/iam v1.1.13 h1:7zWBXG9ERbMLrzQBRhFliAV+kjcRToDTgQT3CTwYyv4=
cloud.google.com/go/iam v1.1.13/go.mod h1:K8mY0uSXwEXS30KrnVb+j54LB/ntfZu1dr+4zFMNbus=
cloud.google.com/go/kms v1.18.4 h1:dYN3OCsQ6wJLLtOnI8DGUwQ5shMusXsWCCC+s09ATsk=
cloud.google.com/go/kms v1.18.4/go.mod h1:SG1bgQ3UWW6/KdPo9uuJnzELXY5YTTMJtDYvajiQ22g=
cloud.google.com/go/longrunning v0.5.11 h1:Havn1kGjz3whCfoD8dxMLP73Ph5w+ODyZB9RUsDxtGk=
cloud.google.com/go/longrunning v0.5.11/go.mod h1:rDn7//lmlfWV1Dx6IB4RatCPenTwwmqXuiP0/RgoEO4=
cloud.google.com/go/kms v1.18.5 h1:75LSlVs60hyHK3ubs2OHd4sE63OAMcM2BdSJc2bkuM4=
cloud.google.com/go/kms v1.18.5/go.mod h1:yXunGUGzabH8rjUPImp2ndHiGolHeWJJ0LODLedicIY=
cloud.google.com/go/longrunning v0.5.12 h1:5LqSIdERr71CqfUsFlJdBpOkBH8FBCFD7P1nTWy3TYE=
cloud.google.com/go/longrunning v0.5.12/go.mod h1:S5hMV8CDJ6r50t2ubVJSKQVv5u0rmik5//KgLO3k4lU=
cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs=
cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0=
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
@ -79,8 +79,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 h1:yiwVzJW2ZxZTurVbYWA7QOrA
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4/go.mod h1:0oxfLkpz3rQ/CHlx5hB7H69YUpFiI1tql6Q6Ne+1bCw=
github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 h1:ZsDKRLXGWHk8WdtyYMoGNO7bTudrvuKpDKgMVRlepGE=
github.com/aws/aws-sdk-go-v2/service/sts v1.30.3/go.mod h1:zwySh8fpFyXp9yOr/KVzxOl8SRqgf/IDw5aUt9UKFcQ=
github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE=
github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4=
github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I=
@ -284,8 +284,8 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mhale/smtpd v0.8.3 h1:8j8YNXajksoSLZja3HdwvYVZPuJSqAxFsib3adzRRt8=
github.com/mhale/smtpd v0.8.3/go.mod h1:MQl+y2hwIEQCXtNhe5+55n0GZOjSmeqORDIXbqUL3x4=
github.com/miekg/dns v1.1.61 h1:nLxbwF3XxhwVSm8g9Dghm9MHPaUZuqhPiGL+675ZmEs=
github.com/miekg/dns v1.1.61/go.mod h1:mnAarhS3nWaW+NVP2wTkYVIZyHNJ098SJZUki3eykwQ=
github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ=
github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ=
github.com/minio/sio v0.4.0 h1:u4SWVEm5lXSqU42ZWawV0D9I5AZ5YMmo2RXpEQ/kRhc=
github.com/minio/sio v0.4.0/go.mod h1:oBSjJeGbBdRMZZwna07sX9EFzZy+ywu5aofRiV1g79I=
github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU=
@ -316,8 +316,8 @@ github.com/pquerna/otp v1.4.0 h1:wZvl1TIVxKRThZIBiwOOHOGP/1+nZyWBil9Y2XNEDzg=
github.com/pquerna/otp v1.4.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg=
github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
github.com/prometheus/client_golang v1.20.0 h1:jBzTZ7B099Rg24tny+qngoynol8LtVYlA2bqx3vEloI=
github.com/prometheus/client_golang v1.20.0/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
@ -343,8 +343,8 @@ github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJ
github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY=
github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
github.com/sftpgo/sdk v0.1.8 h1:HAywJl9jZnigFGztA/CWLieOW+R+HH6js6o6/qYvuSY=
github.com/sftpgo/sdk v0.1.8/go.mod h1:Isl0IEzS/Muvh8Fr4X+NWFsOS/fZQHRD4oPQpoY7C4g=
github.com/sftpgo/sdk v0.1.9-0.20240815080450-426add0ab063 h1:r+XUT9mg/W97xiS6ZJ1BczLwTYiGKCRQ+Z69QZBnAZ8=
github.com/sftpgo/sdk v0.1.9-0.20240815080450-426add0ab063/go.mod h1:Isl0IEzS/Muvh8Fr4X+NWFsOS/fZQHRD4oPQpoY7C4g=
github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI=
github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk=
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
@ -416,8 +416,8 @@ go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8=
go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
gocloud.dev v0.38.0 h1:SpxfaOc/Fp4PeO8ui7wRcCZV0EgXZ+IWcVSLn6ZMSw0=
gocloud.dev v0.38.0/go.mod h1:3XjKvd2E5iVNu/xFImRzjN0d/fkNHe4s0RiKidpEUMQ=
gocloud.dev v0.39.0 h1:EYABYGhAalPUaMrbSKOr5lejxoxvXj99nE8XFtsDgds=
gocloud.dev v0.39.0/go.mod h1:drz+VyYNBvrMTW0KZiBAYEdl8lbNZx+OQ7oQvdrFmSQ=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI=
golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ=
@ -516,19 +516,19 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 h1:LLhsEBxRTBLuKlQxFBYUOU8xyFgXv6cOTp2HASDlsDk=
golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
google.golang.org/api v0.191.0 h1:cJcF09Z+4HAB2t5qTQM1ZtfL/PemsLFkcFG67qq2afk=
google.golang.org/api v0.191.0/go.mod h1:tD5dsFGxFza0hnQveGfVk9QQYKcfp+VzgRqyXFxE0+E=
google.golang.org/api v0.192.0 h1:PljqpNAfZaaSpS+TnANfnNAXKdzHM/B9bKhwRlo7JP0=
google.golang.org/api v0.192.0/go.mod h1:9VcphjvAxPKLmSxVSzPlSRXy/5ARMEw5bf58WoVXafQ=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20240812133136-8ffd90a71988 h1:CT2Thj5AuPV9phrYMtzX11k+XkzMGfRAet42PmoTATM=
google.golang.org/genproto v0.0.0-20240812133136-8ffd90a71988/go.mod h1:7uvplUBj4RjHAxIZ//98LzOvrQ04JBkaixRmCMI29hc=
google.golang.org/genproto/googleapis/api v0.0.0-20240812133136-8ffd90a71988 h1:+/tmTy5zAieooKIXfzDm9KiA3Bv6JBwriRN9LY+yayk=
google.golang.org/genproto/googleapis/api v0.0.0-20240812133136-8ffd90a71988/go.mod h1:4+X6GvPs+25wZKbQq9qyAXrwIRExv7w0Ea6MgZLZiDM=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240812133136-8ffd90a71988 h1:V71AcdLZr2p8dC9dbOIMCpqi4EmRl8wUwnJzXXLmbmc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240812133136-8ffd90a71988/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY=
google.golang.org/genproto v0.0.0-20240814211410-ddb44dafa142 h1:oLiyxGgE+rt22duwci1+TG7bg2/L1LQsXwfjPlmuJA0=
google.golang.org/genproto v0.0.0-20240814211410-ddb44dafa142/go.mod h1:G11eXq53iI5Q+kyNOmCvnzBaxEA2Q/Ik5Tj7nqBE8j4=
google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8=
google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 h1:e7S5W7MGGLaSu8j3YjdezkZ+m1/Nm0uRVRMEMGk26Xs=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=

View file

@ -278,6 +278,9 @@ func updateEncryptedSecrets(fsConfig *vfs.Filesystem, currentFsConfig *vfs.Files
if fsConfig.S3Config.AccessSecret.IsNotPlainAndNotEmpty() {
fsConfig.S3Config.AccessSecret = currentFsConfig.S3Config.AccessSecret
}
if fsConfig.S3Config.SSECustomerKey.IsNotPlainAndNotEmpty() {
fsConfig.S3Config.SSECustomerKey = currentFsConfig.S3Config.SSECustomerKey
}
case sdk.AzureBlobFilesystemProvider:
if fsConfig.AzBlobConfig.AccountKey.IsNotPlainAndNotEmpty() {
fsConfig.AzBlobConfig.AccountKey = currentFsConfig.AzBlobConfig.AccountKey

View file

@ -4934,6 +4934,12 @@ func TestUserRedactedPassword(t *testing.T) {
assert.Contains(t, err.Error(), "cannot save a user with a redacted secret")
}
u.FsConfig.S3Config.AccessSecret = kms.NewPlainSecret("secret")
u.FsConfig.S3Config.SSECustomerKey = kms.NewSecret(sdkkms.SecretStatusRedacted, "mysecretkey", "", "")
_, resp, err = httpdtest.AddUser(u, http.StatusBadRequest)
assert.NoError(t, err, string(resp))
assert.Contains(t, string(resp), "cannot save a user with a redacted secret")
u.FsConfig.S3Config.SSECustomerKey = kms.NewPlainSecret("key")
user, _, err := httpdtest.AddUser(u, http.StatusCreated)
assert.NoError(t, err)
@ -5653,6 +5659,7 @@ func TestUserS3Config(t *testing.T) {
user.FsConfig.S3Config.Bucket = "test" //nolint:goconst
user.FsConfig.S3Config.AccessKey = "Server-Access-Key"
user.FsConfig.S3Config.AccessSecret = kms.NewPlainSecret("Server-Access-Secret")
user.FsConfig.S3Config.SSECustomerKey = kms.NewPlainSecret("SSE-encryption-key")
user.FsConfig.S3Config.RoleARN = "myRoleARN"
user.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000"
user.FsConfig.S3Config.UploadPartSize = 8
@ -5686,6 +5693,10 @@ func TestUserS3Config(t *testing.T) {
assert.NotEmpty(t, user.FsConfig.S3Config.AccessSecret.GetPayload())
assert.Empty(t, user.FsConfig.S3Config.AccessSecret.GetAdditionalData())
assert.Empty(t, user.FsConfig.S3Config.AccessSecret.GetKey())
assert.Equal(t, sdkkms.SecretStatusSecretBox, user.FsConfig.S3Config.SSECustomerKey.GetStatus())
assert.NotEmpty(t, user.FsConfig.S3Config.SSECustomerKey.GetPayload())
assert.Empty(t, user.FsConfig.S3Config.SSECustomerKey.GetAdditionalData())
assert.Empty(t, user.FsConfig.S3Config.SSECustomerKey.GetKey())
assert.Equal(t, 60, user.FsConfig.S3Config.DownloadPartMaxTime)
assert.Equal(t, 40, user.FsConfig.S3Config.UploadPartMaxTime)
assert.True(t, user.FsConfig.S3Config.SkipTLSVerify)
@ -5710,13 +5721,14 @@ func TestUserS3Config(t *testing.T) {
user.ID = 0
user.CreatedAt = 0
user.VirtualFolders = nil
user.FsConfig.S3Config.SSECustomerKey = kms.NewEmptySecret()
secret := kms.NewSecret(sdkkms.SecretStatusSecretBox, "Server-Access-Secret", "", "")
user.FsConfig.S3Config.AccessSecret = secret
_, _, err = httpdtest.AddUser(user, http.StatusCreated)
assert.Error(t, err)
user.FsConfig.S3Config.AccessSecret.SetStatus(sdkkms.SecretStatusPlain)
user, _, err = httpdtest.AddUser(user, http.StatusCreated)
assert.NoError(t, err)
user, resp, err := httpdtest.AddUser(user, http.StatusCreated)
assert.NoError(t, err, string(resp))
initialSecretPayload := user.FsConfig.S3Config.AccessSecret.GetPayload()
assert.Equal(t, sdkkms.SecretStatusSecretBox, user.FsConfig.S3Config.AccessSecret.GetStatus())
assert.NotEmpty(t, initialSecretPayload)
@ -6093,6 +6105,7 @@ func TestUserHiddenFields(t *testing.T) {
u1.FsConfig.S3Config.Region = "us-east-1"
u1.FsConfig.S3Config.AccessKey = "S3-Access-Key"
u1.FsConfig.S3Config.AccessSecret = kms.NewPlainSecret("S3-Access-Secret")
u1.FsConfig.S3Config.SSECustomerKey = kms.NewPlainSecret("SSE-secret-key")
user1, _, err := httpdtest.AddUser(u1, http.StatusCreated)
assert.NoError(t, err)
@ -6165,6 +6178,10 @@ func TestUserHiddenFields(t *testing.T) {
assert.Empty(t, user1.FsConfig.S3Config.AccessSecret.GetAdditionalData())
assert.NotEmpty(t, user1.FsConfig.S3Config.AccessSecret.GetStatus())
assert.NotEmpty(t, user1.FsConfig.S3Config.AccessSecret.GetPayload())
assert.Empty(t, user1.FsConfig.S3Config.SSECustomerKey.GetKey())
assert.Empty(t, user1.FsConfig.S3Config.SSECustomerKey.GetAdditionalData())
assert.NotEmpty(t, user1.FsConfig.S3Config.SSECustomerKey.GetStatus())
assert.NotEmpty(t, user1.FsConfig.S3Config.SSECustomerKey.GetPayload())
user2, _, err = httpdtest.GetUserByUsername(user2.Username, http.StatusOK)
assert.NoError(t, err)
@ -6219,12 +6236,22 @@ func TestUserHiddenFields(t *testing.T) {
assert.NotEmpty(t, user1.FsConfig.S3Config.AccessSecret.GetAdditionalData())
assert.NotEmpty(t, user1.FsConfig.S3Config.AccessSecret.GetStatus())
assert.NotEmpty(t, user1.FsConfig.S3Config.AccessSecret.GetPayload())
assert.NotEmpty(t, user1.FsConfig.S3Config.SSECustomerKey.GetKey())
assert.NotEmpty(t, user1.FsConfig.S3Config.SSECustomerKey.GetAdditionalData())
assert.NotEmpty(t, user1.FsConfig.S3Config.SSECustomerKey.GetStatus())
assert.NotEmpty(t, user1.FsConfig.S3Config.SSECustomerKey.GetPayload())
err = user1.FsConfig.S3Config.AccessSecret.Decrypt()
assert.NoError(t, err)
err = user1.FsConfig.S3Config.SSECustomerKey.Decrypt()
assert.NoError(t, err)
assert.Equal(t, sdkkms.SecretStatusPlain, user1.FsConfig.S3Config.AccessSecret.GetStatus())
assert.Equal(t, u1.FsConfig.S3Config.AccessSecret.GetPayload(), user1.FsConfig.S3Config.AccessSecret.GetPayload())
assert.Empty(t, user1.FsConfig.S3Config.AccessSecret.GetKey())
assert.Empty(t, user1.FsConfig.S3Config.AccessSecret.GetAdditionalData())
assert.Equal(t, sdkkms.SecretStatusPlain, user1.FsConfig.S3Config.SSECustomerKey.GetStatus())
assert.Equal(t, u1.FsConfig.S3Config.SSECustomerKey.GetPayload(), user1.FsConfig.S3Config.SSECustomerKey.GetPayload())
assert.Empty(t, user1.FsConfig.S3Config.SSECustomerKey.GetKey())
assert.Empty(t, user1.FsConfig.S3Config.SSECustomerKey.GetAdditionalData())
user2, err = dataprovider.UserExists(user2.Username, "")
assert.NoError(t, err)
@ -22133,6 +22160,7 @@ func TestUserTemplateMock(t *testing.T) {
form.Set("s3_region", user.FsConfig.S3Config.Region)
form.Set("s3_access_key", "%username%")
form.Set("s3_access_secret", "%password%")
form.Set("s3_sse_customer_key", "%password%")
form.Set("s3_key_prefix", "base/%username%")
form.Set("max_upload_file_size", "0")
form.Set("default_shares_expiration", "0")
@ -22232,13 +22260,21 @@ func TestUserTemplateMock(t *testing.T) {
require.Equal(t, path.Join("base", user1.Username)+"/", user1.FsConfig.S3Config.KeyPrefix)
require.Equal(t, path.Join("base", user2.Username)+"/", user2.FsConfig.S3Config.KeyPrefix)
require.True(t, user1.FsConfig.S3Config.AccessSecret.IsEncrypted())
require.True(t, user1.FsConfig.S3Config.SSECustomerKey.IsEncrypted())
err = user1.FsConfig.S3Config.AccessSecret.Decrypt()
require.NoError(t, err)
err = user1.FsConfig.S3Config.SSECustomerKey.Decrypt()
require.NoError(t, err)
require.Equal(t, "password1", user1.FsConfig.S3Config.AccessSecret.GetPayload())
require.Equal(t, "password1", user1.FsConfig.S3Config.SSECustomerKey.GetPayload())
require.True(t, user2.FsConfig.S3Config.AccessSecret.IsEncrypted())
require.True(t, user2.FsConfig.S3Config.SSECustomerKey.IsEncrypted())
err = user2.FsConfig.S3Config.AccessSecret.Decrypt()
require.NoError(t, err)
err = user2.FsConfig.S3Config.SSECustomerKey.Decrypt()
require.NoError(t, err)
require.Equal(t, "password2", user2.FsConfig.S3Config.AccessSecret.GetPayload())
require.Equal(t, "password2", user2.FsConfig.S3Config.SSECustomerKey.GetPayload())
require.True(t, user1.Filters.Hooks.ExternalAuthDisabled)
require.True(t, user1.Filters.Hooks.CheckPasswordDisabled)
require.False(t, user1.Filters.Hooks.PreLoginDisabled)
@ -22484,6 +22520,7 @@ func TestFolderTemplateMock(t *testing.T) {
form.Set("s3_region", "us-east-1")
form.Set("s3_access_key", "%name%")
form.Set("s3_access_secret", "pwd%name%")
form.Set("s3_sse_customer_key", "key%name%")
form.Set("s3_key_prefix", "base/%name%")
b, contentType, _ = getMultipartFormData(form, "", "")
@ -22523,18 +22560,27 @@ func TestFolderTemplateMock(t *testing.T) {
require.NoError(t, err)
require.Equal(t, fmt.Sprintf("pwd%s", folder1), folder.FsConfig.S3Config.AccessSecret.GetPayload())
require.Equal(t, path.Join("base", folder1)+"/", folder.FsConfig.S3Config.KeyPrefix)
err = folder.FsConfig.S3Config.SSECustomerKey.Decrypt()
require.NoError(t, err)
require.Equal(t, fmt.Sprintf("key%s", folder1), folder.FsConfig.S3Config.SSECustomerKey.GetPayload())
case folder2:
require.Equal(t, folder2, folder.FsConfig.S3Config.AccessKey)
err = folder.FsConfig.S3Config.AccessSecret.Decrypt()
require.NoError(t, err)
require.Equal(t, "pwd"+folder2, folder.FsConfig.S3Config.AccessSecret.GetPayload())
require.Equal(t, "base/"+folder2+"/", folder.FsConfig.S3Config.KeyPrefix)
err = folder.FsConfig.S3Config.SSECustomerKey.Decrypt()
require.NoError(t, err)
require.Equal(t, "key"+folder2, folder.FsConfig.S3Config.SSECustomerKey.GetPayload())
default:
require.Equal(t, folder3, folder.FsConfig.S3Config.AccessKey)
err = folder.FsConfig.S3Config.AccessSecret.Decrypt()
require.NoError(t, err)
require.Equal(t, "pwd"+folder3, folder.FsConfig.S3Config.AccessSecret.GetPayload())
require.Equal(t, "base/"+folder3+"/", folder.FsConfig.S3Config.KeyPrefix)
err = folder.FsConfig.S3Config.SSECustomerKey.Decrypt()
require.NoError(t, err)
require.Equal(t, "key"+folder3, folder.FsConfig.S3Config.SSECustomerKey.GetPayload())
}
}
@ -22583,6 +22629,7 @@ func TestWebUserS3Mock(t *testing.T) {
user.FsConfig.S3Config.Region = "eu-west-1"
user.FsConfig.S3Config.AccessKey = "access-key"
user.FsConfig.S3Config.AccessSecret = kms.NewPlainSecret("access-secret")
user.FsConfig.S3Config.SSECustomerKey = kms.NewPlainSecret("enc-key")
user.FsConfig.S3Config.RoleARN = "arn:aws:iam::123456789012:user/Development/product_1234/*"
user.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/path?a=b"
user.FsConfig.S3Config.StorageClass = "Standard"
@ -22623,6 +22670,7 @@ func TestWebUserS3Mock(t *testing.T) {
form.Set("s3_region", user.FsConfig.S3Config.Region)
form.Set("s3_access_key", user.FsConfig.S3Config.AccessKey)
form.Set("s3_access_secret", user.FsConfig.S3Config.AccessSecret.GetPayload())
form.Set("s3_sse_customer_key", user.FsConfig.S3Config.SSECustomerKey.GetPayload())
form.Set("s3_role_arn", user.FsConfig.S3Config.RoleARN)
form.Set("s3_storage_class", user.FsConfig.S3Config.StorageClass)
form.Set("s3_acl", user.FsConfig.S3Config.ACL)
@ -22747,6 +22795,10 @@ func TestWebUserS3Mock(t *testing.T) {
assert.NotEmpty(t, updateUser.FsConfig.S3Config.AccessSecret.GetPayload())
assert.Empty(t, updateUser.FsConfig.S3Config.AccessSecret.GetKey())
assert.Empty(t, updateUser.FsConfig.S3Config.AccessSecret.GetAdditionalData())
assert.Equal(t, sdkkms.SecretStatusSecretBox, updateUser.FsConfig.S3Config.SSECustomerKey.GetStatus())
assert.NotEmpty(t, updateUser.FsConfig.S3Config.SSECustomerKey.GetPayload())
assert.Empty(t, updateUser.FsConfig.S3Config.SSECustomerKey.GetKey())
assert.Empty(t, updateUser.FsConfig.S3Config.SSECustomerKey.GetAdditionalData())
assert.Equal(t, user.Description, updateUser.Description)
assert.True(t, updateUser.Filters.Hooks.PreLoginDisabled)
assert.False(t, updateUser.Filters.Hooks.ExternalAuthDisabled)
@ -22756,6 +22808,7 @@ func TestWebUserS3Mock(t *testing.T) {
assert.Equal(t, 1, updateUser.Filters.FTPSecurity)
// now check that a redacted password is not saved
form.Set("s3_access_secret", redactedSecret)
form.Set("s3_sse_customer_key", redactedSecret)
b, contentType, _ = getMultipartFormData(form, "", "")
req, _ = http.NewRequest(http.MethodPost, path.Join(webUserPath, user.Username), &b)
setJWTCookieForReq(req, webToken)
@ -22774,10 +22827,15 @@ func TestWebUserS3Mock(t *testing.T) {
assert.Equal(t, updateUser.FsConfig.S3Config.AccessSecret.GetPayload(), lastUpdatedUser.FsConfig.S3Config.AccessSecret.GetPayload())
assert.Empty(t, lastUpdatedUser.FsConfig.S3Config.AccessSecret.GetKey())
assert.Empty(t, lastUpdatedUser.FsConfig.S3Config.AccessSecret.GetAdditionalData())
assert.Equal(t, sdkkms.SecretStatusSecretBox, lastUpdatedUser.FsConfig.S3Config.SSECustomerKey.GetStatus())
assert.Equal(t, updateUser.FsConfig.S3Config.SSECustomerKey.GetPayload(), lastUpdatedUser.FsConfig.S3Config.SSECustomerKey.GetPayload())
assert.Empty(t, lastUpdatedUser.FsConfig.S3Config.SSECustomerKey.GetKey())
assert.Empty(t, lastUpdatedUser.FsConfig.S3Config.SSECustomerKey.GetAdditionalData())
assert.Equal(t, lastPwdChange, lastUpdatedUser.LastPasswordChange)
// now clear credentials
form.Set("s3_access_key", "")
form.Set("s3_access_secret", "")
form.Set("s3_sse_customer_key", "")
b, contentType, _ = getMultipartFormData(form, "", "")
req, _ = http.NewRequest(http.MethodPost, path.Join(webUserPath, user.Username), &b)
setJWTCookieForReq(req, webToken)
@ -22792,6 +22850,7 @@ func TestWebUserS3Mock(t *testing.T) {
err = render.DecodeJSON(rr.Body, &userGet)
assert.NoError(t, err)
assert.Nil(t, userGet.FsConfig.S3Config.AccessSecret)
assert.Nil(t, userGet.FsConfig.S3Config.SSECustomerKey)
req, _ = http.NewRequest(http.MethodDelete, path.Join(userPath, user.Username), nil)
setBearerForReq(req, apiToken)
@ -25187,6 +25246,7 @@ func TestS3WebFolderMock(t *testing.T) {
S3Region := "eu-west-1"
S3AccessKey := "access-key"
S3AccessSecret := kms.NewPlainSecret("folder-access-secret")
S3SSEKey := kms.NewPlainSecret("folder-sse-key")
S3SessionToken := "fake session token"
S3RoleARN := "arn:aws:iam::123456789012:user/Development/product_1234/*"
S3Endpoint := "http://127.0.0.1:9000/path?b=c"
@ -25208,6 +25268,7 @@ func TestS3WebFolderMock(t *testing.T) {
form.Set("s3_region", S3Region)
form.Set("s3_access_key", S3AccessKey)
form.Set("s3_access_secret", S3AccessSecret.GetPayload())
form.Set("s3_sse_customer_key", S3SSEKey.GetPayload())
form.Set("s3_session_token", S3SessionToken)
form.Set("s3_role_arn", S3RoleARN)
form.Set("s3_storage_class", S3StorageClass)
@ -25255,6 +25316,7 @@ func TestS3WebFolderMock(t *testing.T) {
assert.Equal(t, S3Region, folder.FsConfig.S3Config.Region)
assert.Equal(t, S3AccessKey, folder.FsConfig.S3Config.AccessKey)
assert.NotEmpty(t, folder.FsConfig.S3Config.AccessSecret.GetPayload())
assert.NotEmpty(t, folder.FsConfig.S3Config.SSECustomerKey.GetPayload())
assert.Equal(t, S3Endpoint, folder.FsConfig.S3Config.Endpoint)
assert.Equal(t, S3StorageClass, folder.FsConfig.S3Config.StorageClass)
assert.Equal(t, S3ACL, folder.FsConfig.S3Config.ACL)
@ -25305,6 +25367,7 @@ func TestS3WebFolderMock(t *testing.T) {
assert.Equal(t, S3AccessKey, folder.FsConfig.S3Config.AccessKey)
assert.Equal(t, S3RoleARN, folder.FsConfig.S3Config.RoleARN)
assert.NotEmpty(t, folder.FsConfig.S3Config.AccessSecret.GetPayload())
assert.NotEmpty(t, folder.FsConfig.S3Config.SSECustomerKey.GetPayload())
assert.Equal(t, S3Endpoint, folder.FsConfig.S3Config.Endpoint)
assert.Equal(t, S3StorageClass, folder.FsConfig.S3Config.StorageClass)
assert.Equal(t, S3KeyPrefix, folder.FsConfig.S3Config.KeyPrefix)

View file

@ -1530,6 +1530,7 @@ func getS3Config(r *http.Request) (vfs.S3FsConfig, error) {
config.AccessKey = strings.TrimSpace(r.Form.Get("s3_access_key"))
config.RoleARN = strings.TrimSpace(r.Form.Get("s3_role_arn"))
config.AccessSecret = getSecretFromFormField(r, "s3_access_secret")
config.SSECustomerKey = getSecretFromFormField(r, "s3_sse_customer_key")
config.Endpoint = strings.TrimSpace(r.Form.Get("s3_endpoint"))
config.StorageClass = strings.TrimSpace(r.Form.Get("s3_storage_class"))
config.ACL = strings.TrimSpace(r.Form.Get("s3_acl"))
@ -1855,6 +1856,10 @@ func getS3FsFromTemplate(fsConfig vfs.S3FsConfig, replacements map[string]string
payload := replacePlaceholders(fsConfig.AccessSecret.GetPayload(), replacements)
fsConfig.AccessSecret = kms.NewPlainSecret(payload)
}
if fsConfig.SSECustomerKey != nil && fsConfig.SSECustomerKey.IsPlain() {
payload := replacePlaceholders(fsConfig.SSECustomerKey.GetPayload(), replacements)
fsConfig.SSECustomerKey = kms.NewPlainSecret(payload)
}
return fsConfig
}

View file

@ -2188,6 +2188,9 @@ func compareS3Config(expected *vfs.Filesystem, actual *vfs.Filesystem) error { /
if err := checkEncryptedSecret(expected.S3Config.AccessSecret, actual.S3Config.AccessSecret); err != nil {
return fmt.Errorf("fs S3 access secret mismatch: %v", err)
}
if err := checkEncryptedSecret(expected.S3Config.SSECustomerKey, actual.S3Config.SSECustomerKey); err != nil {
return fmt.Errorf("fs S3 SSE customer key mismatch: %v", err)
}
if expected.S3Config.Endpoint != actual.S3Config.Endpoint {
return errors.New("fs S3 endpoint mismatch")
}

View file

@ -45,6 +45,7 @@ type Filesystem struct {
// SetEmptySecrets sets the secrets to empty
func (f *Filesystem) SetEmptySecrets() {
f.S3Config.AccessSecret = kms.NewEmptySecret()
f.S3Config.SSECustomerKey = kms.NewEmptySecret()
f.GCSConfig.Credentials = kms.NewEmptySecret()
f.AzBlobConfig.AccountKey = kms.NewEmptySecret()
f.AzBlobConfig.SASURL = kms.NewEmptySecret()
@ -61,6 +62,9 @@ func (f *Filesystem) SetEmptySecretsIfNil() {
if f.S3Config.AccessSecret == nil {
f.S3Config.AccessSecret = kms.NewEmptySecret()
}
if f.S3Config.SSECustomerKey == nil {
f.S3Config.SSECustomerKey = kms.NewEmptySecret()
}
if f.GCSConfig.Credentials == nil {
f.GCSConfig.Credentials = kms.NewEmptySecret()
}
@ -97,6 +101,9 @@ func (f *Filesystem) SetNilSecretsIfEmpty() {
if f.S3Config.AccessSecret != nil && f.S3Config.AccessSecret.IsEmpty() {
f.S3Config.AccessSecret = nil
}
if f.S3Config.SSECustomerKey != nil && f.S3Config.SSECustomerKey.IsEmpty() {
f.S3Config.SSECustomerKey = nil
}
if f.GCSConfig.Credentials != nil && f.GCSConfig.Credentials.IsEmpty() {
f.GCSConfig.Credentials = nil
}
@ -260,6 +267,9 @@ func (f *Filesystem) HasRedactedSecret() bool {
// TODO move vfs specific code into each *FsConfig struct
switch f.Provider {
case sdk.S3FilesystemProvider:
if f.S3Config.SSECustomerKey.IsRedacted() {
return true
}
return f.S3Config.AccessSecret.IsRedacted()
case sdk.GCSFilesystemProvider:
return f.GCSConfig.Credentials.IsRedacted()
@ -334,7 +344,8 @@ func (f *Filesystem) GetACopy() Filesystem {
ForcePathStyle: f.S3Config.ForcePathStyle,
SkipTLSVerify: f.S3Config.SkipTLSVerify,
},
AccessSecret: f.S3Config.AccessSecret.Clone(),
AccessSecret: f.S3Config.AccessSecret.Clone(),
SSECustomerKey: f.S3Config.SSECustomerKey.Clone(),
},
GCSConfig: GCSFsConfig{
BaseGCSFsConfig: sdk.BaseGCSFsConfig{

View file

@ -19,7 +19,10 @@ package vfs
import (
"context"
"crypto/md5"
"crypto/sha256"
"crypto/tls"
"encoding/base64"
"errors"
"fmt"
"io"
@ -72,10 +75,13 @@ type S3Fs struct {
connectionID string
localTempDir string
// if not empty this fs is mouted as virtual folder in the specified path
mountPath string
config *S3FsConfig
svc *s3.Client
ctxTimeout time.Duration
mountPath string
config *S3FsConfig
svc *s3.Client
ctxTimeout time.Duration
sseCustomerKey string
sseCustomerKeyMD5 string
sseCustomerAlgo string
}
func init() {
@ -121,6 +127,23 @@ func NewS3Fs(connectionID, localTempDir, mountPath string, s3Config S3FsConfig)
fs.config.SessionToken),
)
}
if !fs.config.SSECustomerKey.IsEmpty() {
if err := fs.config.SSECustomerKey.TryDecrypt(); err != nil {
return fs, err
}
key := fs.config.SSECustomerKey.GetPayload()
if len(key) == 32 {
md5sumBinary := md5.Sum([]byte(key))
fs.sseCustomerKey = base64.StdEncoding.EncodeToString([]byte(key))
fs.sseCustomerKeyMD5 = base64.StdEncoding.EncodeToString(md5sumBinary[:])
} else {
keyHash := sha256.Sum256([]byte(key))
md5sumBinary := md5.Sum(keyHash[:])
fs.sseCustomerKey = base64.StdEncoding.EncodeToString(keyHash[:])
fs.sseCustomerKeyMD5 = base64.StdEncoding.EncodeToString(md5sumBinary[:])
}
fs.sseCustomerAlgo = "AES256"
}
fs.setConfigDefaults()
@ -242,9 +265,12 @@ func (fs *S3Fs) Open(name string, offset int64) (File, PipeReader, func(), error
defer cancelFn()
n, err := downloader.Download(ctx, w, &s3.GetObjectInput{
Bucket: aws.String(fs.config.Bucket),
Key: aws.String(name),
Range: streamRange,
Bucket: aws.String(fs.config.Bucket),
Key: aws.String(name),
Range: streamRange,
SSECustomerKey: util.NilIfEmpty(fs.sseCustomerKey),
SSECustomerAlgorithm: util.NilIfEmpty(fs.sseCustomerAlgo),
SSECustomerKeyMD5: util.NilIfEmpty(fs.sseCustomerKeyMD5),
})
w.CloseWithError(err) //nolint:errcheck
fsLog(fs, logger.LevelDebug, "download completed, path: %q size: %v, err: %+v", name, n, err)
@ -293,12 +319,15 @@ func (fs *S3Fs) Create(name string, flag, checks int) (File, PipeWriter, func(),
contentType = mime.TypeByExtension(path.Ext(name))
}
_, err := uploader.Upload(ctx, &s3.PutObjectInput{
Bucket: aws.String(fs.config.Bucket),
Key: aws.String(name),
Body: r,
ACL: types.ObjectCannedACL(fs.config.ACL),
StorageClass: types.StorageClass(fs.config.StorageClass),
ContentType: util.NilIfEmpty(contentType),
Bucket: aws.String(fs.config.Bucket),
Key: aws.String(name),
Body: r,
ACL: types.ObjectCannedACL(fs.config.ACL),
StorageClass: types.StorageClass(fs.config.StorageClass),
ContentType: util.NilIfEmpty(contentType),
SSECustomerKey: util.NilIfEmpty(fs.sseCustomerKey),
SSECustomerAlgorithm: util.NilIfEmpty(fs.sseCustomerAlgo),
SSECustomerKeyMD5: util.NilIfEmpty(fs.sseCustomerKeyMD5),
})
r.CloseWithError(err) //nolint:errcheck
p.Done(err)
@ -703,12 +732,18 @@ func (fs *S3Fs) copyFileInternal(source, target string, srcInfo os.FileInfo) err
defer cancelFn()
copyObject := &s3.CopyObjectInput{
Bucket: aws.String(fs.config.Bucket),
CopySource: aws.String(copySource),
Key: aws.String(target),
StorageClass: types.StorageClass(fs.config.StorageClass),
ACL: types.ObjectCannedACL(fs.config.ACL),
ContentType: util.NilIfEmpty(contentType),
Bucket: aws.String(fs.config.Bucket),
CopySource: aws.String(copySource),
Key: aws.String(target),
StorageClass: types.StorageClass(fs.config.StorageClass),
ACL: types.ObjectCannedACL(fs.config.ACL),
ContentType: util.NilIfEmpty(contentType),
CopySourceSSECustomerKey: util.NilIfEmpty(fs.sseCustomerKey),
CopySourceSSECustomerAlgorithm: util.NilIfEmpty(fs.sseCustomerAlgo),
CopySourceSSECustomerKeyMD5: util.NilIfEmpty(fs.sseCustomerKeyMD5),
SSECustomerKey: util.NilIfEmpty(fs.sseCustomerKey),
SSECustomerAlgorithm: util.NilIfEmpty(fs.sseCustomerAlgo),
SSECustomerKeyMD5: util.NilIfEmpty(fs.sseCustomerKeyMD5),
}
metadata := getMetadata(srcInfo)
@ -812,11 +847,14 @@ func (fs *S3Fs) doMultipartCopy(source, target, contentType string, fileSize int
defer cancelFn()
res, err := fs.svc.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
Bucket: aws.String(fs.config.Bucket),
Key: aws.String(target),
StorageClass: types.StorageClass(fs.config.StorageClass),
ACL: types.ObjectCannedACL(fs.config.ACL),
ContentType: util.NilIfEmpty(contentType),
Bucket: aws.String(fs.config.Bucket),
Key: aws.String(target),
StorageClass: types.StorageClass(fs.config.StorageClass),
ACL: types.ObjectCannedACL(fs.config.ACL),
ContentType: util.NilIfEmpty(contentType),
SSECustomerKey: util.NilIfEmpty(fs.sseCustomerKey),
SSECustomerAlgorithm: util.NilIfEmpty(fs.sseCustomerAlgo),
SSECustomerKeyMD5: util.NilIfEmpty(fs.sseCustomerKeyMD5),
})
if err != nil {
return fmt.Errorf("unable to create multipart copy request: %w", err)
@ -871,12 +909,18 @@ func (fs *S3Fs) doMultipartCopy(source, target, contentType string, fileSize int
defer innerCancelFn()
partResp, err := fs.svc.UploadPartCopy(innerCtx, &s3.UploadPartCopyInput{
Bucket: aws.String(fs.config.Bucket),
CopySource: aws.String(source),
Key: aws.String(target),
PartNumber: &partNum,
UploadId: aws.String(uploadID),
CopySourceRange: aws.String(fmt.Sprintf("bytes=%d-%d", partStart, partEnd-1)),
Bucket: aws.String(fs.config.Bucket),
CopySource: aws.String(source),
Key: aws.String(target),
PartNumber: &partNum,
UploadId: aws.String(uploadID),
CopySourceRange: aws.String(fmt.Sprintf("bytes=%d-%d", partStart, partEnd-1)),
CopySourceSSECustomerKey: util.NilIfEmpty(fs.sseCustomerKey),
CopySourceSSECustomerAlgorithm: util.NilIfEmpty(fs.sseCustomerAlgo),
CopySourceSSECustomerKeyMD5: util.NilIfEmpty(fs.sseCustomerKeyMD5),
SSECustomerKey: util.NilIfEmpty(fs.sseCustomerKey),
SSECustomerAlgorithm: util.NilIfEmpty(fs.sseCustomerAlgo),
SSECustomerKeyMD5: util.NilIfEmpty(fs.sseCustomerKeyMD5),
})
if err != nil {
errOnce.Do(func() {
@ -959,8 +1003,11 @@ func (fs *S3Fs) headObject(name string) (*s3.HeadObjectOutput, error) {
defer cancelFn()
obj, err := fs.svc.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(fs.config.Bucket),
Key: aws.String(name),
Bucket: aws.String(fs.config.Bucket),
Key: aws.String(name),
SSECustomerKey: util.NilIfEmpty(fs.sseCustomerKey),
SSECustomerAlgorithm: util.NilIfEmpty(fs.sseCustomerAlgo),
SSECustomerKeyMD5: util.NilIfEmpty(fs.sseCustomerKeyMD5),
})
metric.S3HeadObjectCompleted(err)
return obj, err
@ -1002,8 +1049,11 @@ func (fs *S3Fs) downloadToWriter(name string, w PipeWriter) (int64, error) {
})
n, err := downloader.Download(ctx, w, &s3.GetObjectInput{
Bucket: aws.String(fs.config.Bucket),
Key: aws.String(name),
Bucket: aws.String(fs.config.Bucket),
Key: aws.String(name),
SSECustomerKey: util.NilIfEmpty(fs.sseCustomerKey),
SSECustomerAlgorithm: util.NilIfEmpty(fs.sseCustomerAlgo),
SSECustomerKeyMD5: util.NilIfEmpty(fs.sseCustomerKeyMD5),
})
fsLog(fs, logger.LevelDebug, "download before resuming upload completed, path %q size: %d, err: %+v",
name, n, err)

View file

@ -267,7 +267,8 @@ func (q *QuotaCheckResult) GetRemainingFiles() int {
// S3FsConfig defines the configuration for S3 based filesystem
type S3FsConfig struct {
sdk.BaseS3FsConfig
AccessSecret *kms.Secret `json:"access_secret,omitempty"`
AccessSecret *kms.Secret `json:"access_secret,omitempty"`
SSECustomerKey *kms.Secret `json:"sse_customer_key,omitempty"`
}
// HideConfidentialData hides confidential data
@ -275,6 +276,9 @@ func (c *S3FsConfig) HideConfidentialData() {
if c.AccessSecret != nil {
c.AccessSecret.Hide()
}
if c.SSECustomerKey != nil {
c.SSECustomerKey.Hide()
}
}
func (c *S3FsConfig) isEqual(other S3FsConfig) bool {
@ -337,6 +341,15 @@ func (c *S3FsConfig) areMultipartFieldsEqual(other S3FsConfig) bool {
}
func (c *S3FsConfig) isSecretEqual(other S3FsConfig) bool {
if c.SSECustomerKey == nil {
c.SSECustomerKey = kms.NewEmptySecret()
}
if other.SSECustomerKey == nil {
other.SSECustomerKey = kms.NewEmptySecret()
}
if !c.SSECustomerKey.IsEqual(other.SSECustomerKey) {
return false
}
if c.AccessSecret == nil {
c.AccessSecret = kms.NewEmptySecret()
}
@ -365,6 +378,12 @@ func (c *S3FsConfig) checkCredentials() error {
if !c.AccessSecret.IsEmpty() && !c.AccessSecret.IsValidInput() {
return errors.New("invalid access_secret")
}
if c.SSECustomerKey.IsEncrypted() && !c.SSECustomerKey.IsValid() {
return errors.New("invalid encrypted sse_customer_key")
}
if !c.SSECustomerKey.IsEmpty() && !c.SSECustomerKey.IsValidInput() {
return errors.New("invalid sse_customer_key")
}
return nil
}
@ -388,6 +407,16 @@ func (c *S3FsConfig) ValidateAndEncryptCredentials(additionalData string) error
)
}
}
if c.SSECustomerKey.IsPlain() {
c.SSECustomerKey.SetAdditionalData(additionalData)
err := c.SSECustomerKey.Encrypt()
if err != nil {
return util.NewI18nError(
util.NewValidationError(fmt.Sprintf("could not encrypt s3 SSE customer key: %v", err)),
util.I18nErrorFsValidation,
)
}
}
return nil
}
@ -434,6 +463,9 @@ func (c *S3FsConfig) validate() error {
if c.AccessSecret == nil {
c.AccessSecret = kms.NewEmptySecret()
}
if c.SSECustomerKey == nil {
c.SSECustomerKey = kms.NewEmptySecret()
}
if c.Bucket == "" {
return util.NewI18nError(errors.New("bucket cannot be empty"), util.I18nErrorBucketRequired)
}

View file

@ -1485,8 +1485,11 @@ func TestUserCacheIsolation(t *testing.T) {
LockSystem: webdav.NewMemLS(),
}
cachedUser.User.FsConfig.S3Config.AccessSecret = kms.NewPlainSecret("test secret")
cachedUser.User.FsConfig.S3Config.SSECustomerKey = kms.NewPlainSecret("test key")
err = cachedUser.User.FsConfig.S3Config.AccessSecret.Encrypt()
assert.NoError(t, err)
err = cachedUser.User.FsConfig.S3Config.SSECustomerKey.Encrypt()
assert.NoError(t, err)
dataprovider.CacheWebDAVUser(cachedUser)
cachedUser, ok := dataprovider.GetCachedWebDAVUser(username)
@ -1500,6 +1503,9 @@ func TestUserCacheIsolation(t *testing.T) {
assert.True(t, cachedUser.User.FsConfig.S3Config.AccessSecret.IsEncrypted())
err = cachedUser.User.FsConfig.S3Config.AccessSecret.Decrypt()
assert.NoError(t, err)
assert.True(t, cachedUser.User.FsConfig.S3Config.SSECustomerKey.IsEncrypted())
err = cachedUser.User.FsConfig.S3Config.SSECustomerKey.Decrypt()
assert.NoError(t, err)
cachedUser.User.FsConfig.Provider = sdk.S3FilesystemProvider
_, err = cachedUser.User.GetFilesystem("")
assert.Error(t, err, "we don't have to get the previously cached filesystem!")
@ -1508,6 +1514,7 @@ func TestUserCacheIsolation(t *testing.T) {
if assert.True(t, ok) {
assert.Equal(t, sdk.LocalFilesystemProvider, cachedUser.User.FsConfig.Provider)
assert.False(t, cachedUser.User.FsConfig.S3Config.AccessSecret.IsEncrypted())
assert.False(t, cachedUser.User.FsConfig.S3Config.SSECustomerKey.IsEncrypted())
}
err = dataprovider.DeleteUser(username, "", "", "")

View file

@ -5576,6 +5576,8 @@ components:
type: string
access_secret:
$ref: '#/components/schemas/Secret'
sse_customer_key:
$ref: '#/components/schemas/Secret'
role_arn:
type: string
description: 'Optional IAM Role ARN to assume'

View file

@ -597,6 +597,8 @@
"region": "Region",
"access_key": "Access Key",
"access_secret": "Access Secret",
"sse_customer_key": "Server-side encryption key",
"sse_customer_key_help": "You can store your data encrypted with this key, but if you lose or change this key, you will lose all files encrypted with it. Files that are not encrypted or encrypted with a different key will not be accessible",
"endpoint": "Endpoint",
"endpoint_help": "For AWS S3, leave blank to use the default endpoint for the specified region",
"sftp_endpoint_help": "Endpoint as host:port. The port is always required",

View file

@ -597,6 +597,8 @@
"region": "Regione",
"access_key": "Chiave di accesso",
"access_secret": "Chiave di accesso segreta",
"sse_customer_key": "Chiave di crittografia",
"sse_customer_key_help": "Puoi archiviare i tuoi dati crittografati con questa chiave, ma se perdi o modifichi inavvertitamente questa chiave, perderai tutti i file crittografati con essa. I file non crittografati o crittografati con una chiave diversa non saranno accessibili",
"endpoint": "Endpoint",
"endpoint_help": "Per AWS S3, lasciare vuoto per utilizzare l'endpoint predefinito per la regione specificata",
"sftp_endpoint_help": "Endpoint come host:porta. La porta è sempre richiesta",

View file

@ -173,6 +173,15 @@ explicit grant from the SFTPGo Team (support@sftpgo.com).
</div>
</div>
<div class="form-group row mt-10 fsconfig-s3">
<label for="idS3SSECustomerKey" data-i18n="storage.sse_customer_key" class="col-md-3 col-form-label">SSE Customer Key</label>
<div class="col-md-9">
<input id="idS3SSECustomerKey" type="password" class="form-control" name="s3_sse_customer_key" autocomplete="new-password" spellcheck="false"
value="{{if .S3Config.SSECustomerKey.IsEncrypted}}{{.RedactedSecret}}{{else}}{{.S3Config.SSECustomerKey.GetPayload}}{{end}}" aria-describedby="idS3SSECustomerKeyHelp"/>
<div id="idS3SSECustomerKeyHelp" class="form-text" data-i18n="storage.sse_customer_key_help"></div>
</div>
</div>
<div class="form-group row align-items-center mt-10 fsconfig-s3">
<div class="col-md-5">
<div class="form-check form-switch form-check-custom form-check-solid">