From 2fbf608895cc45fdc27a331774b6e3a32a59992d Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Thu, 15 Aug 2024 10:09:06 +0200 Subject: [PATCH] S3: add SSE customer key Signed-off-by: Nicola Murino --- go.mod | 22 +++--- go.sum | 52 ++++++------- internal/httpd/api_user.go | 3 + internal/httpd/httpd_test.go | 67 +++++++++++++++- internal/httpd/webadmin.go | 5 ++ internal/httpdtest/httpdtest.go | 3 + internal/vfs/filesystem.go | 13 +++- internal/vfs/s3fs.go | 118 ++++++++++++++++++++--------- internal/vfs/vfs.go | 34 ++++++++- internal/webdavd/internal_test.go | 7 ++ openapi/openapi.yaml | 2 + static/locales/en/translation.json | 2 + static/locales/it/translation.json | 2 + templates/webadmin/fsconfig.html | 9 +++ 14 files changed, 264 insertions(+), 75 deletions(-) diff --git a/go.mod b/go.mod index 1b68d447..ac344173 100644 --- a/go.mod +++ b/go.mod @@ -47,12 +47,12 @@ require ( github.com/pires/go-proxyproto v0.7.0 github.com/pkg/sftp v1.13.7-0.20240410063531-637088883317 github.com/pquerna/otp v1.4.0 - github.com/prometheus/client_golang v1.19.1 + github.com/prometheus/client_golang v1.20.0 github.com/robfig/cron/v3 v3.0.1 github.com/rs/cors v1.11.0 github.com/rs/xid v1.5.0 github.com/rs/zerolog v1.33.0 - github.com/sftpgo/sdk v0.1.8 + github.com/sftpgo/sdk v0.1.9-0.20240815080450-426add0ab063 github.com/shirou/gopsutil/v3 v3.24.5 github.com/spf13/afero v1.11.0 github.com/spf13/cobra v1.8.1 @@ -66,20 +66,20 @@ require ( github.com/yl2chen/cidranger v1.0.3-0.20210928021809-d1cb2c52f37a go.etcd.io/bbolt v1.3.10 go.uber.org/automaxprocs v1.5.3 - gocloud.dev v0.38.0 + gocloud.dev v0.39.0 golang.org/x/crypto v0.26.0 golang.org/x/net v0.28.0 golang.org/x/oauth2 v0.22.0 golang.org/x/sys v0.24.0 golang.org/x/term v0.23.0 golang.org/x/time v0.6.0 - google.golang.org/api v0.191.0 + google.golang.org/api v0.192.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) require ( - cloud.google.com/go v0.115.0 // indirect - cloud.google.com/go/auth v0.8.0 // indirect + cloud.google.com/go v0.115.1 // indirect + cloud.google.com/go/auth v0.8.1 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect cloud.google.com/go/compute/metadata v0.5.0 // indirect cloud.google.com/go/iam v1.1.13 // indirect @@ -97,7 +97,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect - github.com/aws/smithy-go v1.20.3 // indirect + github.com/aws/smithy-go v1.20.4 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/boombuler/barcode v1.0.2 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect @@ -138,7 +138,7 @@ require ( github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/miekg/dns v1.1.61 // indirect + github.com/miekg/dns v1.1.62 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -173,9 +173,9 @@ require ( golang.org/x/text v0.17.0 // indirect golang.org/x/tools v0.24.0 // indirect golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect - google.golang.org/genproto v0.0.0-20240812133136-8ffd90a71988 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240812133136-8ffd90a71988 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240812133136-8ffd90a71988 // indirect + google.golang.org/genproto v0.0.0-20240814211410-ddb44dafa142 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect google.golang.org/grpc v1.65.0 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index caeab2c1..c359c240 100644 --- a/go.sum +++ b/go.sum @@ -1,18 +1,18 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= -cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= -cloud.google.com/go/auth v0.8.0 h1:y8jUJLl/Fg+qNBWxP/Hox2ezJvjkrPb952PC1p0G6A4= -cloud.google.com/go/auth v0.8.0/go.mod h1:qGVp/Y3kDRSDZ5gFD/XPUfYQ9xW1iI7q8RIRoCyBbJc= +cloud.google.com/go v0.115.1 h1:Jo0SM9cQnSkYfp44+v+NQXHpcHqlnRJk2qxh6yvxxxQ= +cloud.google.com/go v0.115.1/go.mod h1:DuujITeaufu3gL68/lOFIirVNJwQeyf5UXyi+Wbgknc= +cloud.google.com/go/auth v0.8.1 h1:QZW9FjC5lZzN864p13YxvAtGUlQ+KgRL+8Sg45Z6vxo= +cloud.google.com/go/auth v0.8.1/go.mod h1:qGVp/Y3kDRSDZ5gFD/XPUfYQ9xW1iI7q8RIRoCyBbJc= cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= cloud.google.com/go/iam v1.1.13 h1:7zWBXG9ERbMLrzQBRhFliAV+kjcRToDTgQT3CTwYyv4= cloud.google.com/go/iam v1.1.13/go.mod h1:K8mY0uSXwEXS30KrnVb+j54LB/ntfZu1dr+4zFMNbus= -cloud.google.com/go/kms v1.18.4 h1:dYN3OCsQ6wJLLtOnI8DGUwQ5shMusXsWCCC+s09ATsk= -cloud.google.com/go/kms v1.18.4/go.mod h1:SG1bgQ3UWW6/KdPo9uuJnzELXY5YTTMJtDYvajiQ22g= -cloud.google.com/go/longrunning v0.5.11 h1:Havn1kGjz3whCfoD8dxMLP73Ph5w+ODyZB9RUsDxtGk= -cloud.google.com/go/longrunning v0.5.11/go.mod h1:rDn7//lmlfWV1Dx6IB4RatCPenTwwmqXuiP0/RgoEO4= +cloud.google.com/go/kms v1.18.5 h1:75LSlVs60hyHK3ubs2OHd4sE63OAMcM2BdSJc2bkuM4= +cloud.google.com/go/kms v1.18.5/go.mod h1:yXunGUGzabH8rjUPImp2ndHiGolHeWJJ0LODLedicIY= +cloud.google.com/go/longrunning v0.5.12 h1:5LqSIdERr71CqfUsFlJdBpOkBH8FBCFD7P1nTWy3TYE= +cloud.google.com/go/longrunning v0.5.12/go.mod h1:S5hMV8CDJ6r50t2ubVJSKQVv5u0rmik5//KgLO3k4lU= cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= @@ -79,8 +79,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 h1:yiwVzJW2ZxZTurVbYWA7QOrA github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4/go.mod h1:0oxfLkpz3rQ/CHlx5hB7H69YUpFiI1tql6Q6Ne+1bCw= github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 h1:ZsDKRLXGWHk8WdtyYMoGNO7bTudrvuKpDKgMVRlepGE= github.com/aws/aws-sdk-go-v2/service/sts v1.30.3/go.mod h1:zwySh8fpFyXp9yOr/KVzxOl8SRqgf/IDw5aUt9UKFcQ= -github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE= -github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4= +github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I= @@ -284,8 +284,8 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mhale/smtpd v0.8.3 h1:8j8YNXajksoSLZja3HdwvYVZPuJSqAxFsib3adzRRt8= github.com/mhale/smtpd v0.8.3/go.mod h1:MQl+y2hwIEQCXtNhe5+55n0GZOjSmeqORDIXbqUL3x4= -github.com/miekg/dns v1.1.61 h1:nLxbwF3XxhwVSm8g9Dghm9MHPaUZuqhPiGL+675ZmEs= -github.com/miekg/dns v1.1.61/go.mod h1:mnAarhS3nWaW+NVP2wTkYVIZyHNJ098SJZUki3eykwQ= +github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ= +github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ= github.com/minio/sio v0.4.0 h1:u4SWVEm5lXSqU42ZWawV0D9I5AZ5YMmo2RXpEQ/kRhc= github.com/minio/sio v0.4.0/go.mod h1:oBSjJeGbBdRMZZwna07sX9EFzZy+ywu5aofRiV1g79I= github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= @@ -316,8 +316,8 @@ github.com/pquerna/otp v1.4.0 h1:wZvl1TIVxKRThZIBiwOOHOGP/1+nZyWBil9Y2XNEDzg= github.com/pquerna/otp v1.4.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= -github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= -github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_golang v1.20.0 h1:jBzTZ7B099Rg24tny+qngoynol8LtVYlA2bqx3vEloI= +github.com/prometheus/client_golang v1.20.0/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= @@ -343,8 +343,8 @@ github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJ github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= -github.com/sftpgo/sdk v0.1.8 h1:HAywJl9jZnigFGztA/CWLieOW+R+HH6js6o6/qYvuSY= -github.com/sftpgo/sdk v0.1.8/go.mod h1:Isl0IEzS/Muvh8Fr4X+NWFsOS/fZQHRD4oPQpoY7C4g= +github.com/sftpgo/sdk v0.1.9-0.20240815080450-426add0ab063 h1:r+XUT9mg/W97xiS6ZJ1BczLwTYiGKCRQ+Z69QZBnAZ8= +github.com/sftpgo/sdk v0.1.9-0.20240815080450-426add0ab063/go.mod h1:Isl0IEzS/Muvh8Fr4X+NWFsOS/fZQHRD4oPQpoY7C4g= github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= @@ -416,8 +416,8 @@ go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -gocloud.dev v0.38.0 h1:SpxfaOc/Fp4PeO8ui7wRcCZV0EgXZ+IWcVSLn6ZMSw0= -gocloud.dev v0.38.0/go.mod h1:3XjKvd2E5iVNu/xFImRzjN0d/fkNHe4s0RiKidpEUMQ= +gocloud.dev v0.39.0 h1:EYABYGhAalPUaMrbSKOr5lejxoxvXj99nE8XFtsDgds= +gocloud.dev v0.39.0/go.mod h1:drz+VyYNBvrMTW0KZiBAYEdl8lbNZx+OQ7oQvdrFmSQ= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI= golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= @@ -516,19 +516,19 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 h1:LLhsEBxRTBLuKlQxFBYUOU8xyFgXv6cOTp2HASDlsDk= golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -google.golang.org/api v0.191.0 h1:cJcF09Z+4HAB2t5qTQM1ZtfL/PemsLFkcFG67qq2afk= -google.golang.org/api v0.191.0/go.mod h1:tD5dsFGxFza0hnQveGfVk9QQYKcfp+VzgRqyXFxE0+E= +google.golang.org/api v0.192.0 h1:PljqpNAfZaaSpS+TnANfnNAXKdzHM/B9bKhwRlo7JP0= +google.golang.org/api v0.192.0/go.mod h1:9VcphjvAxPKLmSxVSzPlSRXy/5ARMEw5bf58WoVXafQ= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20240812133136-8ffd90a71988 h1:CT2Thj5AuPV9phrYMtzX11k+XkzMGfRAet42PmoTATM= -google.golang.org/genproto v0.0.0-20240812133136-8ffd90a71988/go.mod h1:7uvplUBj4RjHAxIZ//98LzOvrQ04JBkaixRmCMI29hc= -google.golang.org/genproto/googleapis/api v0.0.0-20240812133136-8ffd90a71988 h1:+/tmTy5zAieooKIXfzDm9KiA3Bv6JBwriRN9LY+yayk= -google.golang.org/genproto/googleapis/api v0.0.0-20240812133136-8ffd90a71988/go.mod h1:4+X6GvPs+25wZKbQq9qyAXrwIRExv7w0Ea6MgZLZiDM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240812133136-8ffd90a71988 h1:V71AcdLZr2p8dC9dbOIMCpqi4EmRl8wUwnJzXXLmbmc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240812133136-8ffd90a71988/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/genproto v0.0.0-20240814211410-ddb44dafa142 h1:oLiyxGgE+rt22duwci1+TG7bg2/L1LQsXwfjPlmuJA0= +google.golang.org/genproto v0.0.0-20240814211410-ddb44dafa142/go.mod h1:G11eXq53iI5Q+kyNOmCvnzBaxEA2Q/Ik5Tj7nqBE8j4= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 h1:e7S5W7MGGLaSu8j3YjdezkZ+m1/Nm0uRVRMEMGk26Xs= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= diff --git a/internal/httpd/api_user.go b/internal/httpd/api_user.go index 7b55082a..749cefe2 100644 --- a/internal/httpd/api_user.go +++ b/internal/httpd/api_user.go @@ -278,6 +278,9 @@ func updateEncryptedSecrets(fsConfig *vfs.Filesystem, currentFsConfig *vfs.Files if fsConfig.S3Config.AccessSecret.IsNotPlainAndNotEmpty() { fsConfig.S3Config.AccessSecret = currentFsConfig.S3Config.AccessSecret } + if fsConfig.S3Config.SSECustomerKey.IsNotPlainAndNotEmpty() { + fsConfig.S3Config.SSECustomerKey = currentFsConfig.S3Config.SSECustomerKey + } case sdk.AzureBlobFilesystemProvider: if fsConfig.AzBlobConfig.AccountKey.IsNotPlainAndNotEmpty() { fsConfig.AzBlobConfig.AccountKey = currentFsConfig.AzBlobConfig.AccountKey diff --git a/internal/httpd/httpd_test.go b/internal/httpd/httpd_test.go index 960e8f81..fc43b71d 100644 --- a/internal/httpd/httpd_test.go +++ b/internal/httpd/httpd_test.go @@ -4934,6 +4934,12 @@ func TestUserRedactedPassword(t *testing.T) { assert.Contains(t, err.Error(), "cannot save a user with a redacted secret") } u.FsConfig.S3Config.AccessSecret = kms.NewPlainSecret("secret") + u.FsConfig.S3Config.SSECustomerKey = kms.NewSecret(sdkkms.SecretStatusRedacted, "mysecretkey", "", "") + _, resp, err = httpdtest.AddUser(u, http.StatusBadRequest) + assert.NoError(t, err, string(resp)) + assert.Contains(t, string(resp), "cannot save a user with a redacted secret") + + u.FsConfig.S3Config.SSECustomerKey = kms.NewPlainSecret("key") user, _, err := httpdtest.AddUser(u, http.StatusCreated) assert.NoError(t, err) @@ -5653,6 +5659,7 @@ func TestUserS3Config(t *testing.T) { user.FsConfig.S3Config.Bucket = "test" //nolint:goconst user.FsConfig.S3Config.AccessKey = "Server-Access-Key" user.FsConfig.S3Config.AccessSecret = kms.NewPlainSecret("Server-Access-Secret") + user.FsConfig.S3Config.SSECustomerKey = kms.NewPlainSecret("SSE-encryption-key") user.FsConfig.S3Config.RoleARN = "myRoleARN" user.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000" user.FsConfig.S3Config.UploadPartSize = 8 @@ -5686,6 +5693,10 @@ func TestUserS3Config(t *testing.T) { assert.NotEmpty(t, user.FsConfig.S3Config.AccessSecret.GetPayload()) assert.Empty(t, user.FsConfig.S3Config.AccessSecret.GetAdditionalData()) assert.Empty(t, user.FsConfig.S3Config.AccessSecret.GetKey()) + assert.Equal(t, sdkkms.SecretStatusSecretBox, user.FsConfig.S3Config.SSECustomerKey.GetStatus()) + assert.NotEmpty(t, user.FsConfig.S3Config.SSECustomerKey.GetPayload()) + assert.Empty(t, user.FsConfig.S3Config.SSECustomerKey.GetAdditionalData()) + assert.Empty(t, user.FsConfig.S3Config.SSECustomerKey.GetKey()) assert.Equal(t, 60, user.FsConfig.S3Config.DownloadPartMaxTime) assert.Equal(t, 40, user.FsConfig.S3Config.UploadPartMaxTime) assert.True(t, user.FsConfig.S3Config.SkipTLSVerify) @@ -5710,13 +5721,14 @@ func TestUserS3Config(t *testing.T) { user.ID = 0 user.CreatedAt = 0 user.VirtualFolders = nil + user.FsConfig.S3Config.SSECustomerKey = kms.NewEmptySecret() secret := kms.NewSecret(sdkkms.SecretStatusSecretBox, "Server-Access-Secret", "", "") user.FsConfig.S3Config.AccessSecret = secret _, _, err = httpdtest.AddUser(user, http.StatusCreated) assert.Error(t, err) user.FsConfig.S3Config.AccessSecret.SetStatus(sdkkms.SecretStatusPlain) - user, _, err = httpdtest.AddUser(user, http.StatusCreated) - assert.NoError(t, err) + user, resp, err := httpdtest.AddUser(user, http.StatusCreated) + assert.NoError(t, err, string(resp)) initialSecretPayload := user.FsConfig.S3Config.AccessSecret.GetPayload() assert.Equal(t, sdkkms.SecretStatusSecretBox, user.FsConfig.S3Config.AccessSecret.GetStatus()) assert.NotEmpty(t, initialSecretPayload) @@ -6093,6 +6105,7 @@ func TestUserHiddenFields(t *testing.T) { u1.FsConfig.S3Config.Region = "us-east-1" u1.FsConfig.S3Config.AccessKey = "S3-Access-Key" u1.FsConfig.S3Config.AccessSecret = kms.NewPlainSecret("S3-Access-Secret") + u1.FsConfig.S3Config.SSECustomerKey = kms.NewPlainSecret("SSE-secret-key") user1, _, err := httpdtest.AddUser(u1, http.StatusCreated) assert.NoError(t, err) @@ -6165,6 +6178,10 @@ func TestUserHiddenFields(t *testing.T) { assert.Empty(t, user1.FsConfig.S3Config.AccessSecret.GetAdditionalData()) assert.NotEmpty(t, user1.FsConfig.S3Config.AccessSecret.GetStatus()) assert.NotEmpty(t, user1.FsConfig.S3Config.AccessSecret.GetPayload()) + assert.Empty(t, user1.FsConfig.S3Config.SSECustomerKey.GetKey()) + assert.Empty(t, user1.FsConfig.S3Config.SSECustomerKey.GetAdditionalData()) + assert.NotEmpty(t, user1.FsConfig.S3Config.SSECustomerKey.GetStatus()) + assert.NotEmpty(t, user1.FsConfig.S3Config.SSECustomerKey.GetPayload()) user2, _, err = httpdtest.GetUserByUsername(user2.Username, http.StatusOK) assert.NoError(t, err) @@ -6219,12 +6236,22 @@ func TestUserHiddenFields(t *testing.T) { assert.NotEmpty(t, user1.FsConfig.S3Config.AccessSecret.GetAdditionalData()) assert.NotEmpty(t, user1.FsConfig.S3Config.AccessSecret.GetStatus()) assert.NotEmpty(t, user1.FsConfig.S3Config.AccessSecret.GetPayload()) + assert.NotEmpty(t, user1.FsConfig.S3Config.SSECustomerKey.GetKey()) + assert.NotEmpty(t, user1.FsConfig.S3Config.SSECustomerKey.GetAdditionalData()) + assert.NotEmpty(t, user1.FsConfig.S3Config.SSECustomerKey.GetStatus()) + assert.NotEmpty(t, user1.FsConfig.S3Config.SSECustomerKey.GetPayload()) err = user1.FsConfig.S3Config.AccessSecret.Decrypt() assert.NoError(t, err) + err = user1.FsConfig.S3Config.SSECustomerKey.Decrypt() + assert.NoError(t, err) assert.Equal(t, sdkkms.SecretStatusPlain, user1.FsConfig.S3Config.AccessSecret.GetStatus()) assert.Equal(t, u1.FsConfig.S3Config.AccessSecret.GetPayload(), user1.FsConfig.S3Config.AccessSecret.GetPayload()) assert.Empty(t, user1.FsConfig.S3Config.AccessSecret.GetKey()) assert.Empty(t, user1.FsConfig.S3Config.AccessSecret.GetAdditionalData()) + assert.Equal(t, sdkkms.SecretStatusPlain, user1.FsConfig.S3Config.SSECustomerKey.GetStatus()) + assert.Equal(t, u1.FsConfig.S3Config.SSECustomerKey.GetPayload(), user1.FsConfig.S3Config.SSECustomerKey.GetPayload()) + assert.Empty(t, user1.FsConfig.S3Config.SSECustomerKey.GetKey()) + assert.Empty(t, user1.FsConfig.S3Config.SSECustomerKey.GetAdditionalData()) user2, err = dataprovider.UserExists(user2.Username, "") assert.NoError(t, err) @@ -22133,6 +22160,7 @@ func TestUserTemplateMock(t *testing.T) { form.Set("s3_region", user.FsConfig.S3Config.Region) form.Set("s3_access_key", "%username%") form.Set("s3_access_secret", "%password%") + form.Set("s3_sse_customer_key", "%password%") form.Set("s3_key_prefix", "base/%username%") form.Set("max_upload_file_size", "0") form.Set("default_shares_expiration", "0") @@ -22232,13 +22260,21 @@ func TestUserTemplateMock(t *testing.T) { require.Equal(t, path.Join("base", user1.Username)+"/", user1.FsConfig.S3Config.KeyPrefix) require.Equal(t, path.Join("base", user2.Username)+"/", user2.FsConfig.S3Config.KeyPrefix) require.True(t, user1.FsConfig.S3Config.AccessSecret.IsEncrypted()) + require.True(t, user1.FsConfig.S3Config.SSECustomerKey.IsEncrypted()) err = user1.FsConfig.S3Config.AccessSecret.Decrypt() require.NoError(t, err) + err = user1.FsConfig.S3Config.SSECustomerKey.Decrypt() + require.NoError(t, err) require.Equal(t, "password1", user1.FsConfig.S3Config.AccessSecret.GetPayload()) + require.Equal(t, "password1", user1.FsConfig.S3Config.SSECustomerKey.GetPayload()) require.True(t, user2.FsConfig.S3Config.AccessSecret.IsEncrypted()) + require.True(t, user2.FsConfig.S3Config.SSECustomerKey.IsEncrypted()) err = user2.FsConfig.S3Config.AccessSecret.Decrypt() require.NoError(t, err) + err = user2.FsConfig.S3Config.SSECustomerKey.Decrypt() + require.NoError(t, err) require.Equal(t, "password2", user2.FsConfig.S3Config.AccessSecret.GetPayload()) + require.Equal(t, "password2", user2.FsConfig.S3Config.SSECustomerKey.GetPayload()) require.True(t, user1.Filters.Hooks.ExternalAuthDisabled) require.True(t, user1.Filters.Hooks.CheckPasswordDisabled) require.False(t, user1.Filters.Hooks.PreLoginDisabled) @@ -22484,6 +22520,7 @@ func TestFolderTemplateMock(t *testing.T) { form.Set("s3_region", "us-east-1") form.Set("s3_access_key", "%name%") form.Set("s3_access_secret", "pwd%name%") + form.Set("s3_sse_customer_key", "key%name%") form.Set("s3_key_prefix", "base/%name%") b, contentType, _ = getMultipartFormData(form, "", "") @@ -22523,18 +22560,27 @@ func TestFolderTemplateMock(t *testing.T) { require.NoError(t, err) require.Equal(t, fmt.Sprintf("pwd%s", folder1), folder.FsConfig.S3Config.AccessSecret.GetPayload()) require.Equal(t, path.Join("base", folder1)+"/", folder.FsConfig.S3Config.KeyPrefix) + err = folder.FsConfig.S3Config.SSECustomerKey.Decrypt() + require.NoError(t, err) + require.Equal(t, fmt.Sprintf("key%s", folder1), folder.FsConfig.S3Config.SSECustomerKey.GetPayload()) case folder2: require.Equal(t, folder2, folder.FsConfig.S3Config.AccessKey) err = folder.FsConfig.S3Config.AccessSecret.Decrypt() require.NoError(t, err) require.Equal(t, "pwd"+folder2, folder.FsConfig.S3Config.AccessSecret.GetPayload()) require.Equal(t, "base/"+folder2+"/", folder.FsConfig.S3Config.KeyPrefix) + err = folder.FsConfig.S3Config.SSECustomerKey.Decrypt() + require.NoError(t, err) + require.Equal(t, "key"+folder2, folder.FsConfig.S3Config.SSECustomerKey.GetPayload()) default: require.Equal(t, folder3, folder.FsConfig.S3Config.AccessKey) err = folder.FsConfig.S3Config.AccessSecret.Decrypt() require.NoError(t, err) require.Equal(t, "pwd"+folder3, folder.FsConfig.S3Config.AccessSecret.GetPayload()) require.Equal(t, "base/"+folder3+"/", folder.FsConfig.S3Config.KeyPrefix) + err = folder.FsConfig.S3Config.SSECustomerKey.Decrypt() + require.NoError(t, err) + require.Equal(t, "key"+folder3, folder.FsConfig.S3Config.SSECustomerKey.GetPayload()) } } @@ -22583,6 +22629,7 @@ func TestWebUserS3Mock(t *testing.T) { user.FsConfig.S3Config.Region = "eu-west-1" user.FsConfig.S3Config.AccessKey = "access-key" user.FsConfig.S3Config.AccessSecret = kms.NewPlainSecret("access-secret") + user.FsConfig.S3Config.SSECustomerKey = kms.NewPlainSecret("enc-key") user.FsConfig.S3Config.RoleARN = "arn:aws:iam::123456789012:user/Development/product_1234/*" user.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/path?a=b" user.FsConfig.S3Config.StorageClass = "Standard" @@ -22623,6 +22670,7 @@ func TestWebUserS3Mock(t *testing.T) { form.Set("s3_region", user.FsConfig.S3Config.Region) form.Set("s3_access_key", user.FsConfig.S3Config.AccessKey) form.Set("s3_access_secret", user.FsConfig.S3Config.AccessSecret.GetPayload()) + form.Set("s3_sse_customer_key", user.FsConfig.S3Config.SSECustomerKey.GetPayload()) form.Set("s3_role_arn", user.FsConfig.S3Config.RoleARN) form.Set("s3_storage_class", user.FsConfig.S3Config.StorageClass) form.Set("s3_acl", user.FsConfig.S3Config.ACL) @@ -22747,6 +22795,10 @@ func TestWebUserS3Mock(t *testing.T) { assert.NotEmpty(t, updateUser.FsConfig.S3Config.AccessSecret.GetPayload()) assert.Empty(t, updateUser.FsConfig.S3Config.AccessSecret.GetKey()) assert.Empty(t, updateUser.FsConfig.S3Config.AccessSecret.GetAdditionalData()) + assert.Equal(t, sdkkms.SecretStatusSecretBox, updateUser.FsConfig.S3Config.SSECustomerKey.GetStatus()) + assert.NotEmpty(t, updateUser.FsConfig.S3Config.SSECustomerKey.GetPayload()) + assert.Empty(t, updateUser.FsConfig.S3Config.SSECustomerKey.GetKey()) + assert.Empty(t, updateUser.FsConfig.S3Config.SSECustomerKey.GetAdditionalData()) assert.Equal(t, user.Description, updateUser.Description) assert.True(t, updateUser.Filters.Hooks.PreLoginDisabled) assert.False(t, updateUser.Filters.Hooks.ExternalAuthDisabled) @@ -22756,6 +22808,7 @@ func TestWebUserS3Mock(t *testing.T) { assert.Equal(t, 1, updateUser.Filters.FTPSecurity) // now check that a redacted password is not saved form.Set("s3_access_secret", redactedSecret) + form.Set("s3_sse_customer_key", redactedSecret) b, contentType, _ = getMultipartFormData(form, "", "") req, _ = http.NewRequest(http.MethodPost, path.Join(webUserPath, user.Username), &b) setJWTCookieForReq(req, webToken) @@ -22774,10 +22827,15 @@ func TestWebUserS3Mock(t *testing.T) { assert.Equal(t, updateUser.FsConfig.S3Config.AccessSecret.GetPayload(), lastUpdatedUser.FsConfig.S3Config.AccessSecret.GetPayload()) assert.Empty(t, lastUpdatedUser.FsConfig.S3Config.AccessSecret.GetKey()) assert.Empty(t, lastUpdatedUser.FsConfig.S3Config.AccessSecret.GetAdditionalData()) + assert.Equal(t, sdkkms.SecretStatusSecretBox, lastUpdatedUser.FsConfig.S3Config.SSECustomerKey.GetStatus()) + assert.Equal(t, updateUser.FsConfig.S3Config.SSECustomerKey.GetPayload(), lastUpdatedUser.FsConfig.S3Config.SSECustomerKey.GetPayload()) + assert.Empty(t, lastUpdatedUser.FsConfig.S3Config.SSECustomerKey.GetKey()) + assert.Empty(t, lastUpdatedUser.FsConfig.S3Config.SSECustomerKey.GetAdditionalData()) assert.Equal(t, lastPwdChange, lastUpdatedUser.LastPasswordChange) // now clear credentials form.Set("s3_access_key", "") form.Set("s3_access_secret", "") + form.Set("s3_sse_customer_key", "") b, contentType, _ = getMultipartFormData(form, "", "") req, _ = http.NewRequest(http.MethodPost, path.Join(webUserPath, user.Username), &b) setJWTCookieForReq(req, webToken) @@ -22792,6 +22850,7 @@ func TestWebUserS3Mock(t *testing.T) { err = render.DecodeJSON(rr.Body, &userGet) assert.NoError(t, err) assert.Nil(t, userGet.FsConfig.S3Config.AccessSecret) + assert.Nil(t, userGet.FsConfig.S3Config.SSECustomerKey) req, _ = http.NewRequest(http.MethodDelete, path.Join(userPath, user.Username), nil) setBearerForReq(req, apiToken) @@ -25187,6 +25246,7 @@ func TestS3WebFolderMock(t *testing.T) { S3Region := "eu-west-1" S3AccessKey := "access-key" S3AccessSecret := kms.NewPlainSecret("folder-access-secret") + S3SSEKey := kms.NewPlainSecret("folder-sse-key") S3SessionToken := "fake session token" S3RoleARN := "arn:aws:iam::123456789012:user/Development/product_1234/*" S3Endpoint := "http://127.0.0.1:9000/path?b=c" @@ -25208,6 +25268,7 @@ func TestS3WebFolderMock(t *testing.T) { form.Set("s3_region", S3Region) form.Set("s3_access_key", S3AccessKey) form.Set("s3_access_secret", S3AccessSecret.GetPayload()) + form.Set("s3_sse_customer_key", S3SSEKey.GetPayload()) form.Set("s3_session_token", S3SessionToken) form.Set("s3_role_arn", S3RoleARN) form.Set("s3_storage_class", S3StorageClass) @@ -25255,6 +25316,7 @@ func TestS3WebFolderMock(t *testing.T) { assert.Equal(t, S3Region, folder.FsConfig.S3Config.Region) assert.Equal(t, S3AccessKey, folder.FsConfig.S3Config.AccessKey) assert.NotEmpty(t, folder.FsConfig.S3Config.AccessSecret.GetPayload()) + assert.NotEmpty(t, folder.FsConfig.S3Config.SSECustomerKey.GetPayload()) assert.Equal(t, S3Endpoint, folder.FsConfig.S3Config.Endpoint) assert.Equal(t, S3StorageClass, folder.FsConfig.S3Config.StorageClass) assert.Equal(t, S3ACL, folder.FsConfig.S3Config.ACL) @@ -25305,6 +25367,7 @@ func TestS3WebFolderMock(t *testing.T) { assert.Equal(t, S3AccessKey, folder.FsConfig.S3Config.AccessKey) assert.Equal(t, S3RoleARN, folder.FsConfig.S3Config.RoleARN) assert.NotEmpty(t, folder.FsConfig.S3Config.AccessSecret.GetPayload()) + assert.NotEmpty(t, folder.FsConfig.S3Config.SSECustomerKey.GetPayload()) assert.Equal(t, S3Endpoint, folder.FsConfig.S3Config.Endpoint) assert.Equal(t, S3StorageClass, folder.FsConfig.S3Config.StorageClass) assert.Equal(t, S3KeyPrefix, folder.FsConfig.S3Config.KeyPrefix) diff --git a/internal/httpd/webadmin.go b/internal/httpd/webadmin.go index 46fd833a..c37236ab 100644 --- a/internal/httpd/webadmin.go +++ b/internal/httpd/webadmin.go @@ -1530,6 +1530,7 @@ func getS3Config(r *http.Request) (vfs.S3FsConfig, error) { config.AccessKey = strings.TrimSpace(r.Form.Get("s3_access_key")) config.RoleARN = strings.TrimSpace(r.Form.Get("s3_role_arn")) config.AccessSecret = getSecretFromFormField(r, "s3_access_secret") + config.SSECustomerKey = getSecretFromFormField(r, "s3_sse_customer_key") config.Endpoint = strings.TrimSpace(r.Form.Get("s3_endpoint")) config.StorageClass = strings.TrimSpace(r.Form.Get("s3_storage_class")) config.ACL = strings.TrimSpace(r.Form.Get("s3_acl")) @@ -1855,6 +1856,10 @@ func getS3FsFromTemplate(fsConfig vfs.S3FsConfig, replacements map[string]string payload := replacePlaceholders(fsConfig.AccessSecret.GetPayload(), replacements) fsConfig.AccessSecret = kms.NewPlainSecret(payload) } + if fsConfig.SSECustomerKey != nil && fsConfig.SSECustomerKey.IsPlain() { + payload := replacePlaceholders(fsConfig.SSECustomerKey.GetPayload(), replacements) + fsConfig.SSECustomerKey = kms.NewPlainSecret(payload) + } return fsConfig } diff --git a/internal/httpdtest/httpdtest.go b/internal/httpdtest/httpdtest.go index af53577b..f6ec7aed 100644 --- a/internal/httpdtest/httpdtest.go +++ b/internal/httpdtest/httpdtest.go @@ -2188,6 +2188,9 @@ func compareS3Config(expected *vfs.Filesystem, actual *vfs.Filesystem) error { / if err := checkEncryptedSecret(expected.S3Config.AccessSecret, actual.S3Config.AccessSecret); err != nil { return fmt.Errorf("fs S3 access secret mismatch: %v", err) } + if err := checkEncryptedSecret(expected.S3Config.SSECustomerKey, actual.S3Config.SSECustomerKey); err != nil { + return fmt.Errorf("fs S3 SSE customer key mismatch: %v", err) + } if expected.S3Config.Endpoint != actual.S3Config.Endpoint { return errors.New("fs S3 endpoint mismatch") } diff --git a/internal/vfs/filesystem.go b/internal/vfs/filesystem.go index cf013097..3d871c51 100644 --- a/internal/vfs/filesystem.go +++ b/internal/vfs/filesystem.go @@ -45,6 +45,7 @@ type Filesystem struct { // SetEmptySecrets sets the secrets to empty func (f *Filesystem) SetEmptySecrets() { f.S3Config.AccessSecret = kms.NewEmptySecret() + f.S3Config.SSECustomerKey = kms.NewEmptySecret() f.GCSConfig.Credentials = kms.NewEmptySecret() f.AzBlobConfig.AccountKey = kms.NewEmptySecret() f.AzBlobConfig.SASURL = kms.NewEmptySecret() @@ -61,6 +62,9 @@ func (f *Filesystem) SetEmptySecretsIfNil() { if f.S3Config.AccessSecret == nil { f.S3Config.AccessSecret = kms.NewEmptySecret() } + if f.S3Config.SSECustomerKey == nil { + f.S3Config.SSECustomerKey = kms.NewEmptySecret() + } if f.GCSConfig.Credentials == nil { f.GCSConfig.Credentials = kms.NewEmptySecret() } @@ -97,6 +101,9 @@ func (f *Filesystem) SetNilSecretsIfEmpty() { if f.S3Config.AccessSecret != nil && f.S3Config.AccessSecret.IsEmpty() { f.S3Config.AccessSecret = nil } + if f.S3Config.SSECustomerKey != nil && f.S3Config.SSECustomerKey.IsEmpty() { + f.S3Config.SSECustomerKey = nil + } if f.GCSConfig.Credentials != nil && f.GCSConfig.Credentials.IsEmpty() { f.GCSConfig.Credentials = nil } @@ -260,6 +267,9 @@ func (f *Filesystem) HasRedactedSecret() bool { // TODO move vfs specific code into each *FsConfig struct switch f.Provider { case sdk.S3FilesystemProvider: + if f.S3Config.SSECustomerKey.IsRedacted() { + return true + } return f.S3Config.AccessSecret.IsRedacted() case sdk.GCSFilesystemProvider: return f.GCSConfig.Credentials.IsRedacted() @@ -334,7 +344,8 @@ func (f *Filesystem) GetACopy() Filesystem { ForcePathStyle: f.S3Config.ForcePathStyle, SkipTLSVerify: f.S3Config.SkipTLSVerify, }, - AccessSecret: f.S3Config.AccessSecret.Clone(), + AccessSecret: f.S3Config.AccessSecret.Clone(), + SSECustomerKey: f.S3Config.SSECustomerKey.Clone(), }, GCSConfig: GCSFsConfig{ BaseGCSFsConfig: sdk.BaseGCSFsConfig{ diff --git a/internal/vfs/s3fs.go b/internal/vfs/s3fs.go index ee56bc24..20a8f092 100644 --- a/internal/vfs/s3fs.go +++ b/internal/vfs/s3fs.go @@ -19,7 +19,10 @@ package vfs import ( "context" + "crypto/md5" + "crypto/sha256" "crypto/tls" + "encoding/base64" "errors" "fmt" "io" @@ -72,10 +75,13 @@ type S3Fs struct { connectionID string localTempDir string // if not empty this fs is mouted as virtual folder in the specified path - mountPath string - config *S3FsConfig - svc *s3.Client - ctxTimeout time.Duration + mountPath string + config *S3FsConfig + svc *s3.Client + ctxTimeout time.Duration + sseCustomerKey string + sseCustomerKeyMD5 string + sseCustomerAlgo string } func init() { @@ -121,6 +127,23 @@ func NewS3Fs(connectionID, localTempDir, mountPath string, s3Config S3FsConfig) fs.config.SessionToken), ) } + if !fs.config.SSECustomerKey.IsEmpty() { + if err := fs.config.SSECustomerKey.TryDecrypt(); err != nil { + return fs, err + } + key := fs.config.SSECustomerKey.GetPayload() + if len(key) == 32 { + md5sumBinary := md5.Sum([]byte(key)) + fs.sseCustomerKey = base64.StdEncoding.EncodeToString([]byte(key)) + fs.sseCustomerKeyMD5 = base64.StdEncoding.EncodeToString(md5sumBinary[:]) + } else { + keyHash := sha256.Sum256([]byte(key)) + md5sumBinary := md5.Sum(keyHash[:]) + fs.sseCustomerKey = base64.StdEncoding.EncodeToString(keyHash[:]) + fs.sseCustomerKeyMD5 = base64.StdEncoding.EncodeToString(md5sumBinary[:]) + } + fs.sseCustomerAlgo = "AES256" + } fs.setConfigDefaults() @@ -242,9 +265,12 @@ func (fs *S3Fs) Open(name string, offset int64) (File, PipeReader, func(), error defer cancelFn() n, err := downloader.Download(ctx, w, &s3.GetObjectInput{ - Bucket: aws.String(fs.config.Bucket), - Key: aws.String(name), - Range: streamRange, + Bucket: aws.String(fs.config.Bucket), + Key: aws.String(name), + Range: streamRange, + SSECustomerKey: util.NilIfEmpty(fs.sseCustomerKey), + SSECustomerAlgorithm: util.NilIfEmpty(fs.sseCustomerAlgo), + SSECustomerKeyMD5: util.NilIfEmpty(fs.sseCustomerKeyMD5), }) w.CloseWithError(err) //nolint:errcheck fsLog(fs, logger.LevelDebug, "download completed, path: %q size: %v, err: %+v", name, n, err) @@ -293,12 +319,15 @@ func (fs *S3Fs) Create(name string, flag, checks int) (File, PipeWriter, func(), contentType = mime.TypeByExtension(path.Ext(name)) } _, err := uploader.Upload(ctx, &s3.PutObjectInput{ - Bucket: aws.String(fs.config.Bucket), - Key: aws.String(name), - Body: r, - ACL: types.ObjectCannedACL(fs.config.ACL), - StorageClass: types.StorageClass(fs.config.StorageClass), - ContentType: util.NilIfEmpty(contentType), + Bucket: aws.String(fs.config.Bucket), + Key: aws.String(name), + Body: r, + ACL: types.ObjectCannedACL(fs.config.ACL), + StorageClass: types.StorageClass(fs.config.StorageClass), + ContentType: util.NilIfEmpty(contentType), + SSECustomerKey: util.NilIfEmpty(fs.sseCustomerKey), + SSECustomerAlgorithm: util.NilIfEmpty(fs.sseCustomerAlgo), + SSECustomerKeyMD5: util.NilIfEmpty(fs.sseCustomerKeyMD5), }) r.CloseWithError(err) //nolint:errcheck p.Done(err) @@ -703,12 +732,18 @@ func (fs *S3Fs) copyFileInternal(source, target string, srcInfo os.FileInfo) err defer cancelFn() copyObject := &s3.CopyObjectInput{ - Bucket: aws.String(fs.config.Bucket), - CopySource: aws.String(copySource), - Key: aws.String(target), - StorageClass: types.StorageClass(fs.config.StorageClass), - ACL: types.ObjectCannedACL(fs.config.ACL), - ContentType: util.NilIfEmpty(contentType), + Bucket: aws.String(fs.config.Bucket), + CopySource: aws.String(copySource), + Key: aws.String(target), + StorageClass: types.StorageClass(fs.config.StorageClass), + ACL: types.ObjectCannedACL(fs.config.ACL), + ContentType: util.NilIfEmpty(contentType), + CopySourceSSECustomerKey: util.NilIfEmpty(fs.sseCustomerKey), + CopySourceSSECustomerAlgorithm: util.NilIfEmpty(fs.sseCustomerAlgo), + CopySourceSSECustomerKeyMD5: util.NilIfEmpty(fs.sseCustomerKeyMD5), + SSECustomerKey: util.NilIfEmpty(fs.sseCustomerKey), + SSECustomerAlgorithm: util.NilIfEmpty(fs.sseCustomerAlgo), + SSECustomerKeyMD5: util.NilIfEmpty(fs.sseCustomerKeyMD5), } metadata := getMetadata(srcInfo) @@ -812,11 +847,14 @@ func (fs *S3Fs) doMultipartCopy(source, target, contentType string, fileSize int defer cancelFn() res, err := fs.svc.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: aws.String(fs.config.Bucket), - Key: aws.String(target), - StorageClass: types.StorageClass(fs.config.StorageClass), - ACL: types.ObjectCannedACL(fs.config.ACL), - ContentType: util.NilIfEmpty(contentType), + Bucket: aws.String(fs.config.Bucket), + Key: aws.String(target), + StorageClass: types.StorageClass(fs.config.StorageClass), + ACL: types.ObjectCannedACL(fs.config.ACL), + ContentType: util.NilIfEmpty(contentType), + SSECustomerKey: util.NilIfEmpty(fs.sseCustomerKey), + SSECustomerAlgorithm: util.NilIfEmpty(fs.sseCustomerAlgo), + SSECustomerKeyMD5: util.NilIfEmpty(fs.sseCustomerKeyMD5), }) if err != nil { return fmt.Errorf("unable to create multipart copy request: %w", err) @@ -871,12 +909,18 @@ func (fs *S3Fs) doMultipartCopy(source, target, contentType string, fileSize int defer innerCancelFn() partResp, err := fs.svc.UploadPartCopy(innerCtx, &s3.UploadPartCopyInput{ - Bucket: aws.String(fs.config.Bucket), - CopySource: aws.String(source), - Key: aws.String(target), - PartNumber: &partNum, - UploadId: aws.String(uploadID), - CopySourceRange: aws.String(fmt.Sprintf("bytes=%d-%d", partStart, partEnd-1)), + Bucket: aws.String(fs.config.Bucket), + CopySource: aws.String(source), + Key: aws.String(target), + PartNumber: &partNum, + UploadId: aws.String(uploadID), + CopySourceRange: aws.String(fmt.Sprintf("bytes=%d-%d", partStart, partEnd-1)), + CopySourceSSECustomerKey: util.NilIfEmpty(fs.sseCustomerKey), + CopySourceSSECustomerAlgorithm: util.NilIfEmpty(fs.sseCustomerAlgo), + CopySourceSSECustomerKeyMD5: util.NilIfEmpty(fs.sseCustomerKeyMD5), + SSECustomerKey: util.NilIfEmpty(fs.sseCustomerKey), + SSECustomerAlgorithm: util.NilIfEmpty(fs.sseCustomerAlgo), + SSECustomerKeyMD5: util.NilIfEmpty(fs.sseCustomerKeyMD5), }) if err != nil { errOnce.Do(func() { @@ -959,8 +1003,11 @@ func (fs *S3Fs) headObject(name string) (*s3.HeadObjectOutput, error) { defer cancelFn() obj, err := fs.svc.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: aws.String(fs.config.Bucket), - Key: aws.String(name), + Bucket: aws.String(fs.config.Bucket), + Key: aws.String(name), + SSECustomerKey: util.NilIfEmpty(fs.sseCustomerKey), + SSECustomerAlgorithm: util.NilIfEmpty(fs.sseCustomerAlgo), + SSECustomerKeyMD5: util.NilIfEmpty(fs.sseCustomerKeyMD5), }) metric.S3HeadObjectCompleted(err) return obj, err @@ -1002,8 +1049,11 @@ func (fs *S3Fs) downloadToWriter(name string, w PipeWriter) (int64, error) { }) n, err := downloader.Download(ctx, w, &s3.GetObjectInput{ - Bucket: aws.String(fs.config.Bucket), - Key: aws.String(name), + Bucket: aws.String(fs.config.Bucket), + Key: aws.String(name), + SSECustomerKey: util.NilIfEmpty(fs.sseCustomerKey), + SSECustomerAlgorithm: util.NilIfEmpty(fs.sseCustomerAlgo), + SSECustomerKeyMD5: util.NilIfEmpty(fs.sseCustomerKeyMD5), }) fsLog(fs, logger.LevelDebug, "download before resuming upload completed, path %q size: %d, err: %+v", name, n, err) diff --git a/internal/vfs/vfs.go b/internal/vfs/vfs.go index 5778c24f..03bec8bd 100644 --- a/internal/vfs/vfs.go +++ b/internal/vfs/vfs.go @@ -267,7 +267,8 @@ func (q *QuotaCheckResult) GetRemainingFiles() int { // S3FsConfig defines the configuration for S3 based filesystem type S3FsConfig struct { sdk.BaseS3FsConfig - AccessSecret *kms.Secret `json:"access_secret,omitempty"` + AccessSecret *kms.Secret `json:"access_secret,omitempty"` + SSECustomerKey *kms.Secret `json:"sse_customer_key,omitempty"` } // HideConfidentialData hides confidential data @@ -275,6 +276,9 @@ func (c *S3FsConfig) HideConfidentialData() { if c.AccessSecret != nil { c.AccessSecret.Hide() } + if c.SSECustomerKey != nil { + c.SSECustomerKey.Hide() + } } func (c *S3FsConfig) isEqual(other S3FsConfig) bool { @@ -337,6 +341,15 @@ func (c *S3FsConfig) areMultipartFieldsEqual(other S3FsConfig) bool { } func (c *S3FsConfig) isSecretEqual(other S3FsConfig) bool { + if c.SSECustomerKey == nil { + c.SSECustomerKey = kms.NewEmptySecret() + } + if other.SSECustomerKey == nil { + other.SSECustomerKey = kms.NewEmptySecret() + } + if !c.SSECustomerKey.IsEqual(other.SSECustomerKey) { + return false + } if c.AccessSecret == nil { c.AccessSecret = kms.NewEmptySecret() } @@ -365,6 +378,12 @@ func (c *S3FsConfig) checkCredentials() error { if !c.AccessSecret.IsEmpty() && !c.AccessSecret.IsValidInput() { return errors.New("invalid access_secret") } + if c.SSECustomerKey.IsEncrypted() && !c.SSECustomerKey.IsValid() { + return errors.New("invalid encrypted sse_customer_key") + } + if !c.SSECustomerKey.IsEmpty() && !c.SSECustomerKey.IsValidInput() { + return errors.New("invalid sse_customer_key") + } return nil } @@ -388,6 +407,16 @@ func (c *S3FsConfig) ValidateAndEncryptCredentials(additionalData string) error ) } } + if c.SSECustomerKey.IsPlain() { + c.SSECustomerKey.SetAdditionalData(additionalData) + err := c.SSECustomerKey.Encrypt() + if err != nil { + return util.NewI18nError( + util.NewValidationError(fmt.Sprintf("could not encrypt s3 SSE customer key: %v", err)), + util.I18nErrorFsValidation, + ) + } + } return nil } @@ -434,6 +463,9 @@ func (c *S3FsConfig) validate() error { if c.AccessSecret == nil { c.AccessSecret = kms.NewEmptySecret() } + if c.SSECustomerKey == nil { + c.SSECustomerKey = kms.NewEmptySecret() + } if c.Bucket == "" { return util.NewI18nError(errors.New("bucket cannot be empty"), util.I18nErrorBucketRequired) } diff --git a/internal/webdavd/internal_test.go b/internal/webdavd/internal_test.go index 57cfeb93..a3751e22 100644 --- a/internal/webdavd/internal_test.go +++ b/internal/webdavd/internal_test.go @@ -1485,8 +1485,11 @@ func TestUserCacheIsolation(t *testing.T) { LockSystem: webdav.NewMemLS(), } cachedUser.User.FsConfig.S3Config.AccessSecret = kms.NewPlainSecret("test secret") + cachedUser.User.FsConfig.S3Config.SSECustomerKey = kms.NewPlainSecret("test key") err = cachedUser.User.FsConfig.S3Config.AccessSecret.Encrypt() assert.NoError(t, err) + err = cachedUser.User.FsConfig.S3Config.SSECustomerKey.Encrypt() + assert.NoError(t, err) dataprovider.CacheWebDAVUser(cachedUser) cachedUser, ok := dataprovider.GetCachedWebDAVUser(username) @@ -1500,6 +1503,9 @@ func TestUserCacheIsolation(t *testing.T) { assert.True(t, cachedUser.User.FsConfig.S3Config.AccessSecret.IsEncrypted()) err = cachedUser.User.FsConfig.S3Config.AccessSecret.Decrypt() assert.NoError(t, err) + assert.True(t, cachedUser.User.FsConfig.S3Config.SSECustomerKey.IsEncrypted()) + err = cachedUser.User.FsConfig.S3Config.SSECustomerKey.Decrypt() + assert.NoError(t, err) cachedUser.User.FsConfig.Provider = sdk.S3FilesystemProvider _, err = cachedUser.User.GetFilesystem("") assert.Error(t, err, "we don't have to get the previously cached filesystem!") @@ -1508,6 +1514,7 @@ func TestUserCacheIsolation(t *testing.T) { if assert.True(t, ok) { assert.Equal(t, sdk.LocalFilesystemProvider, cachedUser.User.FsConfig.Provider) assert.False(t, cachedUser.User.FsConfig.S3Config.AccessSecret.IsEncrypted()) + assert.False(t, cachedUser.User.FsConfig.S3Config.SSECustomerKey.IsEncrypted()) } err = dataprovider.DeleteUser(username, "", "", "") diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index b66974fb..f14abc87 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -5576,6 +5576,8 @@ components: type: string access_secret: $ref: '#/components/schemas/Secret' + sse_customer_key: + $ref: '#/components/schemas/Secret' role_arn: type: string description: 'Optional IAM Role ARN to assume' diff --git a/static/locales/en/translation.json b/static/locales/en/translation.json index 0d686bfc..4b0ac4f1 100644 --- a/static/locales/en/translation.json +++ b/static/locales/en/translation.json @@ -597,6 +597,8 @@ "region": "Region", "access_key": "Access Key", "access_secret": "Access Secret", + "sse_customer_key": "Server-side encryption key", + "sse_customer_key_help": "You can store your data encrypted with this key, but if you lose or change this key, you will lose all files encrypted with it. Files that are not encrypted or encrypted with a different key will not be accessible", "endpoint": "Endpoint", "endpoint_help": "For AWS S3, leave blank to use the default endpoint for the specified region", "sftp_endpoint_help": "Endpoint as host:port. The port is always required", diff --git a/static/locales/it/translation.json b/static/locales/it/translation.json index 201be86e..58b2681e 100644 --- a/static/locales/it/translation.json +++ b/static/locales/it/translation.json @@ -597,6 +597,8 @@ "region": "Regione", "access_key": "Chiave di accesso", "access_secret": "Chiave di accesso segreta", + "sse_customer_key": "Chiave di crittografia", + "sse_customer_key_help": "Puoi archiviare i tuoi dati crittografati con questa chiave, ma se perdi o modifichi inavvertitamente questa chiave, perderai tutti i file crittografati con essa. I file non crittografati o crittografati con una chiave diversa non saranno accessibili", "endpoint": "Endpoint", "endpoint_help": "Per AWS S3, lasciare vuoto per utilizzare l'endpoint predefinito per la regione specificata", "sftp_endpoint_help": "Endpoint come host:porta. La porta รจ sempre richiesta", diff --git a/templates/webadmin/fsconfig.html b/templates/webadmin/fsconfig.html index 50bb5060..43fa3a8f 100644 --- a/templates/webadmin/fsconfig.html +++ b/templates/webadmin/fsconfig.html @@ -173,6 +173,15 @@ explicit grant from the SFTPGo Team (support@sftpgo.com). +
+ +
+ +
+
+
+