beenull/sftpgo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

s3: allow to skip TLS verification

Browse source 
Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
This commit is contained in:
Nicola Murino 2023-11-05 19:27:11 +01:00
parent 9456884584
commit 654ce2e349
No known key found for this signature in database
GPG key ID: 935D2952DEC4EECF
11 changed files with 65 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
docs/portable-mode.md
View file

@ -130,6 +130,13 @@ Flags:
prefix and its contents
--s3-region string
--s3-role-arn string
--s3-skip-tls-verify If enabled the S3 client accepts any TLS
certificate presented by the server and
any host name in that certificate.
In this mode, TLS is susceptible to
man-in-the-middle attacks.
This should be used only for testing.
--s3-storage-class string
--s3-upload-concurrency int How many parts are uploaded in
parallel (default 2)

2
go.mod
View file

@ -53,7 +53,7 @@ require (
github.com/rs/cors v1.10.1
github.com/rs/xid v1.5.0
github.com/rs/zerolog v1.31.0
github.com/sftpgo/sdk v0.1.6-0.20231011150824-e8d35102c725
github.com/sftpgo/sdk v0.1.6-0.20231105181545-b44c8058fc25
github.com/shirou/gopsutil/v3 v3.23.10
github.com/spf13/afero v1.10.0
github.com/spf13/cobra v1.8.0

4
go.sum
View file

@ -445,8 +445,8 @@ github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJ
github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY=
github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
github.com/sftpgo/sdk v0.1.6-0.20231011150824-e8d35102c725 h1:bV58DOKSIzTqz+UI3q81kRRvsaV0p8b2189nOy7vn7Q=
github.com/sftpgo/sdk v0.1.6-0.20231011150824-e8d35102c725/go.mod h1:KE3KY7v13gSR+8PJMp5LYKQaWELafEM2ZPhmAo7rclM=
github.com/sftpgo/sdk v0.1.6-0.20231105181545-b44c8058fc25 h1:R8cTb41ZX5WSYw8q8ufTKQfOvXh7aLQWqdnteDY/96U=
github.com/sftpgo/sdk v0.1.6-0.20231105181545-b44c8058fc25/go.mod h1:6s/PFoLUd7FXG3wGlrdVhrA0SJOwri2h9kzTph/2oiU=
github.com/shirou/gopsutil/v3 v3.23.10 h1:/N42opWlYzegYaVkWejXWJpbzKv2JDy3mrgGzKsh9hM=
github.com/shirou/gopsutil/v3 v3.23.10/go.mod h1:JIE26kpucQi+innVlAUnIEOSBhBUkirr5b44yr55+WE=
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=

9
internal/cmd/portable.go
View file

@ -65,6 +65,7 @@ var (
portableS3ULPartSize int
portableS3ULConcurrency int
portableS3ForcePathStyle bool
portableS3SkipTLSVerify bool
portableGCSBucket string
portableGCSCredentialsFile string
portableGCSAutoCredentials int
@ -240,6 +241,7 @@ Please take a look at the usage below to customize the serving parameters`,
UploadPartSize: int64(portableS3ULPartSize),
UploadConcurrency: portableS3ULConcurrency,
ForcePathStyle: portableS3ForcePathStyle,
SkipTLSVerify: portableS3SkipTLSVerify,
},
AccessSecret: kms.NewPlainSecret(portableS3AccessSecret),
},
@ -373,6 +375,13 @@ prefix and its contents`)
portableCmd.Flags().IntVar(&portableS3ULConcurrency, "s3-upload-concurrency", 2, `How many parts are uploaded in
parallel`)
portableCmd.Flags().BoolVar(&portableS3ForcePathStyle, "s3-force-path-style", false, `Force path style bucket URL`)
portableCmd.Flags().BoolVar(&portableS3SkipTLSVerify, "s3-skip-tls-verify", false, `If enabled the S3 client accepts any TLS
certificate presented by the server and
any host name in that certificate.
In this mode, TLS is susceptible to
man-in-the-middle attacks.
This should be used only for testing.
`)
portableCmd.Flags().StringVar(&portableGCSBucket, "gcs-bucket", "", "")
portableCmd.Flags().StringVar(&portableGCSStorageClass, "gcs-storage-class", "", "")
portableCmd.Flags().StringVar(&portableGCSKeyPrefix, "gcs-key-prefix", "", `Allows to restrict access to the

6
internal/httpd/httpd_test.go
View file

@ -5215,6 +5215,7 @@ func TestUserS3Config(t *testing.T) {
user.FsConfig.S3Config.DownloadPartMaxTime = 60
user.FsConfig.S3Config.UploadPartMaxTime = 40
user.FsConfig.S3Config.ForcePathStyle = true
user.FsConfig.S3Config.SkipTLSVerify = true
user.FsConfig.S3Config.DownloadPartSize = 6
folderName := "vfolderName"
user.VirtualFolders = append(user.VirtualFolders, vfs.VirtualFolder{
@ -5243,6 +5244,7 @@ func TestUserS3Config(t *testing.T) {
assert.Empty(t, user.FsConfig.S3Config.AccessSecret.GetKey())
assert.Equal(t, 60, user.FsConfig.S3Config.DownloadPartMaxTime)
assert.Equal(t, 40, user.FsConfig.S3Config.UploadPartMaxTime)
assert.True(t, user.FsConfig.S3Config.SkipTLSVerify)
if assert.Len(t, user.VirtualFolders, 1) {
folder := user.VirtualFolders[0]
assert.Equal(t, sdkkms.SecretStatusSecretBox, folder.FsConfig.CryptConfig.Passphrase.GetStatus())
@ -20957,6 +20959,7 @@ func TestWebUserS3Mock(t *testing.T) {
user.FsConfig.S3Config.DownloadPartSize = 6
user.FsConfig.S3Config.DownloadConcurrency = 3
user.FsConfig.S3Config.ForcePathStyle = true
user.FsConfig.S3Config.SkipTLSVerify = true
user.FsConfig.S3Config.ACL = "public-read"
user.Description = "s3 tèst user"
form := make(url.Values)
@ -21005,6 +21008,7 @@ func TestWebUserS3Mock(t *testing.T) {
form.Set("password_strength", "0")
form.Set("ftp_security", "1")
form.Set("s3_force_path_style", "checked")
form.Set("s3_skip_tls_verify", "checked")
form.Set("description", user.Description)
form.Add("hooks", "pre_login_disabled")
form.Add("allow_api_key_auth", "1")
@ -21093,6 +21097,7 @@ func TestWebUserS3Mock(t *testing.T) {
assert.Equal(t, updateUser.FsConfig.S3Config.DownloadConcurrency, user.FsConfig.S3Config.DownloadConcurrency)
assert.Equal(t, lastPwdChange, updateUser.LastPasswordChange)
assert.True(t, updateUser.FsConfig.S3Config.ForcePathStyle)
assert.True(t, updateUser.FsConfig.S3Config.SkipTLSVerify)
if assert.Equal(t, 2, len(updateUser.Filters.FilePatterns)) {
for _, filter := range updateUser.Filters.FilePatterns {
switch filter.Path {
@ -23538,6 +23543,7 @@ func TestS3WebFolderMock(t *testing.T) {
assert.Equal(t, S3DownloadConcurrency, folder.FsConfig.S3Config.DownloadConcurrency)
assert.Equal(t, int64(S3DownloadPartSize), folder.FsConfig.S3Config.DownloadPartSize)
assert.False(t, folder.FsConfig.S3Config.ForcePathStyle)
assert.False(t, folder.FsConfig.S3Config.SkipTLSVerify)
// update
S3UploadConcurrency = 10
form.Set("s3_upload_concurrency", "b")

1
internal/httpd/webadmin.go
View file

@ -1588,6 +1588,7 @@ func getS3Config(r *http.Request) (vfs.S3FsConfig, error) {
return config, fmt.Errorf("invalid s3 download concurrency: %w", err)
}
config.ForcePathStyle = r.Form.Get("s3_force_path_style") != ""
config.SkipTLSVerify = r.Form.Get("s3_skip_tls_verify") != ""
config.DownloadPartMaxTime, err = strconv.Atoi(r.Form.Get("s3_download_part_max_time"))
if err != nil {
return config, fmt.Errorf("invalid s3 download part max time: %w", err)

3
internal/httpdtest/httpdtest.go
View file

@ -2199,6 +2199,9 @@ func compareS3Config(expected *vfs.Filesystem, actual *vfs.Filesystem) error { /
if expected.S3Config.ForcePathStyle != actual.S3Config.ForcePathStyle {
return errors.New("fs S3 force path style mismatch")
}
if expected.S3Config.SkipTLSVerify != actual.S3Config.SkipTLSVerify {
return errors.New("fs S3 skip TLS verify mismatch")
}
if expected.S3Config.DownloadPartMaxTime != actual.S3Config.DownloadPartMaxTime {
return errors.New("fs S3 download part max time mismatch")
}

1
internal/vfs/filesystem.go
View file

@ -321,6 +321,7 @@ func (f *Filesystem) GetACopy() Filesystem {
DownloadPartMaxTime: f.S3Config.DownloadPartMaxTime,
UploadPartMaxTime: f.S3Config.UploadPartMaxTime,
ForcePathStyle: f.S3Config.ForcePathStyle,
SkipTLSVerify: f.S3Config.SkipTLSVerify,
},
AccessSecret: f.S3Config.AccessSecret.Clone(),
},

26
internal/vfs/s3fs.go
View file

@ -19,6 +19,7 @@ package vfs
import (
"context"
"crypto/tls"
"errors"
"fmt"
"mime"
@ -97,7 +98,9 @@ func NewS3Fs(connectionID, localTempDir, mountPath string, s3Config S3FsConfig)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
awsConfig, err := config.LoadDefaultConfig(ctx, config.WithHTTPClient(getAWSHTTPClient(0, 30*time.Second)))
awsConfig, err := config.LoadDefaultConfig(ctx, config.WithHTTPClient(
getAWSHTTPClient(0, 30*time.Second, fs.config.SkipTLSVerify)),
)
if err != nil {
return fs, fmt.Errorf("unable to get AWS config: %w", err)
}
@ -215,7 +218,8 @@ func (fs *S3Fs) Open(name string, offset int64) (File, *PipeReader, func(), erro
d.PartSize = fs.config.DownloadPartSize
if offset == 0 && fs.config.DownloadPartMaxTime > 0 {
d.ClientOptions = append(d.ClientOptions, func(o *s3.Options) {
o.HTTPClient = getAWSHTTPClient(fs.config.DownloadPartMaxTime, 100*time.Millisecond)
o.HTTPClient = getAWSHTTPClient(fs.config.DownloadPartMaxTime, 100*time.Millisecond,
fs.config.SkipTLSVerify)
})
}
})
@ -264,7 +268,8 @@ func (fs *S3Fs) Create(name string, flag, checks int) (File, PipeWriter, func(),
u.PartSize = fs.config.UploadPartSize
if fs.config.UploadPartMaxTime > 0 {
u.ClientOptions = append(u.ClientOptions, func(o *s3.Options) {
o.HTTPClient = getAWSHTTPClient(fs.config.UploadPartMaxTime, 100*time.Millisecond)
o.HTTPClient = getAWSHTTPClient(fs.config.UploadPartMaxTime, 100*time.Millisecond,
fs.config.SkipTLSVerify)
})
}
})
@ -1071,7 +1076,8 @@ func (fs *S3Fs) downloadToWriter(name string, w PipeWriter) (int64, error) {
d.PartSize = fs.config.DownloadPartSize
if fs.config.DownloadPartMaxTime > 0 {
d.ClientOptions = append(d.ClientOptions, func(o *s3.Options) {
o.HTTPClient = getAWSHTTPClient(fs.config.DownloadPartMaxTime, 100*time.Millisecond)
o.HTTPClient = getAWSHTTPClient(fs.config.DownloadPartMaxTime, 100*time.Millisecond,
fs.config.SkipTLSVerify)
})
}
})
@ -1096,7 +1102,7 @@ func (fs *S3Fs) getStorageID() string {
return fmt.Sprintf("s3://%v", fs.config.Bucket)
}
func getAWSHTTPClient(timeout int, idleConnectionTimeout time.Duration) *awshttp.BuildableClient {
func getAWSHTTPClient(timeout int, idleConnectionTimeout time.Duration, skipTLSVerify bool) *awshttp.BuildableClient {
c := awshttp.NewBuildableClient().
WithDialerOptions(func(d *net.Dialer) {
d.Timeout = 8 * time.Second
@ -1105,6 +1111,16 @@ func getAWSHTTPClient(timeout int, idleConnectionTimeout time.Duration) *awshttp
tr.IdleConnTimeout = idleConnectionTimeout
tr.WriteBufferSize = s3TransferBufferSize
tr.ReadBufferSize = s3TransferBufferSize
if skipTLSVerify {
if tr.TLSClientConfig != nil {
tr.TLSClientConfig.InsecureSkipVerify = skipTLSVerify
} else {
tr.TLSClientConfig = &tls.Config{
MinVersion: awshttp.DefaultHTTPTransportTLSMinVersion,
InsecureSkipVerify: skipTLSVerify,
}
}
}
})
if timeout > 0 {
c = c.WithTimeout(time.Duration(timeout) * time.Second)

4
internal/vfs/vfs.go
View file

@ -262,10 +262,12 @@ func (c *S3FsConfig) isEqual(other S3FsConfig) bool {
if !c.areMultipartFieldsEqual(other) {
return false
}
if c.ForcePathStyle != other.ForcePathStyle {
return false
}
if c.SkipTLSVerify != other.SkipTLSVerify {
return false
}
return c.isSecretEqual(other)
}

11
templates/webadmin/fsconfig.html
View file

@ -232,6 +232,17 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
</div>
</div>
<div class="form-group fsconfig fsconfig-s3fs">
<div class="form-check">
<input type="checkbox" class="form-check-input" id="idS3SkipTLSVerify" name="s3_skip_tls_verify"
{{if .S3Config.SkipTLSVerify}}checked{{end}}>
<label for="idS3SkipTLSVerify" class="form-check-label" aria-describedby="S3SkipTLSVerifyHelpBlock">Skip TLS verify</label>
<small id="S3SkipTLSVerifyHelpBlock" class="form-text text-muted">
In this mode, TLS is susceptible to man-in-the-middle attacks. This should be used only for testing
</small>
</div>
</div>
<div class="form-group row fsconfig fsconfig-gcsfs">
<label for="idGCSBucket" class="col-sm-2 col-form-label">Bucket</label>
<div class="col-sm-10">