mirror of
https://github.com/drakkan/sftpgo.git
synced 2024-11-25 09:00:27 +00:00
s3: allow to skip TLS verification
Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
This commit is contained in:
parent
9456884584
commit
654ce2e349
11 changed files with 65 additions and 9 deletions
|
@ -130,6 +130,13 @@ Flags:
|
|||
prefix and its contents
|
||||
--s3-region string
|
||||
--s3-role-arn string
|
||||
--s3-skip-tls-verify If enabled the S3 client accepts any TLS
|
||||
certificate presented by the server and
|
||||
any host name in that certificate.
|
||||
In this mode, TLS is susceptible to
|
||||
man-in-the-middle attacks.
|
||||
This should be used only for testing.
|
||||
|
||||
--s3-storage-class string
|
||||
--s3-upload-concurrency int How many parts are uploaded in
|
||||
parallel (default 2)
|
||||
|
|
2
go.mod
2
go.mod
|
@ -53,7 +53,7 @@ require (
|
|||
github.com/rs/cors v1.10.1
|
||||
github.com/rs/xid v1.5.0
|
||||
github.com/rs/zerolog v1.31.0
|
||||
github.com/sftpgo/sdk v0.1.6-0.20231011150824-e8d35102c725
|
||||
github.com/sftpgo/sdk v0.1.6-0.20231105181545-b44c8058fc25
|
||||
github.com/shirou/gopsutil/v3 v3.23.10
|
||||
github.com/spf13/afero v1.10.0
|
||||
github.com/spf13/cobra v1.8.0
|
||||
|
|
4
go.sum
4
go.sum
|
@ -445,8 +445,8 @@ github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJ
|
|||
github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY=
|
||||
github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
|
||||
github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
|
||||
github.com/sftpgo/sdk v0.1.6-0.20231011150824-e8d35102c725 h1:bV58DOKSIzTqz+UI3q81kRRvsaV0p8b2189nOy7vn7Q=
|
||||
github.com/sftpgo/sdk v0.1.6-0.20231011150824-e8d35102c725/go.mod h1:KE3KY7v13gSR+8PJMp5LYKQaWELafEM2ZPhmAo7rclM=
|
||||
github.com/sftpgo/sdk v0.1.6-0.20231105181545-b44c8058fc25 h1:R8cTb41ZX5WSYw8q8ufTKQfOvXh7aLQWqdnteDY/96U=
|
||||
github.com/sftpgo/sdk v0.1.6-0.20231105181545-b44c8058fc25/go.mod h1:6s/PFoLUd7FXG3wGlrdVhrA0SJOwri2h9kzTph/2oiU=
|
||||
github.com/shirou/gopsutil/v3 v3.23.10 h1:/N42opWlYzegYaVkWejXWJpbzKv2JDy3mrgGzKsh9hM=
|
||||
github.com/shirou/gopsutil/v3 v3.23.10/go.mod h1:JIE26kpucQi+innVlAUnIEOSBhBUkirr5b44yr55+WE=
|
||||
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
|
||||
|
|
|
@ -65,6 +65,7 @@ var (
|
|||
portableS3ULPartSize int
|
||||
portableS3ULConcurrency int
|
||||
portableS3ForcePathStyle bool
|
||||
portableS3SkipTLSVerify bool
|
||||
portableGCSBucket string
|
||||
portableGCSCredentialsFile string
|
||||
portableGCSAutoCredentials int
|
||||
|
@ -240,6 +241,7 @@ Please take a look at the usage below to customize the serving parameters`,
|
|||
UploadPartSize: int64(portableS3ULPartSize),
|
||||
UploadConcurrency: portableS3ULConcurrency,
|
||||
ForcePathStyle: portableS3ForcePathStyle,
|
||||
SkipTLSVerify: portableS3SkipTLSVerify,
|
||||
},
|
||||
AccessSecret: kms.NewPlainSecret(portableS3AccessSecret),
|
||||
},
|
||||
|
@ -373,6 +375,13 @@ prefix and its contents`)
|
|||
portableCmd.Flags().IntVar(&portableS3ULConcurrency, "s3-upload-concurrency", 2, `How many parts are uploaded in
|
||||
parallel`)
|
||||
portableCmd.Flags().BoolVar(&portableS3ForcePathStyle, "s3-force-path-style", false, `Force path style bucket URL`)
|
||||
portableCmd.Flags().BoolVar(&portableS3SkipTLSVerify, "s3-skip-tls-verify", false, `If enabled the S3 client accepts any TLS
|
||||
certificate presented by the server and
|
||||
any host name in that certificate.
|
||||
In this mode, TLS is susceptible to
|
||||
man-in-the-middle attacks.
|
||||
This should be used only for testing.
|
||||
`)
|
||||
portableCmd.Flags().StringVar(&portableGCSBucket, "gcs-bucket", "", "")
|
||||
portableCmd.Flags().StringVar(&portableGCSStorageClass, "gcs-storage-class", "", "")
|
||||
portableCmd.Flags().StringVar(&portableGCSKeyPrefix, "gcs-key-prefix", "", `Allows to restrict access to the
|
||||
|
|
|
@ -5215,6 +5215,7 @@ func TestUserS3Config(t *testing.T) {
|
|||
user.FsConfig.S3Config.DownloadPartMaxTime = 60
|
||||
user.FsConfig.S3Config.UploadPartMaxTime = 40
|
||||
user.FsConfig.S3Config.ForcePathStyle = true
|
||||
user.FsConfig.S3Config.SkipTLSVerify = true
|
||||
user.FsConfig.S3Config.DownloadPartSize = 6
|
||||
folderName := "vfolderName"
|
||||
user.VirtualFolders = append(user.VirtualFolders, vfs.VirtualFolder{
|
||||
|
@ -5243,6 +5244,7 @@ func TestUserS3Config(t *testing.T) {
|
|||
assert.Empty(t, user.FsConfig.S3Config.AccessSecret.GetKey())
|
||||
assert.Equal(t, 60, user.FsConfig.S3Config.DownloadPartMaxTime)
|
||||
assert.Equal(t, 40, user.FsConfig.S3Config.UploadPartMaxTime)
|
||||
assert.True(t, user.FsConfig.S3Config.SkipTLSVerify)
|
||||
if assert.Len(t, user.VirtualFolders, 1) {
|
||||
folder := user.VirtualFolders[0]
|
||||
assert.Equal(t, sdkkms.SecretStatusSecretBox, folder.FsConfig.CryptConfig.Passphrase.GetStatus())
|
||||
|
@ -20957,6 +20959,7 @@ func TestWebUserS3Mock(t *testing.T) {
|
|||
user.FsConfig.S3Config.DownloadPartSize = 6
|
||||
user.FsConfig.S3Config.DownloadConcurrency = 3
|
||||
user.FsConfig.S3Config.ForcePathStyle = true
|
||||
user.FsConfig.S3Config.SkipTLSVerify = true
|
||||
user.FsConfig.S3Config.ACL = "public-read"
|
||||
user.Description = "s3 tèst user"
|
||||
form := make(url.Values)
|
||||
|
@ -21005,6 +21008,7 @@ func TestWebUserS3Mock(t *testing.T) {
|
|||
form.Set("password_strength", "0")
|
||||
form.Set("ftp_security", "1")
|
||||
form.Set("s3_force_path_style", "checked")
|
||||
form.Set("s3_skip_tls_verify", "checked")
|
||||
form.Set("description", user.Description)
|
||||
form.Add("hooks", "pre_login_disabled")
|
||||
form.Add("allow_api_key_auth", "1")
|
||||
|
@ -21093,6 +21097,7 @@ func TestWebUserS3Mock(t *testing.T) {
|
|||
assert.Equal(t, updateUser.FsConfig.S3Config.DownloadConcurrency, user.FsConfig.S3Config.DownloadConcurrency)
|
||||
assert.Equal(t, lastPwdChange, updateUser.LastPasswordChange)
|
||||
assert.True(t, updateUser.FsConfig.S3Config.ForcePathStyle)
|
||||
assert.True(t, updateUser.FsConfig.S3Config.SkipTLSVerify)
|
||||
if assert.Equal(t, 2, len(updateUser.Filters.FilePatterns)) {
|
||||
for _, filter := range updateUser.Filters.FilePatterns {
|
||||
switch filter.Path {
|
||||
|
@ -23538,6 +23543,7 @@ func TestS3WebFolderMock(t *testing.T) {
|
|||
assert.Equal(t, S3DownloadConcurrency, folder.FsConfig.S3Config.DownloadConcurrency)
|
||||
assert.Equal(t, int64(S3DownloadPartSize), folder.FsConfig.S3Config.DownloadPartSize)
|
||||
assert.False(t, folder.FsConfig.S3Config.ForcePathStyle)
|
||||
assert.False(t, folder.FsConfig.S3Config.SkipTLSVerify)
|
||||
// update
|
||||
S3UploadConcurrency = 10
|
||||
form.Set("s3_upload_concurrency", "b")
|
||||
|
|
|
@ -1588,6 +1588,7 @@ func getS3Config(r *http.Request) (vfs.S3FsConfig, error) {
|
|||
return config, fmt.Errorf("invalid s3 download concurrency: %w", err)
|
||||
}
|
||||
config.ForcePathStyle = r.Form.Get("s3_force_path_style") != ""
|
||||
config.SkipTLSVerify = r.Form.Get("s3_skip_tls_verify") != ""
|
||||
config.DownloadPartMaxTime, err = strconv.Atoi(r.Form.Get("s3_download_part_max_time"))
|
||||
if err != nil {
|
||||
return config, fmt.Errorf("invalid s3 download part max time: %w", err)
|
||||
|
|
|
@ -2199,6 +2199,9 @@ func compareS3Config(expected *vfs.Filesystem, actual *vfs.Filesystem) error { /
|
|||
if expected.S3Config.ForcePathStyle != actual.S3Config.ForcePathStyle {
|
||||
return errors.New("fs S3 force path style mismatch")
|
||||
}
|
||||
if expected.S3Config.SkipTLSVerify != actual.S3Config.SkipTLSVerify {
|
||||
return errors.New("fs S3 skip TLS verify mismatch")
|
||||
}
|
||||
if expected.S3Config.DownloadPartMaxTime != actual.S3Config.DownloadPartMaxTime {
|
||||
return errors.New("fs S3 download part max time mismatch")
|
||||
}
|
||||
|
|
|
@ -321,6 +321,7 @@ func (f *Filesystem) GetACopy() Filesystem {
|
|||
DownloadPartMaxTime: f.S3Config.DownloadPartMaxTime,
|
||||
UploadPartMaxTime: f.S3Config.UploadPartMaxTime,
|
||||
ForcePathStyle: f.S3Config.ForcePathStyle,
|
||||
SkipTLSVerify: f.S3Config.SkipTLSVerify,
|
||||
},
|
||||
AccessSecret: f.S3Config.AccessSecret.Clone(),
|
||||
},
|
||||
|
|
|
@ -19,6 +19,7 @@ package vfs
|
|||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"mime"
|
||||
|
@ -97,7 +98,9 @@ func NewS3Fs(connectionID, localTempDir, mountPath string, s3Config S3FsConfig)
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
awsConfig, err := config.LoadDefaultConfig(ctx, config.WithHTTPClient(getAWSHTTPClient(0, 30*time.Second)))
|
||||
awsConfig, err := config.LoadDefaultConfig(ctx, config.WithHTTPClient(
|
||||
getAWSHTTPClient(0, 30*time.Second, fs.config.SkipTLSVerify)),
|
||||
)
|
||||
if err != nil {
|
||||
return fs, fmt.Errorf("unable to get AWS config: %w", err)
|
||||
}
|
||||
|
@ -215,7 +218,8 @@ func (fs *S3Fs) Open(name string, offset int64) (File, *PipeReader, func(), erro
|
|||
d.PartSize = fs.config.DownloadPartSize
|
||||
if offset == 0 && fs.config.DownloadPartMaxTime > 0 {
|
||||
d.ClientOptions = append(d.ClientOptions, func(o *s3.Options) {
|
||||
o.HTTPClient = getAWSHTTPClient(fs.config.DownloadPartMaxTime, 100*time.Millisecond)
|
||||
o.HTTPClient = getAWSHTTPClient(fs.config.DownloadPartMaxTime, 100*time.Millisecond,
|
||||
fs.config.SkipTLSVerify)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
@ -264,7 +268,8 @@ func (fs *S3Fs) Create(name string, flag, checks int) (File, PipeWriter, func(),
|
|||
u.PartSize = fs.config.UploadPartSize
|
||||
if fs.config.UploadPartMaxTime > 0 {
|
||||
u.ClientOptions = append(u.ClientOptions, func(o *s3.Options) {
|
||||
o.HTTPClient = getAWSHTTPClient(fs.config.UploadPartMaxTime, 100*time.Millisecond)
|
||||
o.HTTPClient = getAWSHTTPClient(fs.config.UploadPartMaxTime, 100*time.Millisecond,
|
||||
fs.config.SkipTLSVerify)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
@ -1071,7 +1076,8 @@ func (fs *S3Fs) downloadToWriter(name string, w PipeWriter) (int64, error) {
|
|||
d.PartSize = fs.config.DownloadPartSize
|
||||
if fs.config.DownloadPartMaxTime > 0 {
|
||||
d.ClientOptions = append(d.ClientOptions, func(o *s3.Options) {
|
||||
o.HTTPClient = getAWSHTTPClient(fs.config.DownloadPartMaxTime, 100*time.Millisecond)
|
||||
o.HTTPClient = getAWSHTTPClient(fs.config.DownloadPartMaxTime, 100*time.Millisecond,
|
||||
fs.config.SkipTLSVerify)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
@ -1096,7 +1102,7 @@ func (fs *S3Fs) getStorageID() string {
|
|||
return fmt.Sprintf("s3://%v", fs.config.Bucket)
|
||||
}
|
||||
|
||||
func getAWSHTTPClient(timeout int, idleConnectionTimeout time.Duration) *awshttp.BuildableClient {
|
||||
func getAWSHTTPClient(timeout int, idleConnectionTimeout time.Duration, skipTLSVerify bool) *awshttp.BuildableClient {
|
||||
c := awshttp.NewBuildableClient().
|
||||
WithDialerOptions(func(d *net.Dialer) {
|
||||
d.Timeout = 8 * time.Second
|
||||
|
@ -1105,6 +1111,16 @@ func getAWSHTTPClient(timeout int, idleConnectionTimeout time.Duration) *awshttp
|
|||
tr.IdleConnTimeout = idleConnectionTimeout
|
||||
tr.WriteBufferSize = s3TransferBufferSize
|
||||
tr.ReadBufferSize = s3TransferBufferSize
|
||||
if skipTLSVerify {
|
||||
if tr.TLSClientConfig != nil {
|
||||
tr.TLSClientConfig.InsecureSkipVerify = skipTLSVerify
|
||||
} else {
|
||||
tr.TLSClientConfig = &tls.Config{
|
||||
MinVersion: awshttp.DefaultHTTPTransportTLSMinVersion,
|
||||
InsecureSkipVerify: skipTLSVerify,
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
if timeout > 0 {
|
||||
c = c.WithTimeout(time.Duration(timeout) * time.Second)
|
||||
|
|
|
@ -262,10 +262,12 @@ func (c *S3FsConfig) isEqual(other S3FsConfig) bool {
|
|||
if !c.areMultipartFieldsEqual(other) {
|
||||
return false
|
||||
}
|
||||
|
||||
if c.ForcePathStyle != other.ForcePathStyle {
|
||||
return false
|
||||
}
|
||||
if c.SkipTLSVerify != other.SkipTLSVerify {
|
||||
return false
|
||||
}
|
||||
return c.isSecretEqual(other)
|
||||
}
|
||||
|
||||
|
|
|
@ -232,6 +232,17 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-group fsconfig fsconfig-s3fs">
|
||||
<div class="form-check">
|
||||
<input type="checkbox" class="form-check-input" id="idS3SkipTLSVerify" name="s3_skip_tls_verify"
|
||||
{{if .S3Config.SkipTLSVerify}}checked{{end}}>
|
||||
<label for="idS3SkipTLSVerify" class="form-check-label" aria-describedby="S3SkipTLSVerifyHelpBlock">Skip TLS verify</label>
|
||||
<small id="S3SkipTLSVerifyHelpBlock" class="form-text text-muted">
|
||||
In this mode, TLS is susceptible to man-in-the-middle attacks. This should be used only for testing
|
||||
</small>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-group row fsconfig fsconfig-gcsfs">
|
||||
<label for="idGCSBucket" class="col-sm-2 col-form-label">Bucket</label>
|
||||
<div class="col-sm-10">
|
||||
|
|
Loading…
Reference in a new issue