From adad8e658b2fc8e22a58c1346dd6fa148c917a42 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Tue, 16 May 2023 18:08:14 +0200 Subject: [PATCH] osfs: add optional buffering Signed-off-by: Nicola Murino --- go.mod | 4 +- go.sum | 9 +- internal/common/actions_test.go | 2 +- internal/common/common_test.go | 2 +- internal/common/connection_test.go | 12 +- internal/common/transfer.go | 22 +-- internal/common/transfer_test.go | 16 +- internal/dataprovider/dataprovider.go | 7 +- internal/dataprovider/user.go | 51 ++++-- internal/ftpd/cryptfs_test.go | 45 ++++++ internal/ftpd/ftpd_test.go | 15 +- internal/ftpd/internal_test.go | 6 +- internal/httpd/httpd_test.go | 214 +++++++++++++++++++++++++- internal/httpd/webadmin.go | 16 ++ internal/httpdtest/httpdtest.go | 12 ++ internal/sftpd/internal_test.go | 20 +-- internal/sftpd/sftpd_test.go | 147 +++++++++++++++++- internal/vfs/azblobfs.go | 2 +- internal/vfs/cryptfs.go | 58 +++++-- internal/vfs/filesystem.go | 19 ++- internal/vfs/folder.go | 2 +- internal/vfs/gcsfs.go | 2 +- internal/vfs/httpfs.go | 2 +- internal/vfs/osfs.go | 121 +++++++++++++-- internal/vfs/s3fs.go | 2 +- internal/vfs/sftpfs.go | 36 +---- internal/vfs/vfs.go | 64 +++++++- internal/webdavd/file.go | 2 +- internal/webdavd/internal_test.go | 22 +-- internal/webdavd/webdavd_test.go | 69 ++++++++- openapi/openapi.yaml | 25 +++ templates/webadmin/fsconfig.html | 39 +++++ 32 files changed, 895 insertions(+), 170 deletions(-) diff --git a/go.mod b/go.mod index 299414a6..2cb1e65a 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,7 @@ require ( github.com/rs/cors v1.9.0 github.com/rs/xid v1.5.0 github.com/rs/zerolog v1.29.1 - github.com/sftpgo/sdk v0.1.4-0.20230512160325-38e59551f700 + github.com/sftpgo/sdk v0.1.4-0.20230514135418-8b5d36c556e0 github.com/shirou/gopsutil/v3 v3.23.4 github.com/spf13/afero v1.9.5 github.com/spf13/cobra v1.7.0 @@ -148,7 +148,7 @@ require ( github.com/prometheus/procfs v0.9.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect - github.com/spf13/cast v1.5.0 // indirect + github.com/spf13/cast v1.5.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect diff --git a/go.sum b/go.sum index 7281f60a..b2a19b0f 100644 --- a/go.sum +++ b/go.sum @@ -935,8 +935,8 @@ github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHqu github.com/franela/goblin v0.0.0-20210519012713-85d372ac71e2/go.mod h1:VzmDKDJVZI3aJmnRI9VjAn9nJ8qPPsN1fqzr9dqInIo= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= @@ -1842,8 +1842,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= -github.com/sftpgo/sdk v0.1.4-0.20230512160325-38e59551f700 h1:jL6mfKAaFv862AnBUxIfTH9wmnuPjbWyjHQUGDo+Xt0= -github.com/sftpgo/sdk v0.1.4-0.20230512160325-38e59551f700/go.mod h1:gDxDaU3rhp9Y92ddsE7SbQ8jdBNNWK1DKlp5eHXrsb8= +github.com/sftpgo/sdk v0.1.4-0.20230514135418-8b5d36c556e0 h1:qUyCryFuF7zKUpNTLgn2KuQp/uYT3hKi4XU/ChJUmpo= +github.com/sftpgo/sdk v0.1.4-0.20230514135418-8b5d36c556e0/go.mod h1:TjeoMWS0JEXt9RukJveTnaiHj4+MVLtUiDC+mY++Odk= github.com/shirou/gopsutil/v3 v3.23.4 h1:hZwmDxZs7Ewt75DV81r4pFMqbq+di2cbt9FsQBqLD2o= github.com/shirou/gopsutil/v3 v3.23.4/go.mod h1:ZcGxyfzAMRevhUR2+cfhXDH6gQdFYE/t8j1nsU4mPI8= github.com/shoenig/go-m1cpu v0.1.5/go.mod h1:Wwvst4LR89UxjeFtLRMrpgRiyY4xPsejnVZym39dbAQ= @@ -1883,8 +1883,9 @@ github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= +github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= +github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= diff --git a/internal/common/actions_test.go b/internal/common/actions_test.go index 983f170f..e980c3bf 100644 --- a/internal/common/actions_test.go +++ b/internal/common/actions_test.go @@ -262,7 +262,7 @@ func TestPreDeleteAction(t *testing.T) { } user.Permissions = make(map[string][]string) user.Permissions["/"] = []string{dataprovider.PermAny} - fs := vfs.NewOsFs("id", homeDir, "") + fs := vfs.NewOsFs("id", homeDir, "", nil) c := NewBaseConnection("id", ProtocolSFTP, "", "", user) testfile := filepath.Join(user.HomeDir, "testfile") diff --git a/internal/common/common_test.go b/internal/common/common_test.go index 885d13d1..cbb311d8 100644 --- a/internal/common/common_test.go +++ b/internal/common/common_test.go @@ -861,7 +861,7 @@ func TestConnectionStatus(t *testing.T) { Username: username, }, } - fs := vfs.NewOsFs("", os.TempDir(), "") + fs := vfs.NewOsFs("", os.TempDir(), "", nil) c1 := NewBaseConnection("id1", ProtocolSFTP, "", "", user) fakeConn1 := &fakeConnection{ BaseConnection: c1, diff --git a/internal/common/connection_test.go b/internal/common/connection_test.go index c6cdbefb..d680f3ce 100644 --- a/internal/common/connection_test.go +++ b/internal/common/connection_test.go @@ -86,7 +86,7 @@ func (fs *MockOsFs) Walk(_ string, walkFn filepath.WalkFunc) error { func newMockOsFs(hasVirtualFolders bool, connectionID, rootDir, name string, err error) vfs.Fs { return &MockOsFs{ - Fs: vfs.NewOsFs(connectionID, rootDir, ""), + Fs: vfs.NewOsFs(connectionID, rootDir, "", nil), name: name, hasVirtualFolders: hasVirtualFolders, err: err, @@ -114,7 +114,7 @@ func TestRemoveErrors(t *testing.T) { } user.Permissions = make(map[string][]string) user.Permissions["/"] = []string{dataprovider.PermAny} - fs := vfs.NewOsFs("", os.TempDir(), "") + fs := vfs.NewOsFs("", os.TempDir(), "", nil) conn := NewBaseConnection("", ProtocolFTP, "", "", user) err := conn.IsRemoveDirAllowed(fs, mappedPath, "/virtualpath1") if assert.Error(t, err) { @@ -159,7 +159,7 @@ func TestSetStatMode(t *testing.T) { } func TestRecursiveRenameWalkError(t *testing.T) { - fs := vfs.NewOsFs("", filepath.Clean(os.TempDir()), "") + fs := vfs.NewOsFs("", filepath.Clean(os.TempDir()), "", nil) conn := NewBaseConnection("", ProtocolWebDAV, "", "", dataprovider.User{ BaseUser: sdk.BaseUser{ Permissions: map[string][]string{ @@ -193,7 +193,7 @@ func TestRecursiveRenameWalkError(t *testing.T) { } func TestCrossRenameFsErrors(t *testing.T) { - fs := vfs.NewOsFs("", os.TempDir(), "") + fs := vfs.NewOsFs("", os.TempDir(), "", nil) conn := NewBaseConnection("", ProtocolWebDAV, "", "", dataprovider.User{}) res := conn.hasSpaceForCrossRename(fs, vfs.QuotaCheckResult{}, 1, "missingsource") assert.False(t, res) @@ -224,7 +224,7 @@ func TestRenameVirtualFolders(t *testing.T) { }, VirtualPath: vdir, }) - fs := vfs.NewOsFs("", os.TempDir(), "") + fs := vfs.NewOsFs("", os.TempDir(), "", nil) conn := NewBaseConnection("", ProtocolFTP, "", "", u) res := conn.isRenamePermitted(fs, fs, "source", "target", vdir, "vdirtarget", nil) assert.False(t, res) @@ -376,7 +376,7 @@ func TestUpdateQuotaAfterRename(t *testing.T) { } func TestErrorsMapping(t *testing.T) { - fs := vfs.NewOsFs("", os.TempDir(), "") + fs := vfs.NewOsFs("", os.TempDir(), "", nil) conn := NewBaseConnection("", ProtocolSFTP, "", "", dataprovider.User{BaseUser: sdk.BaseUser{HomeDir: os.TempDir()}}) osErrorsProtocols := []string{ProtocolWebDAV, ProtocolFTP, ProtocolHTTP, ProtocolHTTPShare, ProtocolDataRetention, ProtocolOIDC, protocolEventAction} diff --git a/internal/common/transfer.go b/internal/common/transfer.go index 0a0a7cbb..5e02afbe 100644 --- a/internal/common/transfer.go +++ b/internal/common/transfer.go @@ -198,8 +198,8 @@ func (t *BaseTransfer) SetTimes(fsPath string, atime time.Time, mtime time.Time) // If atomic uploads are enabled this differ from fsPath func (t *BaseTransfer) GetRealFsPath(fsPath string) string { if fsPath == t.GetFsPath() { - if t.File != nil { - return t.File.Name() + if t.File != nil || vfs.IsLocalOsFs(t.Fs) { + return t.effectiveFsPath } return t.fsPath } @@ -289,9 +289,9 @@ func (t *BaseTransfer) Truncate(fsPath string, size int64) (int64, error) { return initialSize, err } if size == 0 && t.BytesSent.Load() == 0 { - // for cloud providers the file is always truncated to zero, we don't support append/resume for uploads - // for buffered SFTP we can have buffered bytes so we returns an error - if !vfs.IsBufferedSFTPFs(t.Fs) { + // for cloud providers the file is always truncated to zero, we don't support append/resume for uploads. + // For buffered SFTP and local fs we can have buffered bytes so we returns an error + if !vfs.IsBufferedLocalOrSFTPFs(t.Fs) { return 0, nil } } @@ -373,16 +373,16 @@ func (t *BaseTransfer) Close() error { dataprovider.UpdateUserTransferQuota(&t.Connection.User, t.BytesReceived.Load(), //nolint:errcheck t.BytesSent.Load(), false) } - if t.File != nil && t.Connection.IsQuotaExceededError(t.ErrTransfer) { + if (t.File != nil || vfs.IsLocalOsFs(t.Fs)) && t.Connection.IsQuotaExceededError(t.ErrTransfer) { // if quota is exceeded we try to remove the partial file for uploads to local filesystem - err = t.Fs.Remove(t.File.Name(), false) + err = t.Fs.Remove(t.effectiveFsPath, false) if err == nil { t.BytesReceived.Store(0) t.MinWriteOffset = 0 } t.Connection.Log(logger.LevelWarn, "upload denied due to space limit, delete temporary file: %q, deletion error: %v", - t.File.Name(), err) - } else if t.transferType == TransferUpload && t.effectiveFsPath != t.fsPath { + t.effectiveFsPath, err) + } else if t.isAtomicUpload() { if t.ErrTransfer == nil || Config.UploadMode == UploadModeAtomicWithResume { _, _, err = t.Fs.Rename(t.effectiveFsPath, t.fsPath) t.Connection.Log(logger.LevelDebug, "atomic upload completed, rename: %q -> %q, error: %v", @@ -436,6 +436,10 @@ func (t *BaseTransfer) Close() error { return err } +func (t *BaseTransfer) isAtomicUpload() bool { + return t.transferType == TransferUpload && t.effectiveFsPath != t.fsPath +} + func (t *BaseTransfer) updateTransferTimestamps(uploadFileSize, elapsed int64) { if t.ErrTransfer != nil { return diff --git a/internal/common/transfer_test.go b/internal/common/transfer_test.go index dcd2eac5..c3440e59 100644 --- a/internal/common/transfer_test.go +++ b/internal/common/transfer_test.go @@ -36,7 +36,7 @@ func TestTransferUpdateQuota(t *testing.T) { transfer := BaseTransfer{ Connection: conn, transferType: TransferUpload, - Fs: vfs.NewOsFs("", os.TempDir(), ""), + Fs: vfs.NewOsFs("", os.TempDir(), "", nil), } transfer.BytesReceived.Store(123) errFake := errors.New("fake error") @@ -75,7 +75,7 @@ func TestTransferThrottling(t *testing.T) { DownloadBandwidth: 40, }, } - fs := vfs.NewOsFs("", os.TempDir(), "") + fs := vfs.NewOsFs("", os.TempDir(), "", nil) testFileSize := int64(131072) wantedUploadElapsed := 1000 * (testFileSize / 1024) / u.UploadBandwidth wantedDownloadElapsed := 1000 * (testFileSize / 1024) / u.DownloadBandwidth @@ -107,7 +107,7 @@ func TestTransferThrottling(t *testing.T) { func TestRealPath(t *testing.T) { testFile := filepath.Join(os.TempDir(), "afile.txt") - fs := vfs.NewOsFs("123", os.TempDir(), "") + fs := vfs.NewOsFs("123", os.TempDir(), "", nil) u := dataprovider.User{ BaseUser: sdk.BaseUser{ Username: "user", @@ -141,7 +141,7 @@ func TestRealPath(t *testing.T) { func TestTruncate(t *testing.T) { testFile := filepath.Join(os.TempDir(), "transfer_test_file") - fs := vfs.NewOsFs("123", os.TempDir(), "") + fs := vfs.NewOsFs("123", os.TempDir(), "", nil) u := dataprovider.User{ BaseUser: sdk.BaseUser{ Username: "user", @@ -210,7 +210,7 @@ func TestTransferErrors(t *testing.T) { isCancelled = true } testFile := filepath.Join(os.TempDir(), "transfer_test_file") - fs := vfs.NewOsFs("id", os.TempDir(), "") + fs := vfs.NewOsFs("id", os.TempDir(), "", nil) u := dataprovider.User{ BaseUser: sdk.BaseUser{ Username: "test", @@ -321,7 +321,7 @@ func TestFTPMode(t *testing.T) { transfer := BaseTransfer{ Connection: conn, transferType: TransferUpload, - Fs: vfs.NewOsFs("", os.TempDir(), ""), + Fs: vfs.NewOsFs("", os.TempDir(), "", nil), } transfer.BytesReceived.Store(123) assert.Empty(t, transfer.ftpMode) @@ -399,7 +399,7 @@ func TestTransferQuota(t *testing.T) { conn := NewBaseConnection("", ProtocolSFTP, "", "", user) transfer := NewBaseTransfer(nil, conn, nil, "file.txt", "file.txt", "/transfer_test_file", TransferUpload, - 0, 0, 0, 0, true, vfs.NewOsFs("", os.TempDir(), ""), dataprovider.TransferQuota{}) + 0, 0, 0, 0, true, vfs.NewOsFs("", os.TempDir(), "", nil), dataprovider.TransferQuota{}) err := transfer.CheckRead() assert.NoError(t, err) err = transfer.CheckWrite() @@ -453,7 +453,7 @@ func TestUploadOutsideHomeRenameError(t *testing.T) { transfer := BaseTransfer{ Connection: conn, transferType: TransferUpload, - Fs: vfs.NewOsFs("", filepath.Join(os.TempDir(), "home"), ""), + Fs: vfs.NewOsFs("", filepath.Join(os.TempDir(), "home"), "", nil), } transfer.BytesReceived.Store(123) diff --git a/internal/dataprovider/dataprovider.go b/internal/dataprovider/dataprovider.go index 6edd43b9..7f8eef2f 100644 --- a/internal/dataprovider/dataprovider.go +++ b/internal/dataprovider/dataprovider.go @@ -2622,11 +2622,12 @@ func getVirtualFolderIfInvalid(folder *vfs.BaseVirtualFolder) *vfs.BaseVirtualFo if err := ValidateFolder(folder); err == nil { return folder } - // we try to get the folder from the data provider if only the Name is populated - if folder.MappedPath != "" { + if folder.Name == "" { return folder } - if folder.Name == "" { + // we try to get the folder from the data provider if only the Name is populated + // so if MappedPath or Provider are set just return + if folder.MappedPath != "" { return folder } if folder.FsConfig.Provider != sdk.LocalFilesystemProvider { diff --git a/internal/dataprovider/user.go b/internal/dataprovider/user.go index 9cb020f3..5d53e38f 100644 --- a/internal/dataprovider/user.go +++ b/internal/dataprovider/user.go @@ -173,7 +173,7 @@ func (u *User) getRootFs(connectionID string) (fs vfs.Fs, err error) { case sdk.HTTPFilesystemProvider: return vfs.NewHTTPFs(connectionID, u.GetHomeDir(), "", u.FsConfig.HTTPConfig) default: - return vfs.NewOsFs(connectionID, u.GetHomeDir(), ""), nil + return vfs.NewOsFs(connectionID, u.GetHomeDir(), "", &u.FsConfig.OSConfig), nil } } @@ -218,7 +218,7 @@ func (u *User) checkLocalHomeDir(connectionID string) { case sdk.LocalFilesystemProvider, sdk.CryptedFilesystemProvider: return default: - osFs := vfs.NewOsFs(connectionID, u.GetHomeDir(), "") + osFs := vfs.NewOsFs(connectionID, u.GetHomeDir(), "", nil) osFs.CheckRootPath(u.Username, u.GetUID(), u.GetGID()) } } @@ -1631,7 +1631,7 @@ func (u *User) applyGroupSettings(groupsMapping map[string]Group) { for _, g := range u.Groups { if g.Type == sdk.GroupTypePrimary { if group, ok := groupsMapping[g.Name]; ok { - u.mergeWithPrimaryGroup(group, replacer) + u.mergeWithPrimaryGroup(&group, replacer) } else { providerLog(logger.LevelError, "mapping not found for user %s, group %s", u.Username, g.Name) } @@ -1641,7 +1641,7 @@ func (u *User) applyGroupSettings(groupsMapping map[string]Group) { for _, g := range u.Groups { if g.Type == sdk.GroupTypeSecondary { if group, ok := groupsMapping[g.Name]; ok { - u.mergeAdditiveProperties(group, sdk.GroupTypeSecondary, replacer) + u.mergeAdditiveProperties(&group, sdk.GroupTypeSecondary, replacer) } else { providerLog(logger.LevelError, "mapping not found for user %s, group %s", u.Username, g.Name) } @@ -1674,17 +1674,19 @@ func (u *User) LoadAndApplyGroupSettings() error { } replacer := u.getGroupPlacehodersReplacer() // make sure to always merge with the primary group first - for idx, g := range groups { + for idx := range groups { + g := groups[idx] if g.Name == primaryGroupName { - u.mergeWithPrimaryGroup(g, replacer) + u.mergeWithPrimaryGroup(&g, replacer) lastIdx := len(groups) - 1 groups[idx] = groups[lastIdx] groups = groups[:lastIdx] break } } - for _, g := range groups { - u.mergeAdditiveProperties(g, sdk.GroupTypeSecondary, replacer) + for idx := range groups { + g := groups[idx] + u.mergeAdditiveProperties(&g, sdk.GroupTypeSecondary, replacer) } u.removeDuplicatesAfterGroupMerge() return nil @@ -1718,12 +1720,31 @@ func (u *User) replaceFsConfigPlaceholders(fsConfig vfs.Filesystem, replacer *st return fsConfig } -func (u *User) mergeWithPrimaryGroup(group Group, replacer *strings.Replacer) { +func (u *User) mergeCryptFsConfig(group *Group) { + if group.UserSettings.FsConfig.Provider == sdk.CryptedFilesystemProvider { + if u.FsConfig.CryptConfig.ReadBufferSize == 0 { + u.FsConfig.CryptConfig.ReadBufferSize = group.UserSettings.FsConfig.CryptConfig.ReadBufferSize + } + if u.FsConfig.CryptConfig.WriteBufferSize == 0 { + u.FsConfig.CryptConfig.WriteBufferSize = group.UserSettings.FsConfig.CryptConfig.WriteBufferSize + } + } +} + +func (u *User) mergeWithPrimaryGroup(group *Group, replacer *strings.Replacer) { if group.UserSettings.HomeDir != "" { u.HomeDir = u.replacePlaceholder(group.UserSettings.HomeDir, replacer) } if group.UserSettings.FsConfig.Provider != 0 { u.FsConfig = u.replaceFsConfigPlaceholders(group.UserSettings.FsConfig, replacer) + u.mergeCryptFsConfig(group) + } else { + if u.FsConfig.OSConfig.ReadBufferSize == 0 { + u.FsConfig.OSConfig.ReadBufferSize = group.UserSettings.FsConfig.OSConfig.ReadBufferSize + } + if u.FsConfig.OSConfig.WriteBufferSize == 0 { + u.FsConfig.OSConfig.WriteBufferSize = group.UserSettings.FsConfig.OSConfig.WriteBufferSize + } } if u.MaxSessions == 0 { u.MaxSessions = group.UserSettings.MaxSessions @@ -1748,11 +1769,11 @@ func (u *User) mergeWithPrimaryGroup(group Group, replacer *strings.Replacer) { if u.ExpirationDate == 0 && group.UserSettings.ExpiresIn > 0 { u.ExpirationDate = u.CreatedAt + int64(group.UserSettings.ExpiresIn)*86400000 } - u.mergePrimaryGroupFilters(group.UserSettings.Filters, replacer) + u.mergePrimaryGroupFilters(&group.UserSettings.Filters, replacer) u.mergeAdditiveProperties(group, sdk.GroupTypePrimary, replacer) } -func (u *User) mergePrimaryGroupFilters(filters sdk.BaseUserFilters, replacer *strings.Replacer) { +func (u *User) mergePrimaryGroupFilters(filters *sdk.BaseUserFilters, replacer *strings.Replacer) { if u.Filters.MaxUploadFileSize == 0 { u.Filters.MaxUploadFileSize = filters.MaxUploadFileSize } @@ -1797,7 +1818,7 @@ func (u *User) mergePrimaryGroupFilters(filters sdk.BaseUserFilters, replacer *s } } -func (u *User) mergeAdditiveProperties(group Group, groupType int, replacer *strings.Replacer) { +func (u *User) mergeAdditiveProperties(group *Group, groupType int, replacer *strings.Replacer) { u.mergeVirtualFolders(group, groupType, replacer) u.mergePermissions(group, groupType, replacer) u.mergeFilePatterns(group, groupType, replacer) @@ -1811,7 +1832,7 @@ func (u *User) mergeAdditiveProperties(group Group, groupType int, replacer *str u.Filters.TwoFactorAuthProtocols = append(u.Filters.TwoFactorAuthProtocols, group.UserSettings.Filters.TwoFactorAuthProtocols...) } -func (u *User) mergeVirtualFolders(group Group, groupType int, replacer *strings.Replacer) { +func (u *User) mergeVirtualFolders(group *Group, groupType int, replacer *strings.Replacer) { if len(group.VirtualFolders) > 0 { folderPaths := make(map[string]bool) for _, folder := range u.VirtualFolders { @@ -1831,7 +1852,7 @@ func (u *User) mergeVirtualFolders(group Group, groupType int, replacer *strings } } -func (u *User) mergePermissions(group Group, groupType int, replacer *strings.Replacer) { +func (u *User) mergePermissions(group *Group, groupType int, replacer *strings.Replacer) { for k, v := range group.UserSettings.Permissions { if k == "/" { if groupType == sdk.GroupTypePrimary { @@ -1847,7 +1868,7 @@ func (u *User) mergePermissions(group Group, groupType int, replacer *strings.Re } } -func (u *User) mergeFilePatterns(group Group, groupType int, replacer *strings.Replacer) { +func (u *User) mergeFilePatterns(group *Group, groupType int, replacer *strings.Replacer) { if len(group.UserSettings.Filters.FilePatterns) > 0 { patternPaths := make(map[string]bool) for _, pattern := range u.Filters.FilePatterns { diff --git a/internal/ftpd/cryptfs_test.go b/internal/ftpd/cryptfs_test.go index 11549354..950d99b9 100644 --- a/internal/ftpd/cryptfs_test.go +++ b/internal/ftpd/cryptfs_test.go @@ -136,6 +136,51 @@ func TestBasicFTPHandlingCryptFs(t *testing.T) { 50*time.Millisecond) } +func TestBufferedCryptFs(t *testing.T) { + u := getTestUserWithCryptFs() + u.FsConfig.CryptConfig.OSFsConfig = sdk.OSFsConfig{ + ReadBufferSize: 1, + WriteBufferSize: 1, + } + user, _, err := httpdtest.AddUser(u, http.StatusCreated) + assert.NoError(t, err) + client, err := getFTPClient(user, true, nil) + if assert.NoError(t, err) { + testFilePath := filepath.Join(homeBasePath, testFileName) + testFileSize := int64(65535) + err = createTestFile(testFilePath, testFileSize) + assert.NoError(t, err) + + err = checkBasicFTP(client) + assert.NoError(t, err) + err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0) + assert.NoError(t, err) + // overwrite an existing file + err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0) + assert.NoError(t, err) + localDownloadPath := filepath.Join(homeBasePath, testDLFileName) + err = ftpDownloadFile(testFileName, localDownloadPath, testFileSize, client, 0) + assert.NoError(t, err) + info, err := os.Stat(localDownloadPath) + if assert.NoError(t, err) { + assert.Equal(t, testFileSize, info.Size()) + } + err = os.Remove(testFilePath) + assert.NoError(t, err) + err = os.Remove(localDownloadPath) + assert.NoError(t, err) + err = client.Quit() + assert.NoError(t, err) + } + _, err = httpdtest.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) + assert.Eventually(t, func() bool { return len(common.Connections.GetStats("")) == 0 }, 1*time.Second, 50*time.Millisecond) + assert.Eventually(t, func() bool { return common.Connections.GetClientConnections() == 0 }, 1000*time.Millisecond, + 50*time.Millisecond) +} + func TestZeroBytesTransfersCryptFs(t *testing.T) { u := getTestUserWithCryptFs() user, _, err := httpdtest.AddUser(u, http.StatusCreated) diff --git a/internal/ftpd/ftpd_test.go b/internal/ftpd/ftpd_test.go index cc935e63..5a9e68e7 100644 --- a/internal/ftpd/ftpd_test.go +++ b/internal/ftpd/ftpd_test.go @@ -2083,7 +2083,16 @@ func TestResume(t *testing.T) { assert.NoError(t, err) sftpUser, _, err := httpdtest.AddUser(getTestSFTPUser(), http.StatusCreated) assert.NoError(t, err) - for _, user := range []dataprovider.User{localUser, sftpUser} { + u = getTestUser() + u.FsConfig.OSConfig = sdk.OSFsConfig{ + ReadBufferSize: 1, + WriteBufferSize: 1, + } + u.Username += "_buf" + u.HomeDir += "_buf" + bufferedUser, _, err := httpdtest.AddUser(u, http.StatusCreated) + assert.NoError(t, err) + for _, user := range []dataprovider.User{localUser, sftpUser, bufferedUser} { client, err := getFTPClient(user, true, nil) if assert.NoError(t, err) { testFilePath := filepath.Join(homeBasePath, testFileName) @@ -2166,6 +2175,10 @@ func TestResume(t *testing.T) { assert.NoError(t, err) err = os.RemoveAll(localUser.GetHomeDir()) assert.NoError(t, err) + _, err = httpdtest.RemoveUser(bufferedUser, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(bufferedUser.GetHomeDir()) + assert.NoError(t, err) } //nolint:dupl diff --git a/internal/ftpd/internal_test.go b/internal/ftpd/internal_test.go index 6999182a..3d88e4cd 100644 --- a/internal/ftpd/internal_test.go +++ b/internal/ftpd/internal_test.go @@ -398,7 +398,7 @@ func (fs MockOsFs) Rename(source, target string) (int, int64, error) { func newMockOsFs(err, statErr error, atomicUpload bool, connectionID, rootDir string) vfs.Fs { return &MockOsFs{ - Fs: vfs.NewOsFs(connectionID, rootDir, ""), + Fs: vfs.NewOsFs(connectionID, rootDir, "", nil), err: err, statErr: statErr, isAtomicUploadSupported: atomicUpload, @@ -719,7 +719,7 @@ func TestUploadFileStatError(t *testing.T) { user.Permissions["/"] = []string{dataprovider.PermAny} mockCC := mockFTPClientContext{} connID := fmt.Sprintf("%v", mockCC.ID()) - fs := vfs.NewOsFs(connID, user.HomeDir, "") + fs := vfs.NewOsFs(connID, user.HomeDir, "", nil) connection := &Connection{ BaseConnection: common.NewBaseConnection(connID, common.ProtocolFTP, "", "", user), clientContext: mockCC, @@ -809,7 +809,7 @@ func TestUploadOverwriteErrors(t *testing.T) { _, err = connection.handleFTPUploadToExistingFile(fs, os.O_TRUNC, filepath.Join(os.TempDir(), "sub", "file"), filepath.Join(os.TempDir(), "sub", "file1"), 0, "/sub/file1") assert.Error(t, err) - fs = vfs.NewOsFs(connID, user.GetHomeDir(), "") + fs = vfs.NewOsFs(connID, user.GetHomeDir(), "", nil) _, err = connection.handleFTPUploadToExistingFile(fs, 0, "missing1", "missing2", 0, "missing") assert.Error(t, err) } diff --git a/internal/httpd/httpd_test.go b/internal/httpd/httpd_test.go index 606091d6..659eab92 100644 --- a/internal/httpd/httpd_test.go +++ b/internal/httpd/httpd_test.go @@ -617,6 +617,10 @@ func TestBasicUserHandling(t *testing.T) { assert.True(t, user.HasPassword) user.Email = "invalid@email" + user.FsConfig.OSConfig = sdk.OSFsConfig{ + ReadBufferSize: 1, + WriteBufferSize: 2, + } _, body, err := httpdtest.UpdateUser(user, http.StatusBadRequest, "") assert.NoError(t, err) assert.Contains(t, string(body), "Validation error: email") @@ -887,6 +891,12 @@ func TestGroupRelations(t *testing.T) { _, _, err := httpdtest.AddFolder(vfs.BaseVirtualFolder{ Name: folderName2, MappedPath: mappedPath2, + FsConfig: vfs.Filesystem{ + OSConfig: sdk.OSFsConfig{ + ReadBufferSize: 3, + WriteBufferSize: 5, + }, + }, }, http.StatusCreated) assert.NoError(t, err) g1 := getTestGroup() @@ -1145,21 +1155,47 @@ func TestGroupSettingsOverride(t *testing.T) { folderName1 := filepath.Base(mappedPath1) mappedPath2 := filepath.Join(os.TempDir(), util.GenerateUniqueID()) folderName2 := filepath.Base(mappedPath2) + mappedPath3 := filepath.Join(os.TempDir(), util.GenerateUniqueID()) + folderName3 := filepath.Base(mappedPath3) g1 := getTestGroup() g1.Name += "_1" g1.VirtualFolders = append(g1.VirtualFolders, vfs.VirtualFolder{ BaseVirtualFolder: vfs.BaseVirtualFolder{ Name: folderName1, MappedPath: mappedPath1, + FsConfig: vfs.Filesystem{ + OSConfig: sdk.OSFsConfig{ + ReadBufferSize: 3, + WriteBufferSize: 5, + }, + }, }, VirtualPath: "/vdir1", }) + g1.UserSettings.Permissions = map[string][]string{ + "/dir1": {dataprovider.PermUpload}, + "/dir2": {dataprovider.PermDownload, dataprovider.PermListItems}, + } + g1.UserSettings.FsConfig.OSConfig = sdk.OSFsConfig{ + ReadBufferSize: 6, + WriteBufferSize: 2, + } g2 := getTestGroup() g2.Name += "_2" + g2.UserSettings.Permissions = map[string][]string{ + "/dir1": {dataprovider.PermAny}, + "/dir3": {dataprovider.PermDownload, dataprovider.PermListItems, dataprovider.PermChtimes}, + } g2.VirtualFolders = append(g2.VirtualFolders, vfs.VirtualFolder{ BaseVirtualFolder: vfs.BaseVirtualFolder{ Name: folderName1, MappedPath: mappedPath1, + FsConfig: vfs.Filesystem{ + OSConfig: sdk.OSFsConfig{ + ReadBufferSize: 3, + WriteBufferSize: 5, + }, + }, }, VirtualPath: "/vdir2", }) @@ -1170,6 +1206,19 @@ func TestGroupSettingsOverride(t *testing.T) { }, VirtualPath: "/vdir3", }) + g2.VirtualFolders = append(g2.VirtualFolders, vfs.VirtualFolder{ + BaseVirtualFolder: vfs.BaseVirtualFolder{ + Name: folderName3, + MappedPath: mappedPath3, + FsConfig: vfs.Filesystem{ + OSConfig: sdk.OSFsConfig{ + ReadBufferSize: 1, + WriteBufferSize: 2, + }, + }, + }, + VirtualPath: "/vdir4", + }) group1, resp, err := httpdtest.AddGroup(g1, http.StatusCreated) assert.NoError(t, err, string(resp)) group2, resp, err := httpdtest.AddGroup(g2, http.StatusCreated) @@ -1188,19 +1237,55 @@ func TestGroupSettingsOverride(t *testing.T) { user, _, err := httpdtest.AddUser(u, http.StatusCreated) assert.NoError(t, err) assert.Len(t, user.VirtualFolders, 0) + assert.Len(t, user.Permissions, 1) user, err = dataprovider.CheckUserAndPass(defaultUsername, defaultPassword, "", common.ProtocolHTTP) assert.NoError(t, err) - assert.Len(t, user.VirtualFolders, 3) + + var folderNames []string + if assert.Len(t, user.VirtualFolders, 4) { + for _, f := range user.VirtualFolders { + if !util.Contains(folderNames, f.Name) { + folderNames = append(folderNames, f.Name) + } + switch f.Name { + case folderName1: + assert.Equal(t, mappedPath1, f.MappedPath) + assert.Equal(t, 3, f.BaseVirtualFolder.FsConfig.OSConfig.ReadBufferSize) + assert.Equal(t, 5, f.BaseVirtualFolder.FsConfig.OSConfig.WriteBufferSize) + assert.True(t, util.Contains([]string{"/vdir1", "/vdir2"}, f.VirtualPath)) + case folderName2: + assert.Equal(t, mappedPath2, f.MappedPath) + assert.Equal(t, "/vdir3", f.VirtualPath) + assert.Equal(t, 0, f.BaseVirtualFolder.FsConfig.OSConfig.ReadBufferSize) + assert.Equal(t, 0, f.BaseVirtualFolder.FsConfig.OSConfig.WriteBufferSize) + case folderName3: + assert.Equal(t, mappedPath3, f.MappedPath) + assert.Equal(t, "/vdir4", f.VirtualPath) + assert.Equal(t, 1, f.BaseVirtualFolder.FsConfig.OSConfig.ReadBufferSize) + assert.Equal(t, 2, f.BaseVirtualFolder.FsConfig.OSConfig.WriteBufferSize) + } + } + } + assert.Len(t, folderNames, 3) + assert.Contains(t, folderNames, folderName1) + assert.Contains(t, folderNames, folderName2) + assert.Contains(t, folderNames, folderName3) + assert.Len(t, user.Permissions, 4) + assert.Equal(t, g1.UserSettings.Permissions["/dir1"], user.Permissions["/dir1"]) + assert.Equal(t, g1.UserSettings.Permissions["/dir2"], user.Permissions["/dir2"]) + assert.Equal(t, g2.UserSettings.Permissions["/dir3"], user.Permissions["/dir3"]) + assert.Equal(t, g1.UserSettings.FsConfig.OSConfig.ReadBufferSize, user.FsConfig.OSConfig.ReadBufferSize) + assert.Equal(t, g1.UserSettings.FsConfig.OSConfig.WriteBufferSize, user.FsConfig.OSConfig.WriteBufferSize) user, err = dataprovider.GetUserAfterIDPAuth(defaultUsername, "", common.ProtocolOIDC, nil) assert.NoError(t, err) - assert.Len(t, user.VirtualFolders, 3) + assert.Len(t, user.VirtualFolders, 4) user1, user2, err := dataprovider.GetUserVariants(defaultUsername, "") assert.NoError(t, err) assert.Len(t, user1.VirtualFolders, 0) - assert.Len(t, user2.VirtualFolders, 3) + assert.Len(t, user2.VirtualFolders, 4) assert.Equal(t, int64(0), user1.ExpirationDate) assert.Equal(t, int64(0), user2.ExpirationDate) @@ -1226,7 +1311,7 @@ func TestGroupSettingsOverride(t *testing.T) { assert.NoError(t, err) user, err = dataprovider.CheckUserAndPass(defaultUsername, defaultPassword, "", common.ProtocolHTTP) assert.NoError(t, err) - assert.Len(t, user.VirtualFolders, 3) + assert.Len(t, user.VirtualFolders, 4) assert.Equal(t, sdk.LocalFilesystemProvider, user.FsConfig.Provider) assert.Equal(t, int64(0), user.DownloadBandwidth) assert.Equal(t, int64(0), user.UploadBandwidth) @@ -1272,7 +1357,7 @@ func TestGroupSettingsOverride(t *testing.T) { assert.NoError(t, err) user, err = dataprovider.CheckUserAndPass(defaultUsername, defaultPassword, "", common.ProtocolHTTP) assert.NoError(t, err) - assert.Len(t, user.VirtualFolders, 3) + assert.Len(t, user.VirtualFolders, 4) assert.Equal(t, user.CreatedAt+int64(group1.UserSettings.ExpiresIn)*86400000, user.ExpirationDate) assert.Equal(t, group1.UserSettings.Filters.PasswordStrength, user.Filters.PasswordStrength) assert.Equal(t, sdk.SFTPFilesystemProvider, user.FsConfig.Provider) @@ -1307,6 +1392,8 @@ func TestGroupSettingsOverride(t *testing.T) { assert.NoError(t, err) _, err = httpdtest.RemoveFolder(vfs.BaseVirtualFolder{Name: folderName2}, http.StatusOK) assert.NoError(t, err) + _, err = httpdtest.RemoveFolder(vfs.BaseVirtualFolder{Name: folderName3}, http.StatusOK) + assert.NoError(t, err) } func TestConfigs(t *testing.T) { @@ -12208,7 +12295,7 @@ func TestWebClientMaxConnections(t *testing.T) { checkResponseCode(t, http.StatusOK, rr) // now add a fake connection - fs := vfs.NewOsFs("id", os.TempDir(), "") + fs := vfs.NewOsFs("id", os.TempDir(), "", nil) connection := &httpd.Connection{ BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolHTTP, "", "", user), } @@ -12399,7 +12486,7 @@ func TestMaxSessions(t *testing.T) { apiToken, err := getJWTAPIUserTokenFromTestServer(defaultUsername, defaultPassword) assert.NoError(t, err) // now add a fake connection - fs := vfs.NewOsFs("id", os.TempDir(), "") + fs := vfs.NewOsFs("id", os.TempDir(), "", nil) connection := &httpd.Connection{ BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolHTTP, "", "", user), } @@ -13508,7 +13595,7 @@ func TestShareMaxSessions(t *testing.T) { rr = executeRequest(req) checkResponseCode(t, http.StatusOK, rr) // add a fake connection - fs := vfs.NewOsFs("id", os.TempDir(), "") + fs := vfs.NewOsFs("id", os.TempDir(), "", nil) connection := &httpd.Connection{ BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolHTTP, "", "", user), } @@ -15887,6 +15974,105 @@ func TestWebFilesAPI(t *testing.T) { checkResponseCode(t, http.StatusNotFound, rr) } +func TestBufferedWebFilesAPI(t *testing.T) { + u := getTestUser() + u.FsConfig.OSConfig = sdk.OSFsConfig{ + ReadBufferSize: 1, + WriteBufferSize: 1, + } + vdirPath := "/crypted" + mappedPath := filepath.Join(os.TempDir(), util.GenerateUniqueID()) + folderName := filepath.Base(mappedPath) + u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{ + BaseVirtualFolder: vfs.BaseVirtualFolder{ + Name: folderName, + MappedPath: mappedPath, + FsConfig: vfs.Filesystem{ + Provider: sdk.CryptedFilesystemProvider, + CryptConfig: vfs.CryptFsConfig{ + OSFsConfig: sdk.OSFsConfig{ + WriteBufferSize: 3, + ReadBufferSize: 2, + }, + Passphrase: kms.NewPlainSecret(defaultPassword), + }, + }, + }, + VirtualPath: vdirPath, + QuotaFiles: -1, + QuotaSize: -1, + }) + user, _, err := httpdtest.AddUser(u, http.StatusCreated) + assert.NoError(t, err) + webAPIToken, err := getJWTAPIUserTokenFromTestServer(defaultUsername, defaultPassword) + assert.NoError(t, err) + + body := new(bytes.Buffer) + writer := multipart.NewWriter(body) + part1, err := writer.CreateFormFile("filenames", "file1.txt") + assert.NoError(t, err) + _, err = part1.Write([]byte("file1 content")) + assert.NoError(t, err) + err = writer.Close() + assert.NoError(t, err) + reader := bytes.NewReader(body.Bytes()) + + req, err := http.NewRequest(http.MethodPost, userFilesPath, reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr := executeRequest(req) + checkResponseCode(t, http.StatusCreated, rr) + + _, err = reader.Seek(0, io.SeekStart) + assert.NoError(t, err) + req, err = http.NewRequest(http.MethodPost, userFilesPath+"?path="+url.QueryEscape(vdirPath), reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusCreated, rr) + + req, err = http.NewRequest(http.MethodGet, userFilesPath+"?path=file1.txt", nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + assert.Equal(t, "file1 content", rr.Body.String()) + + req, err = http.NewRequest(http.MethodGet, userFilesPath+"?path="+url.QueryEscape(vdirPath+"/file1.txt"), nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + assert.Equal(t, "file1 content", rr.Body.String()) + + req, err = http.NewRequest(http.MethodGet, userFilesPath+"?path=file1.txt", nil) + assert.NoError(t, err) + req.Header.Set("Range", "bytes=2-") + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusPartialContent, rr) + assert.Equal(t, "le1 content", rr.Body.String()) + + req, err = http.NewRequest(http.MethodGet, userFilesPath+"?path="+url.QueryEscape(vdirPath+"/file1.txt"), nil) + assert.NoError(t, err) + req.Header.Set("Range", "bytes=3-6") + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusPartialContent, rr) + assert.Equal(t, "e1 c", rr.Body.String()) + + _, err = httpdtest.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) + _, err = httpdtest.RemoveFolder(vfs.BaseVirtualFolder{Name: folderName}, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(mappedPath) + assert.NoError(t, err) +} + func TestStartDirectory(t *testing.T) { u := getTestUser() u.Filters.StartDirectory = "/start/dir" @@ -19009,6 +19195,8 @@ func TestWebUserAddMock(t *testing.T) { form.Set("username", user.Username) form.Set("email", user.Email) form.Set("home_dir", user.HomeDir) + form.Set("osfs_read_buffer_size", "2") + form.Set("osfs_write_buffer_size", "3") form.Set("password", user.Password) form.Set("primary_group", group1.Name) form.Set("secondary_groups", group2.Name) @@ -19344,6 +19532,8 @@ func TestWebUserAddMock(t *testing.T) { err = render.DecodeJSON(rr.Body, &newUser) assert.NoError(t, err) assert.Equal(t, user.UID, newUser.UID) + assert.Equal(t, 2, newUser.FsConfig.OSConfig.ReadBufferSize) + assert.Equal(t, 3, newUser.FsConfig.OSConfig.WriteBufferSize) assert.Equal(t, user.UploadBandwidth, newUser.UploadBandwidth) assert.Equal(t, user.DownloadBandwidth, newUser.DownloadBandwidth) assert.Equal(t, user.UploadDataTransfer, newUser.UploadDataTransfer) @@ -21088,6 +21278,8 @@ func TestWebUserCryptMock(t *testing.T) { form.Set("denied_ip", "") form.Set("fs_provider", "4") form.Set("crypt_passphrase", "") + form.Set("cryptfs_read_buffer_size", "1") + form.Set("cryptfs_write_buffer_size", "2") form.Set("pattern_path0", "/dir1") form.Set("patterns0", "*.jpg,*.png") form.Set("pattern_type0", "allowed") @@ -21125,6 +21317,8 @@ func TestWebUserCryptMock(t *testing.T) { assert.NotEmpty(t, updateUser.FsConfig.CryptConfig.Passphrase.GetPayload()) assert.Empty(t, updateUser.FsConfig.CryptConfig.Passphrase.GetKey()) assert.Empty(t, updateUser.FsConfig.CryptConfig.Passphrase.GetAdditionalData()) + assert.Equal(t, 1, updateUser.FsConfig.CryptConfig.ReadBufferSize) + assert.Equal(t, 2, updateUser.FsConfig.CryptConfig.WriteBufferSize) // now check that a redacted password is not saved form.Set("crypt_passphrase", redactedSecret+" ") b, contentType, _ = getMultipartFormData(form, "", "") @@ -22637,6 +22831,8 @@ func TestAddWebFoldersMock(t *testing.T) { form.Set("mapped_path", mappedPath) form.Set("name", folderName) form.Set("description", folderDesc) + form.Set("osfs_read_buffer_size", "3") + form.Set("osfs_write_buffer_size", "4") b, contentType, err := getMultipartFormData(form, "", "") assert.NoError(t, err) req, err := http.NewRequest(http.MethodPost, webFolderPath, &b) @@ -22690,6 +22886,8 @@ func TestAddWebFoldersMock(t *testing.T) { assert.Equal(t, mappedPath, folder.MappedPath) assert.Equal(t, folderName, folder.Name) assert.Equal(t, folderDesc, folder.Description) + assert.Equal(t, 3, folder.FsConfig.OSConfig.ReadBufferSize) + assert.Equal(t, 4, folder.FsConfig.OSConfig.WriteBufferSize) // cleanup req, _ = http.NewRequest(http.MethodDelete, path.Join(folderPath, folderName), nil) setBearerForReq(req, apiToken) diff --git a/internal/httpd/webadmin.go b/internal/httpd/webadmin.go index 022ab8b9..5ef82bae 100644 --- a/internal/httpd/webadmin.go +++ b/internal/httpd/webadmin.go @@ -1747,10 +1747,25 @@ func getAzureConfig(r *http.Request) (vfs.AzBlobFsConfig, error) { return config, nil } +func getOsConfigFromPostFields(r *http.Request, readBufferField, writeBufferField string) sdk.OSFsConfig { + config := sdk.OSFsConfig{} + readBuffer, err := strconv.Atoi(r.Form.Get(readBufferField)) + if err == nil { + config.ReadBufferSize = readBuffer + } + writeBuffer, err := strconv.Atoi(r.Form.Get(writeBufferField)) + if err == nil { + config.WriteBufferSize = writeBuffer + } + return config +} + func getFsConfigFromPostFields(r *http.Request) (vfs.Filesystem, error) { var fs vfs.Filesystem fs.Provider = sdk.GetProviderByName(r.Form.Get("fs_provider")) switch fs.Provider { + case sdk.LocalFilesystemProvider: + fs.OSConfig = getOsConfigFromPostFields(r, "osfs_read_buffer_size", "osfs_write_buffer_size") case sdk.S3FilesystemProvider: config, err := getS3Config(r) if err != nil { @@ -1771,6 +1786,7 @@ func getFsConfigFromPostFields(r *http.Request) (vfs.Filesystem, error) { fs.GCSConfig = config case sdk.CryptedFilesystemProvider: fs.CryptConfig.Passphrase = getSecretFromFormField(r, "crypt_passphrase") + fs.CryptConfig.OSFsConfig = getOsConfigFromPostFields(r, "cryptfs_read_buffer_size", "cryptfs_write_buffer_size") case sdk.SFTPFilesystemProvider: config, err := getSFTPConfig(r) if err != nil { diff --git a/internal/httpdtest/httpdtest.go b/internal/httpdtest/httpdtest.go index b4b71589..e53a9763 100644 --- a/internal/httpdtest/httpdtest.go +++ b/internal/httpdtest/httpdtest.go @@ -2129,6 +2129,12 @@ func compareFsConfig(expected *vfs.Filesystem, actual *vfs.Filesystem) error { if expected.Provider != actual.Provider { return errors.New("fs provider mismatch") } + if expected.OSConfig.ReadBufferSize != actual.OSConfig.ReadBufferSize { + return fmt.Errorf("read buffer size mismatch") + } + if expected.OSConfig.WriteBufferSize != actual.OSConfig.WriteBufferSize { + return fmt.Errorf("write buffer size mismatch") + } if err := compareS3Config(expected, actual); err != nil { return err } @@ -2141,6 +2147,12 @@ func compareFsConfig(expected *vfs.Filesystem, actual *vfs.Filesystem) error { if err := checkEncryptedSecret(expected.CryptConfig.Passphrase, actual.CryptConfig.Passphrase); err != nil { return err } + if expected.CryptConfig.ReadBufferSize != actual.CryptConfig.ReadBufferSize { + return fmt.Errorf("crypt read buffer size mismatch") + } + if expected.CryptConfig.WriteBufferSize != actual.CryptConfig.WriteBufferSize { + return fmt.Errorf("crypt write buffer size mismatch") + } if err := compareSFTPFsConfig(expected, actual); err != nil { return err } diff --git a/internal/sftpd/internal_test.go b/internal/sftpd/internal_test.go index 36caac36..baddaea9 100644 --- a/internal/sftpd/internal_test.go +++ b/internal/sftpd/internal_test.go @@ -149,7 +149,7 @@ func (fs MockOsFs) Rename(source, target string) (int, int64, error) { func newMockOsFs(err, statErr error, atomicUpload bool, connectionID, rootDir string) vfs.Fs { return &MockOsFs{ - Fs: vfs.NewOsFs(connectionID, rootDir, ""), + Fs: vfs.NewOsFs(connectionID, rootDir, "", nil), err: err, statErr: statErr, isAtomicUploadSupported: atomicUpload, @@ -183,7 +183,7 @@ func TestUploadResumeInvalidOffset(t *testing.T) { Username: "testuser", }, } - fs := vfs.NewOsFs("", os.TempDir(), "") + fs := vfs.NewOsFs("", os.TempDir(), "", nil) conn := common.NewBaseConnection("", common.ProtocolSFTP, "", "", user) baseTransfer := common.NewBaseTransfer(file, conn, nil, file.Name(), file.Name(), testfile, common.TransferUpload, 10, 0, 0, 0, false, fs, dataprovider.TransferQuota{}) @@ -214,7 +214,7 @@ func TestReadWriteErrors(t *testing.T) { Username: "testuser", }, } - fs := vfs.NewOsFs("", os.TempDir(), "") + fs := vfs.NewOsFs("", os.TempDir(), "", nil) conn := common.NewBaseConnection("", common.ProtocolSFTP, "", "", user) baseTransfer := common.NewBaseTransfer(file, conn, nil, file.Name(), file.Name(), testfile, common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{}) @@ -288,7 +288,7 @@ func TestTransferCancelFn(t *testing.T) { Username: "testuser", }, } - fs := vfs.NewOsFs("", os.TempDir(), "") + fs := vfs.NewOsFs("", os.TempDir(), "", nil) conn := common.NewBaseConnection("", common.ProtocolSFTP, "", "", user) baseTransfer := common.NewBaseTransfer(file, conn, cancelFn, file.Name(), file.Name(), testfile, common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{}) @@ -311,7 +311,7 @@ func TestTransferCancelFn(t *testing.T) { func TestUploadFiles(t *testing.T) { common.Config.UploadMode = common.UploadModeAtomic - fs := vfs.NewOsFs("123", os.TempDir(), "") + fs := vfs.NewOsFs("123", os.TempDir(), "", nil) u := dataprovider.User{} c := Connection{ BaseConnection: common.NewBaseConnection("", common.ProtocolSFTP, "", "", u), @@ -1213,7 +1213,7 @@ func TestSCPParseUploadMessage(t *testing.T) { StdErrBuffer: bytes.NewBuffer(stdErrBuf), ReadError: nil, } - fs := vfs.NewOsFs("", os.TempDir(), "") + fs := vfs.NewOsFs("", os.TempDir(), "", nil) connection := &Connection{ BaseConnection: common.NewBaseConnection("", common.ProtocolSFTP, "", "", dataprovider.User{ BaseUser: sdk.BaseUser{ @@ -1470,7 +1470,7 @@ func TestSCPRecursiveDownloadErrors(t *testing.T) { err := client.Close() assert.NoError(t, err) }() - fs := vfs.NewOsFs("123", os.TempDir(), "") + fs := vfs.NewOsFs("123", os.TempDir(), "", nil) connection := &Connection{ BaseConnection: common.NewBaseConnection("", common.ProtocolSCP, "", "", dataprovider.User{ BaseUser: sdk.BaseUser{ @@ -1593,7 +1593,7 @@ func TestSCPDownloadFileData(t *testing.T) { ReadError: nil, WriteError: writeErr, } - fs := vfs.NewOsFs("", os.TempDir(), "") + fs := vfs.NewOsFs("", os.TempDir(), "", nil) connection := &Connection{ BaseConnection: common.NewBaseConnection("", common.ProtocolSCP, "", "", dataprovider.User{BaseUser: sdk.BaseUser{HomeDir: os.TempDir()}}), channel: &mockSSHChannelReadErr, @@ -1645,7 +1645,7 @@ func TestSCPUploadFiledata(t *testing.T) { Username: "testuser", }, } - fs := vfs.NewOsFs("", os.TempDir(), "") + fs := vfs.NewOsFs("", os.TempDir(), "", nil) connection := &Connection{ BaseConnection: common.NewBaseConnection("", common.ProtocolSCP, "", "", user), channel: &mockSSHChannel, @@ -1736,7 +1736,7 @@ func TestUploadError(t *testing.T) { Username: "testuser", }, } - fs := vfs.NewOsFs("", os.TempDir(), "") + fs := vfs.NewOsFs("", os.TempDir(), "", nil) connection := &Connection{ BaseConnection: common.NewBaseConnection("", common.ProtocolSCP, "", "", user), } diff --git a/internal/sftpd/sftpd_test.go b/internal/sftpd/sftpd_test.go index 7a1c9e35..65180f01 100644 --- a/internal/sftpd/sftpd_test.go +++ b/internal/sftpd/sftpd_test.go @@ -1428,7 +1428,17 @@ func TestUploadResume(t *testing.T) { u = getTestSFTPUser(usePubKey) sftpUser, _, err := httpdtest.AddUser(u, http.StatusCreated) assert.NoError(t, err) - for _, user := range []dataprovider.User{localUser, sftpUser} { + u = getTestUser(usePubKey) + u.FsConfig.OSConfig = sdk.OSFsConfig{ + WriteBufferSize: 1, + ReadBufferSize: 1, + } + u.Username += "_buffered" + u.HomeDir += "_with_buf" + bufferedUser, _, err := httpdtest.AddUser(u, http.StatusCreated) + assert.NoError(t, err) + + for _, user := range []dataprovider.User{localUser, sftpUser, bufferedUser} { conn, client, err := getSftpClient(user, usePubKey) if assert.NoError(t, err) { defer conn.Close() @@ -1475,8 +1485,12 @@ func TestUploadResume(t *testing.T) { assert.NoError(t, err) _, err = httpdtest.RemoveUser(localUser, http.StatusOK) assert.NoError(t, err) + _, err = httpdtest.RemoveUser(bufferedUser, http.StatusOK) + assert.NoError(t, err) err = os.RemoveAll(localUser.GetHomeDir()) assert.NoError(t, err) + err = os.RemoveAll(bufferedUser.GetHomeDir()) + assert.NoError(t, err) } func TestDirCommands(t *testing.T) { @@ -5570,6 +5584,129 @@ func TestNestedVirtualFolders(t *testing.T) { assert.NoError(t, err) } +func TestBufferedUser(t *testing.T) { + usePubKey := true + u := getTestUser(usePubKey) + u.QuotaFiles = 1000 + u.FsConfig.OSConfig = sdk.OSFsConfig{ + WriteBufferSize: 2, + ReadBufferSize: 1, + } + vdirPath := "/crypted" + mappedPath := filepath.Join(os.TempDir(), util.GenerateUniqueID()) + folderName := filepath.Base(mappedPath) + u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{ + BaseVirtualFolder: vfs.BaseVirtualFolder{ + Name: folderName, + MappedPath: mappedPath, + FsConfig: vfs.Filesystem{ + Provider: sdk.CryptedFilesystemProvider, + CryptConfig: vfs.CryptFsConfig{ + OSFsConfig: sdk.OSFsConfig{ + WriteBufferSize: 3, + ReadBufferSize: 2, + }, + Passphrase: kms.NewPlainSecret(defaultPassword), + }, + }, + }, + VirtualPath: vdirPath, + QuotaFiles: -1, + QuotaSize: -1, + }) + user, _, err := httpdtest.AddUser(u, http.StatusCreated) + assert.NoError(t, err) + conn, client, err := getSftpClient(user, usePubKey) + if assert.NoError(t, err) { + defer conn.Close() + defer client.Close() + + expectedQuotaSize := int64(0) + expectedQuotaFiles := 0 + fileSize := int64(32768) + err = writeSFTPFile(testFileName, fileSize, client) + assert.NoError(t, err) + expectedQuotaSize += fileSize + expectedQuotaFiles++ + err = writeSFTPFile(path.Join(vdirPath, testFileName), fileSize, client) + assert.NoError(t, err) + expectedQuotaSize += fileSize + expectedQuotaFiles++ + user, _, err = httpdtest.GetUserByUsername(user.Username, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, expectedQuotaFiles, user.UsedQuotaFiles) + assert.Greater(t, user.UsedQuotaSize, expectedQuotaSize) + localDownloadPath := filepath.Join(homeBasePath, testDLFileName) + err = sftpDownloadFile(testFileName, localDownloadPath, fileSize, client) + assert.NoError(t, err) + err = sftpDownloadFile(path.Join(vdirPath, testFileName), localDownloadPath, fileSize, client) + assert.NoError(t, err) + err = os.Remove(localDownloadPath) + assert.NoError(t, err) + err = client.Remove(testFileName) + assert.NoError(t, err) + err = client.Remove(path.Join(vdirPath, testFileName)) + assert.NoError(t, err) + + data := []byte("test data") + f, err := client.OpenFile(testFileName, os.O_WRONLY|os.O_CREATE) + if assert.NoError(t, err) { + n, err := f.Write(data) + assert.NoError(t, err) + assert.Equal(t, len(data), n) + err = f.Truncate(2) + assert.NoError(t, err) + expectedQuotaSize := int64(2) + expectedQuotaFiles := 0 + user, _, err = httpdtest.GetUserByUsername(user.Username, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, expectedQuotaFiles, user.UsedQuotaFiles) + assert.Equal(t, expectedQuotaSize, user.UsedQuotaSize) + _, err = f.Seek(expectedQuotaSize, io.SeekStart) + assert.NoError(t, err) + n, err = f.Write(data) + assert.NoError(t, err) + assert.Equal(t, len(data), n) + err = f.Truncate(5) + assert.NoError(t, err) + expectedQuotaSize = int64(5) + user, _, err = httpdtest.GetUserByUsername(user.Username, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, expectedQuotaFiles, user.UsedQuotaFiles) + assert.Equal(t, expectedQuotaSize, user.UsedQuotaSize) + _, err = f.Seek(expectedQuotaSize, io.SeekStart) + assert.NoError(t, err) + n, err = f.Write(data) + assert.NoError(t, err) + assert.Equal(t, len(data), n) + err = f.Close() + assert.NoError(t, err) + expectedQuotaSize = int64(5) + int64(len(data)) + expectedQuotaFiles = 1 + user, _, err = httpdtest.GetUserByUsername(user.Username, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, expectedQuotaFiles, user.UsedQuotaFiles) + assert.Equal(t, expectedQuotaSize, user.UsedQuotaSize) + } + // now truncate by path + err = client.Truncate(testFileName, 5) + assert.NoError(t, err) + user, _, err = httpdtest.GetUserByUsername(user.Username, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, 1, user.UsedQuotaFiles) + assert.Equal(t, int64(5), user.UsedQuotaSize) + } + + _, err = httpdtest.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) + _, err = httpdtest.RemoveFolder(vfs.BaseVirtualFolder{Name: folderName}, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(mappedPath) + assert.NoError(t, err) +} + func TestTruncateQuotaLimits(t *testing.T) { usePubKey := true u := getTestUser(usePubKey) @@ -8047,7 +8184,7 @@ func TestRootDirCommands(t *testing.T) { func TestRelativePaths(t *testing.T) { user := getTestUser(true) var path, rel string - filesystems := []vfs.Fs{vfs.NewOsFs("", user.GetHomeDir(), "")} + filesystems := []vfs.Fs{vfs.NewOsFs("", user.GetHomeDir(), "", nil)} keyPrefix := strings.TrimPrefix(user.GetHomeDir(), "/") + "/" s3config := vfs.S3FsConfig{ BaseS3FsConfig: sdk.BaseS3FsConfig{ @@ -8112,7 +8249,7 @@ func TestResolvePaths(t *testing.T) { user := getTestUser(true) var path, resolved string var err error - filesystems := []vfs.Fs{vfs.NewOsFs("", user.GetHomeDir(), "")} + filesystems := []vfs.Fs{vfs.NewOsFs("", user.GetHomeDir(), "", nil)} keyPrefix := strings.TrimPrefix(user.GetHomeDir(), "/") + "/" s3config := vfs.S3FsConfig{ BaseS3FsConfig: sdk.BaseS3FsConfig{ @@ -8175,8 +8312,8 @@ func TestVirtualRelativePaths(t *testing.T) { }) err := os.MkdirAll(mappedPath, os.ModePerm) assert.NoError(t, err) - fsRoot := vfs.NewOsFs("", user.GetHomeDir(), "") - fsVdir := vfs.NewOsFs("", mappedPath, vdirPath) + fsRoot := vfs.NewOsFs("", user.GetHomeDir(), "", nil) + fsVdir := vfs.NewOsFs("", mappedPath, vdirPath, nil) rel := fsVdir.GetRelativePath(mappedPath) assert.Equal(t, vdirPath, rel) rel = fsRoot.GetRelativePath(filepath.Join(mappedPath, "..")) diff --git a/internal/vfs/azblobfs.go b/internal/vfs/azblobfs.go index 739d807b..18c19eca 100644 --- a/internal/vfs/azblobfs.go +++ b/internal/vfs/azblobfs.go @@ -517,7 +517,7 @@ func (*AzureBlobFs) isBadRequestError(err error) bool { // CheckRootPath creates the specified local root directory if it does not exists func (fs *AzureBlobFs) CheckRootPath(username string, uid int, gid int) bool { // we need a local directory for temporary files - osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, "") + osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, "", nil) return osFs.CheckRootPath(username, uid, gid) } diff --git a/internal/vfs/cryptfs.go b/internal/vfs/cryptfs.go index fc0560d7..2a15d213 100644 --- a/internal/vfs/cryptfs.go +++ b/internal/vfs/cryptfs.go @@ -15,6 +15,7 @@ package vfs import ( + "bufio" "bytes" "crypto/rand" "crypto/sha256" @@ -55,10 +56,12 @@ func NewCryptFs(connectionID, rootDir, mountPath string, config CryptFsConfig) ( } fs := &CryptFs{ OsFs: &OsFs{ - name: cryptFsName, - connectionID: connectionID, - rootDir: rootDir, - mountPath: getMountPath(mountPath), + name: cryptFsName, + connectionID: connectionID, + rootDir: rootDir, + mountPath: getMountPath(mountPath), + readBufferSize: config.OSFsConfig.ReadBufferSize * 1024 * 1024, + writeBufferSize: config.OSFsConfig.WriteBufferSize * 1024 * 1024, }, masterKey: []byte(config.Passphrase.GetPayload()), } @@ -103,11 +106,11 @@ func (fs *CryptFs) Open(name string, offset int64) (File, *pipeat.PipeReaderAt, var err error if offset == 0 { - n, err = sio.Decrypt(w, f, fs.getSIOConfig(key)) + n, err = fs.decryptWrapper(w, f, fs.getSIOConfig(key)) } else { var readerAt io.ReaderAt var readed, written int - buf := make([]byte, 65536) + buf := make([]byte, 65568) wrapper := &cryptedFileWrapper{ File: f, } @@ -150,14 +153,8 @@ func (fs *CryptFs) Open(name string, offset int64) (File, *pipeat.PipeReaderAt, } // Create creates or opens the named file for writing -func (fs *CryptFs) Create(name string, flag, _ int) (File, *PipeWriter, func(), error) { - var err error - var f *os.File - if flag == 0 { - f, err = os.Create(name) - } else { - f, err = os.OpenFile(name, flag, 0666) - } +func (fs *CryptFs) Create(name string, _, _ int) (File, *PipeWriter, func(), error) { + f, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) if err != nil { return nil, nil, nil, err } @@ -192,7 +189,18 @@ func (fs *CryptFs) Create(name string, flag, _ int) (File, *PipeWriter, func(), p := NewPipeWriter(w) go func() { - n, err := sio.Encrypt(f, r, fs.getSIOConfig(key)) + var n int64 + var err error + if fs.writeBufferSize <= 0 { + n, err = sio.Encrypt(f, r, fs.getSIOConfig(key)) + } else { + bw := bufio.NewWriterSize(f, fs.writeBufferSize) + n, err = fs.encryptWrapper(bw, r, fs.getSIOConfig(key)) + errFlush := bw.Flush() + if err == nil && errFlush != nil { + err = errFlush + } + } errClose := f.Close() if err == nil && errClose != nil { err = errClose @@ -311,6 +319,26 @@ func (fs *CryptFs) getFileAndEncryptionKey(name string) (*os.File, [32]byte, err return f, key, err } +func (*CryptFs) encryptWrapper(dst io.Writer, src io.Reader, config sio.Config) (int64, error) { + encReader, err := sio.EncryptReader(src, config) + if err != nil { + return 0, err + } + return doCopy(dst, encReader, make([]byte, 65568)) +} + +func (fs *CryptFs) decryptWrapper(dst io.Writer, src io.Reader, config sio.Config) (int64, error) { + if fs.readBufferSize <= 0 { + return sio.Decrypt(dst, src, config) + } + br := bufio.NewReaderSize(src, fs.readBufferSize) + decReader, err := sio.DecryptReader(br, config) + if err != nil { + return 0, err + } + return doCopy(dst, decReader, make([]byte, 65568)) +} + func isZeroBytesDownload(f *os.File, offset int64) (bool, error) { info, err := f.Stat() if err != nil { diff --git a/internal/vfs/filesystem.go b/internal/vfs/filesystem.go index 10929ef8..a23afda7 100644 --- a/internal/vfs/filesystem.go +++ b/internal/vfs/filesystem.go @@ -26,6 +26,7 @@ import ( type Filesystem struct { RedactedSecret string `json:"-"` Provider sdk.FilesystemProvider `json:"provider"` + OSConfig sdk.OSFsConfig `json:"osconfig,omitempty"` S3Config S3FsConfig `json:"s3config,omitempty"` GCSConfig GCSFsConfig `json:"gcsconfig,omitempty"` AzBlobConfig AzBlobFsConfig `json:"azblobconfig,omitempty"` @@ -169,6 +170,7 @@ func (f *Filesystem) Validate(additionalData string) error { if err := f.S3Config.ValidateAndEncryptCredentials(additionalData); err != nil { return err } + f.OSConfig = sdk.OSFsConfig{} f.GCSConfig = GCSFsConfig{} f.AzBlobConfig = AzBlobFsConfig{} f.CryptConfig = CryptFsConfig{} @@ -179,6 +181,7 @@ func (f *Filesystem) Validate(additionalData string) error { if err := f.GCSConfig.ValidateAndEncryptCredentials(additionalData); err != nil { return err } + f.OSConfig = sdk.OSFsConfig{} f.S3Config = S3FsConfig{} f.AzBlobConfig = AzBlobFsConfig{} f.CryptConfig = CryptFsConfig{} @@ -189,6 +192,7 @@ func (f *Filesystem) Validate(additionalData string) error { if err := f.AzBlobConfig.ValidateAndEncryptCredentials(additionalData); err != nil { return err } + f.OSConfig = sdk.OSFsConfig{} f.S3Config = S3FsConfig{} f.GCSConfig = GCSFsConfig{} f.CryptConfig = CryptFsConfig{} @@ -199,16 +203,18 @@ func (f *Filesystem) Validate(additionalData string) error { if err := f.CryptConfig.ValidateAndEncryptCredentials(additionalData); err != nil { return err } + f.OSConfig = sdk.OSFsConfig{} f.S3Config = S3FsConfig{} f.GCSConfig = GCSFsConfig{} f.AzBlobConfig = AzBlobFsConfig{} f.SFTPConfig = SFTPFsConfig{} f.HTTPConfig = HTTPFsConfig{} - return nil + return validateOSFsConfig(&f.CryptConfig.OSFsConfig) case sdk.SFTPFilesystemProvider: if err := f.SFTPConfig.ValidateAndEncryptCredentials(additionalData); err != nil { return err } + f.OSConfig = sdk.OSFsConfig{} f.S3Config = S3FsConfig{} f.GCSConfig = GCSFsConfig{} f.AzBlobConfig = AzBlobFsConfig{} @@ -219,6 +225,7 @@ func (f *Filesystem) Validate(additionalData string) error { if err := f.HTTPConfig.ValidateAndEncryptCredentials(additionalData); err != nil { return err } + f.OSConfig = sdk.OSFsConfig{} f.S3Config = S3FsConfig{} f.GCSConfig = GCSFsConfig{} f.AzBlobConfig = AzBlobFsConfig{} @@ -233,7 +240,7 @@ func (f *Filesystem) Validate(additionalData string) error { f.CryptConfig = CryptFsConfig{} f.SFTPConfig = SFTPFsConfig{} f.HTTPConfig = HTTPFsConfig{} - return nil + return validateOSFsConfig(&f.OSConfig) } } @@ -293,6 +300,10 @@ func (f *Filesystem) GetACopy() Filesystem { f.SetEmptySecretsIfNil() fs := Filesystem{ Provider: f.Provider, + OSConfig: sdk.OSFsConfig{ + ReadBufferSize: f.OSConfig.ReadBufferSize, + WriteBufferSize: f.OSConfig.WriteBufferSize, + }, S3Config: S3FsConfig{ BaseS3FsConfig: sdk.BaseS3FsConfig{ Bucket: f.S3Config.Bucket, @@ -342,6 +353,10 @@ func (f *Filesystem) GetACopy() Filesystem { SASURL: f.AzBlobConfig.SASURL.Clone(), }, CryptConfig: CryptFsConfig{ + OSFsConfig: sdk.OSFsConfig{ + ReadBufferSize: f.CryptConfig.ReadBufferSize, + WriteBufferSize: f.CryptConfig.WriteBufferSize, + }, Passphrase: f.CryptConfig.Passphrase.Clone(), }, SFTPConfig: SFTPFsConfig{ diff --git a/internal/vfs/folder.go b/internal/vfs/folder.go index 4d131a6f..e3af0c9f 100644 --- a/internal/vfs/folder.go +++ b/internal/vfs/folder.go @@ -207,7 +207,7 @@ func (v *VirtualFolder) GetFilesystem(connectionID string, forbiddenSelfUsers [] case sdk.HTTPFilesystemProvider: return NewHTTPFs(connectionID, v.MappedPath, v.VirtualPath, v.FsConfig.HTTPConfig) default: - return NewOsFs(connectionID, v.MappedPath, v.VirtualPath), nil + return NewOsFs(connectionID, v.MappedPath, v.VirtualPath, &v.FsConfig.OSConfig), nil } } diff --git a/internal/vfs/gcsfs.go b/internal/vfs/gcsfs.go index 016e5561..a55f6433 100644 --- a/internal/vfs/gcsfs.go +++ b/internal/vfs/gcsfs.go @@ -472,7 +472,7 @@ func (*GCSFs) IsNotSupported(err error) bool { // CheckRootPath creates the specified local root directory if it does not exists func (fs *GCSFs) CheckRootPath(username string, uid int, gid int) bool { // we need a local directory for temporary files - osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, "") + osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, "", nil) return osFs.CheckRootPath(username, uid, gid) } diff --git a/internal/vfs/httpfs.go b/internal/vfs/httpfs.go index 6bf5a56c..94cccb4d 100644 --- a/internal/vfs/httpfs.go +++ b/internal/vfs/httpfs.go @@ -523,7 +523,7 @@ func (*HTTPFs) IsNotSupported(err error) bool { // CheckRootPath creates the specified local root directory if it does not exists func (fs *HTTPFs) CheckRootPath(username string, uid int, gid int) bool { // we need a local directory for temporary files - osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, "") + osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, "", nil) return osFs.CheckRootPath(username, uid, gid) } diff --git a/internal/vfs/osfs.go b/internal/vfs/osfs.go index 5ee7b61c..5d721cd1 100644 --- a/internal/vfs/osfs.go +++ b/internal/vfs/osfs.go @@ -15,6 +15,7 @@ package vfs import ( + "bufio" "errors" "fmt" "io" @@ -30,6 +31,7 @@ import ( fscopy "github.com/otiai10/copy" "github.com/pkg/sftp" "github.com/rs/xid" + "github.com/sftpgo/sdk" "github.com/drakkan/sftpgo/v2/internal/logger" "github.com/drakkan/sftpgo/v2/internal/util" @@ -54,16 +56,33 @@ type OsFs struct { connectionID string rootDir string // if not empty this fs is mouted as virtual folder in the specified path - mountPath string + mountPath string + localTempDir string + readBufferSize int + writeBufferSize int } // NewOsFs returns an OsFs object that allows to interact with local Os filesystem -func NewOsFs(connectionID, rootDir, mountPath string) Fs { +func NewOsFs(connectionID, rootDir, mountPath string, config *sdk.OSFsConfig) Fs { + var tempDir string + if tempPath != "" { + tempDir = tempPath + } else { + tempDir = filepath.Clean(os.TempDir()) + } + var readBufferSize, writeBufferSize int + if config != nil { + readBufferSize = config.ReadBufferSize * 1024 * 1024 + writeBufferSize = config.WriteBufferSize * 1024 * 1024 + } return &OsFs{ - name: osFsName, - connectionID: connectionID, - rootDir: rootDir, - mountPath: getMountPath(mountPath), + name: osFsName, + connectionID: connectionID, + rootDir: rootDir, + mountPath: getMountPath(mountPath), + localTempDir: tempDir, + readBufferSize: readBufferSize, + writeBufferSize: writeBufferSize, } } @@ -88,7 +107,7 @@ func (fs *OsFs) Lstat(name string) (os.FileInfo, error) { } // Open opens the named file for reading -func (*OsFs) Open(name string, offset int64) (File, *pipeat.PipeReaderAt, func(), error) { +func (fs *OsFs) Open(name string, offset int64) (File, *pipeat.PipeReaderAt, func(), error) { f, err := os.Open(name) if err != nil { return nil, nil, nil, err @@ -100,19 +119,65 @@ func (*OsFs) Open(name string, offset int64) (File, *pipeat.PipeReaderAt, func() return nil, nil, nil, err } } - return f, nil, nil, err + if fs.readBufferSize <= 0 { + return f, nil, nil, err + } + r, w, err := pipeat.PipeInDir(fs.localTempDir) + if err != nil { + f.Close() + return nil, nil, nil, err + } + go func() { + br := bufio.NewReaderSize(f, fs.readBufferSize) + n, err := doCopy(w, br, nil) + w.CloseWithError(err) //nolint:errcheck + f.Close() + fsLog(fs, logger.LevelDebug, "download completed, path: %q size: %v, err: %v", name, n, err) + }() + + return nil, r, nil, nil } // Create creates or opens the named file for writing -func (*OsFs) Create(name string, flag, _ int) (File, *PipeWriter, func(), error) { - var err error - var f *os.File - if flag == 0 { - f, err = os.Create(name) - } else { - f, err = os.OpenFile(name, flag, 0666) +func (fs *OsFs) Create(name string, flag, _ int) (File, *PipeWriter, func(), error) { + if !fs.useWriteBuffering(flag) { + var err error + var f *os.File + if flag == 0 { + f, err = os.Create(name) + } else { + f, err = os.OpenFile(name, flag, 0666) + } + return f, nil, nil, err } - return f, nil, nil, err + f, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + return nil, nil, nil, err + } + r, w, err := pipeat.PipeInDir(fs.localTempDir) + if err != nil { + f.Close() + return nil, nil, nil, err + } + p := NewPipeWriter(w) + + go func() { + bw := bufio.NewWriterSize(f, fs.writeBufferSize) + n, err := doCopy(bw, r, nil) + errFlush := bw.Flush() + if err == nil && errFlush != nil { + err = errFlush + } + errClose := f.Close() + if err == nil && errClose != nil { + err = errClose + } + r.CloseWithError(err) //nolint:errcheck + p.Done(err) + fsLog(fs, logger.LevelDebug, "upload completed, path: %q, readed bytes: %v, err: %v", name, n, err) + }() + + return nil, p, nil, nil } // Rename renames (moves) source to target @@ -124,10 +189,16 @@ func (fs *OsFs) Rename(source, target string) (int, int64, error) { if err != nil && isCrossDeviceError(err) { fsLog(fs, logger.LevelError, "cross device error detected while renaming %q -> %q. Trying a copy and remove, this could take a long time", source, target) + var readBufferSize uint + if fs.readBufferSize > 0 { + readBufferSize = uint(fs.readBufferSize) + } + err = fscopy.Copy(source, target, fscopy.Options{ OnSymlink: func(src string) fscopy.SymlinkAction { return fscopy.Skip }, + CopyBufferSize: readBufferSize, }) if err != nil { fsLog(fs, logger.LevelError, "cross device copy error: %v", err) @@ -509,3 +580,21 @@ func (*OsFs) Close() error { func (*OsFs) GetAvailableDiskSize(dirName string) (*sftp.StatVFS, error) { return getStatFS(dirName) } + +func (fs *OsFs) useWriteBuffering(flag int) bool { + if fs.writeBufferSize <= 0 { + return false + } + if flag == 0 { + return true + } + if flag&os.O_TRUNC == 0 { + fsLog(fs, logger.LevelDebug, "truncate flag missing, buffering write not possible") + return false + } + if flag&os.O_RDWR != 0 { + fsLog(fs, logger.LevelDebug, "read and write flag found, buffering write not possible") + return false + } + return true +} diff --git a/internal/vfs/s3fs.go b/internal/vfs/s3fs.go index 970fc45a..fc86b389 100644 --- a/internal/vfs/s3fs.go +++ b/internal/vfs/s3fs.go @@ -512,7 +512,7 @@ func (*S3Fs) IsNotSupported(err error) bool { // CheckRootPath creates the specified local root directory if it does not exists func (fs *S3Fs) CheckRootPath(username string, uid int, gid int) bool { // we need a local directory for temporary files - osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, "") + osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, "", nil) return osFs.CheckRootPath(username, uid, gid) } diff --git a/internal/vfs/sftpfs.go b/internal/vfs/sftpfs.go index 81a0bbd5..cba19b62 100644 --- a/internal/vfs/sftpfs.go +++ b/internal/vfs/sftpfs.go @@ -404,7 +404,7 @@ func (fs *SFTPFs) Create(name string, flag, _ int) (File, *PipeWriter, func(), e bw := bufio.NewWriterSize(f, int(fs.config.BufferSize)*1024*1024) // we don't use io.Copy since bufio.Writer implements io.WriterTo and // so it calls the sftp.File WriteTo method without buffering - n, err := fs.copy(bw, r) + n, err := doCopy(bw, r, nil) errFlush := bw.Flush() if err == nil && errFlush != nil { err = errFlush @@ -573,7 +573,7 @@ func (*SFTPFs) IsNotSupported(err error) bool { // CheckRootPath creates the specified local root directory if it does not exists func (fs *SFTPFs) CheckRootPath(username string, uid int, gid int) bool { // local directory for temporary files in buffer mode - osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, "") + osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, "", nil) osFs.CheckRootPath(username, uid, gid) if fs.config.Prefix == "/" { return true @@ -841,38 +841,6 @@ func (fs *SFTPFs) Close() error { return nil } -func (fs *SFTPFs) copy(dst io.Writer, src io.Reader) (written int64, err error) { - buf := make([]byte, 32768) - for { - nr, er := src.Read(buf) - if nr > 0 { - nw, ew := dst.Write(buf[0:nr]) - if nw < 0 || nr < nw { - nw = 0 - if ew == nil { - ew = errors.New("invalid write") - } - } - written += int64(nw) - if ew != nil { - err = ew - break - } - if nr != nw { - err = io.ErrShortWrite - break - } - } - if er != nil { - if er != io.EOF { - err = er - } - break - } - } - return written, err -} - func (fs *SFTPFs) createConnection() error { err := fs.conn.OpenConnection() if err != nil { diff --git a/internal/vfs/vfs.go b/internal/vfs/vfs.go index 3022a97b..f8a8ad01 100644 --- a/internal/vfs/vfs.go +++ b/internal/vfs/vfs.go @@ -632,6 +632,7 @@ func (c *AzBlobFsConfig) validate() error { // CryptFsConfig defines the configuration to store local files as encrypted type CryptFsConfig struct { + sdk.OSFsConfig Passphrase *kms.Secret `json:"passphrase,omitempty"` } @@ -757,21 +758,24 @@ func IsHTTPFs(fs Fs) bool { return strings.HasPrefix(fs.Name(), httpFsName) } -// IsBufferedSFTPFs returns true if this is a buffered SFTP filesystem -func IsBufferedSFTPFs(fs Fs) bool { +// IsBufferedLocalOrSFTPFs returns true if this is a buffered SFTP or local filesystem +func IsBufferedLocalOrSFTPFs(fs Fs) bool { + if osFs, ok := fs.(*OsFs); ok { + return osFs.writeBufferSize > 0 + } if !IsSFTPFs(fs) { return false } return !fs.IsUploadResumeSupported() } -// IsLocalOrUnbufferedSFTPFs returns true if fs is local or SFTP with no buffer -func IsLocalOrUnbufferedSFTPFs(fs Fs) bool { - if IsLocalOsFs(fs) { - return true +// FsOpenReturnsFile returns true if fs.Open returns a *os.File handle +func FsOpenReturnsFile(fs Fs) bool { + if osFs, ok := fs.(*OsFs); ok { + return osFs.readBufferSize == 0 } - if IsSFTPFs(fs) { - return fs.IsUploadResumeSupported() + if sftpFs, ok := fs.(*SFTPFs); ok { + return sftpFs.config.BufferSize == 0 } return false } @@ -929,6 +933,50 @@ func fsMetadataCheck(fs fsMetadataChecker, storageID, keyPrefix string) error { } } +func validateOSFsConfig(config *sdk.OSFsConfig) error { + if config.ReadBufferSize < 0 || config.ReadBufferSize > 10 { + return fmt.Errorf("invalid read buffer size must be between 0 and 10 MB") + } + if config.WriteBufferSize < 0 || config.WriteBufferSize > 10 { + return fmt.Errorf("invalid write buffer size must be between 0 and 10 MB") + } + return nil +} + +func doCopy(dst io.Writer, src io.Reader, buf []byte) (written int64, err error) { + if buf == nil { + buf = make([]byte, 32768) + } + for { + nr, er := src.Read(buf) + if nr > 0 { + nw, ew := dst.Write(buf[0:nr]) + if nw < 0 || nr < nw { + nw = 0 + if ew == nil { + ew = errors.New("invalid write") + } + } + written += int64(nw) + if ew != nil { + err = ew + break + } + if nr != nw { + err = io.ErrShortWrite + break + } + } + if er != nil { + if er != io.EOF { + err = er + } + break + } + } + return written, err +} + func getMountPath(mountPath string) string { if mountPath == "/" { return "" diff --git a/internal/webdavd/file.go b/internal/webdavd/file.go index 7f1cca9e..e4779f2d 100644 --- a/internal/webdavd/file.go +++ b/internal/webdavd/file.go @@ -282,7 +282,7 @@ func (f *webDavFile) updateTransferQuotaOnSeek() { } func (f *webDavFile) checkFile() error { - if f.File == nil && vfs.IsLocalOrUnbufferedSFTPFs(f.Fs) { + if f.File == nil && vfs.FsOpenReturnsFile(f.Fs) { file, _, _, err := f.Fs.Open(f.GetFsPath(), 0) if err != nil { f.Connection.Log(logger.LevelWarn, "could not open file %q for seeking: %v", diff --git a/internal/webdavd/internal_test.go b/internal/webdavd/internal_test.go index 3a812d90..5992a7dc 100644 --- a/internal/webdavd/internal_test.go +++ b/internal/webdavd/internal_test.go @@ -327,7 +327,7 @@ func (fs *MockOsFs) GetMimeType(_ string) (string, error) { func newMockOsFs(atomicUpload bool, connectionID, rootDir string, reader *pipeat.PipeReaderAt, err error) vfs.Fs { return &MockOsFs{ - Fs: vfs.NewOsFs(connectionID, rootDir, ""), + Fs: vfs.NewOsFs(connectionID, rootDir, "", nil), isAtomicUploadSupported: atomicUpload, reader: reader, err: err, @@ -484,7 +484,7 @@ func TestResolvePathErrors(t *testing.T) { } user.Permissions = make(map[string][]string) user.Permissions["/"] = []string{dataprovider.PermAny} - fs := vfs.NewOsFs("connID", user.HomeDir, "") + fs := vfs.NewOsFs("connID", user.HomeDir, "", nil) connection := &Connection{ BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolWebDAV, "", "", user), } @@ -517,7 +517,7 @@ func TestResolvePathErrors(t *testing.T) { if runtime.GOOS != "windows" { user.HomeDir = filepath.Clean(os.TempDir()) connection.User = user - fs := vfs.NewOsFs("connID", connection.User.HomeDir, "") + fs := vfs.NewOsFs("connID", connection.User.HomeDir, "", nil) subDir := "sub" testTxtFile := "file.txt" err = os.MkdirAll(filepath.Join(os.TempDir(), subDir, subDir), os.ModePerm) @@ -555,7 +555,7 @@ func TestFileAccessErrors(t *testing.T) { } user.Permissions = make(map[string][]string) user.Permissions["/"] = []string{dataprovider.PermAny} - fs := vfs.NewOsFs("connID", user.HomeDir, "") + fs := vfs.NewOsFs("connID", user.HomeDir, "", nil) connection := &Connection{ BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolWebDAV, "", "", user), } @@ -648,7 +648,7 @@ func TestCheckRequestMethodWithPrefix(t *testing.T) { }, }, } - fs := vfs.NewOsFs("connID", user.HomeDir, "") + fs := vfs.NewOsFs("connID", user.HomeDir, "", nil) connection := &Connection{ BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolWebDAV, "", "", user), } @@ -672,7 +672,7 @@ func TestContentType(t *testing.T) { } user.Permissions = make(map[string][]string) user.Permissions["/"] = []string{dataprovider.PermAny} - fs := vfs.NewOsFs("connID", user.HomeDir, "") + fs := vfs.NewOsFs("connID", user.HomeDir, "", nil) connection := &Connection{ BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolWebDAV, "", "", user), } @@ -699,7 +699,7 @@ func TestContentType(t *testing.T) { baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile+".unknown1", common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{}) davFile = newWebDavFile(baseTransfer, nil, nil) - davFile.Fs = vfs.NewOsFs("id", user.HomeDir, "") + davFile.Fs = vfs.NewOsFs("id", user.HomeDir, "", nil) fi, err = davFile.Stat() if assert.NoError(t, err) { ctype, err := fi.(*webDavFileInfo).ContentType(ctx) @@ -712,7 +712,7 @@ func TestContentType(t *testing.T) { baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile, common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{}) davFile = newWebDavFile(baseTransfer, nil, nil) - davFile.Fs = vfs.NewOsFs("id", user.HomeDir, "") + davFile.Fs = vfs.NewOsFs("id", user.HomeDir, "", nil) fi, err = davFile.Stat() if assert.NoError(t, err) { ctype, err := fi.(*webDavFileInfo).ContentType(ctx) @@ -727,7 +727,7 @@ func TestContentType(t *testing.T) { baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile+".custom", common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{}) davFile = newWebDavFile(baseTransfer, nil, nil) - davFile.Fs = vfs.NewOsFs("id", user.HomeDir, "") + davFile.Fs = vfs.NewOsFs("id", user.HomeDir, "", nil) fi, err = davFile.Stat() if assert.NoError(t, err) { ctype, err := fi.(*webDavFileInfo).ContentType(ctx) @@ -781,7 +781,7 @@ func TestTransferReadWriteErrors(t *testing.T) { } user.Permissions = make(map[string][]string) user.Permissions["/"] = []string{dataprovider.PermAny} - fs := vfs.NewOsFs("connID", user.HomeDir, "") + fs := vfs.NewOsFs("connID", user.HomeDir, "", nil) connection := &Connection{ BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolWebDAV, "", "", user), } @@ -937,7 +937,7 @@ func TestTransferSeek(t *testing.T) { assert.True(t, fs.IsNotExist(err)) davFile.Connection.RemoveTransfer(davFile.BaseTransfer) - fs = vfs.NewOsFs(fs.ConnectionID(), user.GetHomeDir(), "") + fs = vfs.NewOsFs(fs.ConnectionID(), user.GetHomeDir(), "", nil) baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath+"1", testFilePath+"1", testFile, common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{AllowedTotalSize: 100}) davFile = newWebDavFile(baseTransfer, nil, nil) diff --git a/internal/webdavd/webdavd_test.go b/internal/webdavd/webdavd_test.go index 01109a3b..699f5de3 100644 --- a/internal/webdavd/webdavd_test.go +++ b/internal/webdavd/webdavd_test.go @@ -52,6 +52,7 @@ import ( "github.com/drakkan/sftpgo/v2/internal/kms" "github.com/drakkan/sftpgo/v2/internal/logger" "github.com/drakkan/sftpgo/v2/internal/sftpd" + "github.com/drakkan/sftpgo/v2/internal/util" "github.com/drakkan/sftpgo/v2/internal/vfs" "github.com/drakkan/sftpgo/v2/internal/webdavd" ) @@ -720,6 +721,70 @@ func TestBasicHandlingCryptFs(t *testing.T) { 1*time.Second, 100*time.Millisecond) } +func TestBufferedUser(t *testing.T) { + u := getTestUser() + u.FsConfig.OSConfig = sdk.OSFsConfig{ + WriteBufferSize: 2, + ReadBufferSize: 1, + } + vdirPath := "/crypted" + mappedPath := filepath.Join(os.TempDir(), util.GenerateUniqueID()) + folderName := filepath.Base(mappedPath) + u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{ + BaseVirtualFolder: vfs.BaseVirtualFolder{ + Name: folderName, + MappedPath: mappedPath, + FsConfig: vfs.Filesystem{ + Provider: sdk.CryptedFilesystemProvider, + CryptConfig: vfs.CryptFsConfig{ + OSFsConfig: sdk.OSFsConfig{ + WriteBufferSize: 3, + ReadBufferSize: 2, + }, + Passphrase: kms.NewPlainSecret(defaultPassword), + }, + }, + }, + VirtualPath: vdirPath, + QuotaFiles: -1, + QuotaSize: -1, + }) + user, _, err := httpdtest.AddUser(u, http.StatusCreated) + assert.NoError(t, err) + + client := getWebDavClient(user, false, nil) + assert.NoError(t, checkBasicFunc(client)) + + testFilePath := filepath.Join(homeBasePath, testFileName) + testFileSize := int64(65535) + err = createTestFile(testFilePath, testFileSize) + assert.NoError(t, err) + err = uploadFileWithRawClient(testFilePath, testFileName, + user.Username, defaultPassword, false, testFileSize, client) + assert.NoError(t, err) + err = uploadFileWithRawClient(testFilePath, path.Join(vdirPath, testFileName), + user.Username, defaultPassword, false, testFileSize, client) + assert.NoError(t, err) + localDownloadPath := filepath.Join(homeBasePath, testDLFileName) + err = downloadFile(testFileName, localDownloadPath, testFileSize, client) + assert.NoError(t, err) + err = downloadFile(path.Join(vdirPath, testFileName), localDownloadPath, testFileSize, client) + assert.NoError(t, err) + + err = os.Remove(testFilePath) + assert.NoError(t, err) + err = os.Remove(localDownloadPath) + assert.NoError(t, err) + _, err = httpdtest.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) + _, err = httpdtest.RemoveFolder(vfs.BaseVirtualFolder{Name: folderName}, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(mappedPath) + assert.NoError(t, err) +} + func TestLoginEmptyPassword(t *testing.T) { u := getTestUser() u.Password = "" @@ -1497,7 +1562,7 @@ func TestMaxConnections(t *testing.T) { client := getWebDavClient(user, true, nil) assert.NoError(t, checkBasicFunc(client)) // now add a fake connection - fs := vfs.NewOsFs("id", os.TempDir(), "") + fs := vfs.NewOsFs("id", os.TempDir(), "", nil) connection := &webdavd.Connection{ BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolWebDAV, "", "", user), } @@ -1576,7 +1641,7 @@ func TestMaxSessions(t *testing.T) { client := getWebDavClient(user, false, nil) assert.NoError(t, checkBasicFunc(client)) // now add a fake connection - fs := vfs.NewOsFs("id", os.TempDir(), "") + fs := vfs.NewOsFs("id", os.TempDir(), "", nil) connection := &webdavd.Connection{ BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolWebDAV, "", "", user), } diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 70c5b6f5..f98ad7e6 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -5729,11 +5729,34 @@ components: use_emulator: type: boolean description: Azure Blob Storage configuration details + OSFsConfig: + type: object + properties: + read_buffer_size: + type: integer + minimum: 0 + maximum: 10 + description: 'The read buffer size, as MB, to use for downloads. 0 means no buffering, that's fine in most use cases.' + write_buffer_size: + type: integer + minimum: 0 + maximum: 10 + description: 'The write buffer size, as MB, to use for uploads. 0 means no buffering, that's fine in most use cases.' CryptFsConfig: type: object properties: passphrase: $ref: '#/components/schemas/Secret' + read_buffer_size: + type: integer + minimum: 0 + maximum: 10 + description: 'The read buffer size, as MB, to use for downloads. 0 means no buffering, that's fine in most use cases.' + write_buffer_size: + type: integer + minimum: 0 + maximum: 10 + description: 'The write buffer size, as MB, to use for uploads. 0 means no buffering, that's fine in most use cases.' description: Crypt filesystem configuration details SFTPFsConfig: type: object @@ -5804,6 +5827,8 @@ components: properties: provider: $ref: '#/components/schemas/FsProviders' + osconfig: + $ref: '#/components/schemas/OSFsConfig' s3config: $ref: '#/components/schemas/S3Config' gcsconfig: diff --git a/templates/webadmin/fsconfig.html b/templates/webadmin/fsconfig.html index cb2a6cdd..724f176e 100644 --- a/templates/webadmin/fsconfig.html +++ b/templates/webadmin/fsconfig.html @@ -53,6 +53,25 @@ along with this program. If not, see . {{end}} +
+ +
+ + + Buffer size for downloads. 0 means no buffer, that's fine in most use cases. + +
+
+ +
+ + + Buffer size for uploads. 0 means no buffer, that's fine in most use cases. + +
+
@@ -426,6 +445,26 @@ along with this program. If not, see .
+
+ +
+ + + Buffer size for downloads. 0 means no buffer, that's fine in most use cases. + +
+
+ +
+ + + Buffer size for uploads. 0 means no buffer, that's fine in most use cases. + +
+
+