diff --git a/go.mod b/go.mod
index 299414a6..2cb1e65a 100644
--- a/go.mod
+++ b/go.mod
@@ -53,7 +53,7 @@ require (
github.com/rs/cors v1.9.0
github.com/rs/xid v1.5.0
github.com/rs/zerolog v1.29.1
- github.com/sftpgo/sdk v0.1.4-0.20230512160325-38e59551f700
+ github.com/sftpgo/sdk v0.1.4-0.20230514135418-8b5d36c556e0
github.com/shirou/gopsutil/v3 v3.23.4
github.com/spf13/afero v1.9.5
github.com/spf13/cobra v1.7.0
@@ -148,7 +148,7 @@ require (
github.com/prometheus/procfs v0.9.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
- github.com/spf13/cast v1.5.0 // indirect
+ github.com/spf13/cast v1.5.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/tklauser/go-sysconf v0.3.11 // indirect
diff --git a/go.sum b/go.sum
index 7281f60a..b2a19b0f 100644
--- a/go.sum
+++ b/go.sum
@@ -935,8 +935,8 @@ github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHqu
github.com/franela/goblin v0.0.0-20210519012713-85d372ac71e2/go.mod h1:VzmDKDJVZI3aJmnRI9VjAn9nJ8qPPsN1fqzr9dqInIo=
github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
-github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
+github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
@@ -1842,8 +1842,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg
github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo=
-github.com/sftpgo/sdk v0.1.4-0.20230512160325-38e59551f700 h1:jL6mfKAaFv862AnBUxIfTH9wmnuPjbWyjHQUGDo+Xt0=
-github.com/sftpgo/sdk v0.1.4-0.20230512160325-38e59551f700/go.mod h1:gDxDaU3rhp9Y92ddsE7SbQ8jdBNNWK1DKlp5eHXrsb8=
+github.com/sftpgo/sdk v0.1.4-0.20230514135418-8b5d36c556e0 h1:qUyCryFuF7zKUpNTLgn2KuQp/uYT3hKi4XU/ChJUmpo=
+github.com/sftpgo/sdk v0.1.4-0.20230514135418-8b5d36c556e0/go.mod h1:TjeoMWS0JEXt9RukJveTnaiHj4+MVLtUiDC+mY++Odk=
github.com/shirou/gopsutil/v3 v3.23.4 h1:hZwmDxZs7Ewt75DV81r4pFMqbq+di2cbt9FsQBqLD2o=
github.com/shirou/gopsutil/v3 v3.23.4/go.mod h1:ZcGxyfzAMRevhUR2+cfhXDH6gQdFYE/t8j1nsU4mPI8=
github.com/shoenig/go-m1cpu v0.1.5/go.mod h1:Wwvst4LR89UxjeFtLRMrpgRiyY4xPsejnVZym39dbAQ=
@@ -1883,8 +1883,9 @@ github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM=
github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU=
+github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA=
+github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48=
github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
diff --git a/internal/common/actions_test.go b/internal/common/actions_test.go
index 983f170f..e980c3bf 100644
--- a/internal/common/actions_test.go
+++ b/internal/common/actions_test.go
@@ -262,7 +262,7 @@ func TestPreDeleteAction(t *testing.T) {
}
user.Permissions = make(map[string][]string)
user.Permissions["/"] = []string{dataprovider.PermAny}
- fs := vfs.NewOsFs("id", homeDir, "")
+ fs := vfs.NewOsFs("id", homeDir, "", nil)
c := NewBaseConnection("id", ProtocolSFTP, "", "", user)
testfile := filepath.Join(user.HomeDir, "testfile")
diff --git a/internal/common/common_test.go b/internal/common/common_test.go
index 885d13d1..cbb311d8 100644
--- a/internal/common/common_test.go
+++ b/internal/common/common_test.go
@@ -861,7 +861,7 @@ func TestConnectionStatus(t *testing.T) {
Username: username,
},
}
- fs := vfs.NewOsFs("", os.TempDir(), "")
+ fs := vfs.NewOsFs("", os.TempDir(), "", nil)
c1 := NewBaseConnection("id1", ProtocolSFTP, "", "", user)
fakeConn1 := &fakeConnection{
BaseConnection: c1,
diff --git a/internal/common/connection_test.go b/internal/common/connection_test.go
index c6cdbefb..d680f3ce 100644
--- a/internal/common/connection_test.go
+++ b/internal/common/connection_test.go
@@ -86,7 +86,7 @@ func (fs *MockOsFs) Walk(_ string, walkFn filepath.WalkFunc) error {
func newMockOsFs(hasVirtualFolders bool, connectionID, rootDir, name string, err error) vfs.Fs {
return &MockOsFs{
- Fs: vfs.NewOsFs(connectionID, rootDir, ""),
+ Fs: vfs.NewOsFs(connectionID, rootDir, "", nil),
name: name,
hasVirtualFolders: hasVirtualFolders,
err: err,
@@ -114,7 +114,7 @@ func TestRemoveErrors(t *testing.T) {
}
user.Permissions = make(map[string][]string)
user.Permissions["/"] = []string{dataprovider.PermAny}
- fs := vfs.NewOsFs("", os.TempDir(), "")
+ fs := vfs.NewOsFs("", os.TempDir(), "", nil)
conn := NewBaseConnection("", ProtocolFTP, "", "", user)
err := conn.IsRemoveDirAllowed(fs, mappedPath, "/virtualpath1")
if assert.Error(t, err) {
@@ -159,7 +159,7 @@ func TestSetStatMode(t *testing.T) {
}
func TestRecursiveRenameWalkError(t *testing.T) {
- fs := vfs.NewOsFs("", filepath.Clean(os.TempDir()), "")
+ fs := vfs.NewOsFs("", filepath.Clean(os.TempDir()), "", nil)
conn := NewBaseConnection("", ProtocolWebDAV, "", "", dataprovider.User{
BaseUser: sdk.BaseUser{
Permissions: map[string][]string{
@@ -193,7 +193,7 @@ func TestRecursiveRenameWalkError(t *testing.T) {
}
func TestCrossRenameFsErrors(t *testing.T) {
- fs := vfs.NewOsFs("", os.TempDir(), "")
+ fs := vfs.NewOsFs("", os.TempDir(), "", nil)
conn := NewBaseConnection("", ProtocolWebDAV, "", "", dataprovider.User{})
res := conn.hasSpaceForCrossRename(fs, vfs.QuotaCheckResult{}, 1, "missingsource")
assert.False(t, res)
@@ -224,7 +224,7 @@ func TestRenameVirtualFolders(t *testing.T) {
},
VirtualPath: vdir,
})
- fs := vfs.NewOsFs("", os.TempDir(), "")
+ fs := vfs.NewOsFs("", os.TempDir(), "", nil)
conn := NewBaseConnection("", ProtocolFTP, "", "", u)
res := conn.isRenamePermitted(fs, fs, "source", "target", vdir, "vdirtarget", nil)
assert.False(t, res)
@@ -376,7 +376,7 @@ func TestUpdateQuotaAfterRename(t *testing.T) {
}
func TestErrorsMapping(t *testing.T) {
- fs := vfs.NewOsFs("", os.TempDir(), "")
+ fs := vfs.NewOsFs("", os.TempDir(), "", nil)
conn := NewBaseConnection("", ProtocolSFTP, "", "", dataprovider.User{BaseUser: sdk.BaseUser{HomeDir: os.TempDir()}})
osErrorsProtocols := []string{ProtocolWebDAV, ProtocolFTP, ProtocolHTTP, ProtocolHTTPShare,
ProtocolDataRetention, ProtocolOIDC, protocolEventAction}
diff --git a/internal/common/transfer.go b/internal/common/transfer.go
index 0a0a7cbb..5e02afbe 100644
--- a/internal/common/transfer.go
+++ b/internal/common/transfer.go
@@ -198,8 +198,8 @@ func (t *BaseTransfer) SetTimes(fsPath string, atime time.Time, mtime time.Time)
// If atomic uploads are enabled this differ from fsPath
func (t *BaseTransfer) GetRealFsPath(fsPath string) string {
if fsPath == t.GetFsPath() {
- if t.File != nil {
- return t.File.Name()
+ if t.File != nil || vfs.IsLocalOsFs(t.Fs) {
+ return t.effectiveFsPath
}
return t.fsPath
}
@@ -289,9 +289,9 @@ func (t *BaseTransfer) Truncate(fsPath string, size int64) (int64, error) {
return initialSize, err
}
if size == 0 && t.BytesSent.Load() == 0 {
- // for cloud providers the file is always truncated to zero, we don't support append/resume for uploads
- // for buffered SFTP we can have buffered bytes so we returns an error
- if !vfs.IsBufferedSFTPFs(t.Fs) {
+ // for cloud providers the file is always truncated to zero, we don't support append/resume for uploads.
+ // For buffered SFTP and local fs we can have buffered bytes so we returns an error
+ if !vfs.IsBufferedLocalOrSFTPFs(t.Fs) {
return 0, nil
}
}
@@ -373,16 +373,16 @@ func (t *BaseTransfer) Close() error {
dataprovider.UpdateUserTransferQuota(&t.Connection.User, t.BytesReceived.Load(), //nolint:errcheck
t.BytesSent.Load(), false)
}
- if t.File != nil && t.Connection.IsQuotaExceededError(t.ErrTransfer) {
+ if (t.File != nil || vfs.IsLocalOsFs(t.Fs)) && t.Connection.IsQuotaExceededError(t.ErrTransfer) {
// if quota is exceeded we try to remove the partial file for uploads to local filesystem
- err = t.Fs.Remove(t.File.Name(), false)
+ err = t.Fs.Remove(t.effectiveFsPath, false)
if err == nil {
t.BytesReceived.Store(0)
t.MinWriteOffset = 0
}
t.Connection.Log(logger.LevelWarn, "upload denied due to space limit, delete temporary file: %q, deletion error: %v",
- t.File.Name(), err)
- } else if t.transferType == TransferUpload && t.effectiveFsPath != t.fsPath {
+ t.effectiveFsPath, err)
+ } else if t.isAtomicUpload() {
if t.ErrTransfer == nil || Config.UploadMode == UploadModeAtomicWithResume {
_, _, err = t.Fs.Rename(t.effectiveFsPath, t.fsPath)
t.Connection.Log(logger.LevelDebug, "atomic upload completed, rename: %q -> %q, error: %v",
@@ -436,6 +436,10 @@ func (t *BaseTransfer) Close() error {
return err
}
+func (t *BaseTransfer) isAtomicUpload() bool {
+ return t.transferType == TransferUpload && t.effectiveFsPath != t.fsPath
+}
+
func (t *BaseTransfer) updateTransferTimestamps(uploadFileSize, elapsed int64) {
if t.ErrTransfer != nil {
return
diff --git a/internal/common/transfer_test.go b/internal/common/transfer_test.go
index dcd2eac5..c3440e59 100644
--- a/internal/common/transfer_test.go
+++ b/internal/common/transfer_test.go
@@ -36,7 +36,7 @@ func TestTransferUpdateQuota(t *testing.T) {
transfer := BaseTransfer{
Connection: conn,
transferType: TransferUpload,
- Fs: vfs.NewOsFs("", os.TempDir(), ""),
+ Fs: vfs.NewOsFs("", os.TempDir(), "", nil),
}
transfer.BytesReceived.Store(123)
errFake := errors.New("fake error")
@@ -75,7 +75,7 @@ func TestTransferThrottling(t *testing.T) {
DownloadBandwidth: 40,
},
}
- fs := vfs.NewOsFs("", os.TempDir(), "")
+ fs := vfs.NewOsFs("", os.TempDir(), "", nil)
testFileSize := int64(131072)
wantedUploadElapsed := 1000 * (testFileSize / 1024) / u.UploadBandwidth
wantedDownloadElapsed := 1000 * (testFileSize / 1024) / u.DownloadBandwidth
@@ -107,7 +107,7 @@ func TestTransferThrottling(t *testing.T) {
func TestRealPath(t *testing.T) {
testFile := filepath.Join(os.TempDir(), "afile.txt")
- fs := vfs.NewOsFs("123", os.TempDir(), "")
+ fs := vfs.NewOsFs("123", os.TempDir(), "", nil)
u := dataprovider.User{
BaseUser: sdk.BaseUser{
Username: "user",
@@ -141,7 +141,7 @@ func TestRealPath(t *testing.T) {
func TestTruncate(t *testing.T) {
testFile := filepath.Join(os.TempDir(), "transfer_test_file")
- fs := vfs.NewOsFs("123", os.TempDir(), "")
+ fs := vfs.NewOsFs("123", os.TempDir(), "", nil)
u := dataprovider.User{
BaseUser: sdk.BaseUser{
Username: "user",
@@ -210,7 +210,7 @@ func TestTransferErrors(t *testing.T) {
isCancelled = true
}
testFile := filepath.Join(os.TempDir(), "transfer_test_file")
- fs := vfs.NewOsFs("id", os.TempDir(), "")
+ fs := vfs.NewOsFs("id", os.TempDir(), "", nil)
u := dataprovider.User{
BaseUser: sdk.BaseUser{
Username: "test",
@@ -321,7 +321,7 @@ func TestFTPMode(t *testing.T) {
transfer := BaseTransfer{
Connection: conn,
transferType: TransferUpload,
- Fs: vfs.NewOsFs("", os.TempDir(), ""),
+ Fs: vfs.NewOsFs("", os.TempDir(), "", nil),
}
transfer.BytesReceived.Store(123)
assert.Empty(t, transfer.ftpMode)
@@ -399,7 +399,7 @@ func TestTransferQuota(t *testing.T) {
conn := NewBaseConnection("", ProtocolSFTP, "", "", user)
transfer := NewBaseTransfer(nil, conn, nil, "file.txt", "file.txt", "/transfer_test_file", TransferUpload,
- 0, 0, 0, 0, true, vfs.NewOsFs("", os.TempDir(), ""), dataprovider.TransferQuota{})
+ 0, 0, 0, 0, true, vfs.NewOsFs("", os.TempDir(), "", nil), dataprovider.TransferQuota{})
err := transfer.CheckRead()
assert.NoError(t, err)
err = transfer.CheckWrite()
@@ -453,7 +453,7 @@ func TestUploadOutsideHomeRenameError(t *testing.T) {
transfer := BaseTransfer{
Connection: conn,
transferType: TransferUpload,
- Fs: vfs.NewOsFs("", filepath.Join(os.TempDir(), "home"), ""),
+ Fs: vfs.NewOsFs("", filepath.Join(os.TempDir(), "home"), "", nil),
}
transfer.BytesReceived.Store(123)
diff --git a/internal/dataprovider/dataprovider.go b/internal/dataprovider/dataprovider.go
index 6edd43b9..7f8eef2f 100644
--- a/internal/dataprovider/dataprovider.go
+++ b/internal/dataprovider/dataprovider.go
@@ -2622,11 +2622,12 @@ func getVirtualFolderIfInvalid(folder *vfs.BaseVirtualFolder) *vfs.BaseVirtualFo
if err := ValidateFolder(folder); err == nil {
return folder
}
- // we try to get the folder from the data provider if only the Name is populated
- if folder.MappedPath != "" {
+ if folder.Name == "" {
return folder
}
- if folder.Name == "" {
+ // we try to get the folder from the data provider if only the Name is populated
+ // so if MappedPath or Provider are set just return
+ if folder.MappedPath != "" {
return folder
}
if folder.FsConfig.Provider != sdk.LocalFilesystemProvider {
diff --git a/internal/dataprovider/user.go b/internal/dataprovider/user.go
index 9cb020f3..5d53e38f 100644
--- a/internal/dataprovider/user.go
+++ b/internal/dataprovider/user.go
@@ -173,7 +173,7 @@ func (u *User) getRootFs(connectionID string) (fs vfs.Fs, err error) {
case sdk.HTTPFilesystemProvider:
return vfs.NewHTTPFs(connectionID, u.GetHomeDir(), "", u.FsConfig.HTTPConfig)
default:
- return vfs.NewOsFs(connectionID, u.GetHomeDir(), ""), nil
+ return vfs.NewOsFs(connectionID, u.GetHomeDir(), "", &u.FsConfig.OSConfig), nil
}
}
@@ -218,7 +218,7 @@ func (u *User) checkLocalHomeDir(connectionID string) {
case sdk.LocalFilesystemProvider, sdk.CryptedFilesystemProvider:
return
default:
- osFs := vfs.NewOsFs(connectionID, u.GetHomeDir(), "")
+ osFs := vfs.NewOsFs(connectionID, u.GetHomeDir(), "", nil)
osFs.CheckRootPath(u.Username, u.GetUID(), u.GetGID())
}
}
@@ -1631,7 +1631,7 @@ func (u *User) applyGroupSettings(groupsMapping map[string]Group) {
for _, g := range u.Groups {
if g.Type == sdk.GroupTypePrimary {
if group, ok := groupsMapping[g.Name]; ok {
- u.mergeWithPrimaryGroup(group, replacer)
+ u.mergeWithPrimaryGroup(&group, replacer)
} else {
providerLog(logger.LevelError, "mapping not found for user %s, group %s", u.Username, g.Name)
}
@@ -1641,7 +1641,7 @@ func (u *User) applyGroupSettings(groupsMapping map[string]Group) {
for _, g := range u.Groups {
if g.Type == sdk.GroupTypeSecondary {
if group, ok := groupsMapping[g.Name]; ok {
- u.mergeAdditiveProperties(group, sdk.GroupTypeSecondary, replacer)
+ u.mergeAdditiveProperties(&group, sdk.GroupTypeSecondary, replacer)
} else {
providerLog(logger.LevelError, "mapping not found for user %s, group %s", u.Username, g.Name)
}
@@ -1674,17 +1674,19 @@ func (u *User) LoadAndApplyGroupSettings() error {
}
replacer := u.getGroupPlacehodersReplacer()
// make sure to always merge with the primary group first
- for idx, g := range groups {
+ for idx := range groups {
+ g := groups[idx]
if g.Name == primaryGroupName {
- u.mergeWithPrimaryGroup(g, replacer)
+ u.mergeWithPrimaryGroup(&g, replacer)
lastIdx := len(groups) - 1
groups[idx] = groups[lastIdx]
groups = groups[:lastIdx]
break
}
}
- for _, g := range groups {
- u.mergeAdditiveProperties(g, sdk.GroupTypeSecondary, replacer)
+ for idx := range groups {
+ g := groups[idx]
+ u.mergeAdditiveProperties(&g, sdk.GroupTypeSecondary, replacer)
}
u.removeDuplicatesAfterGroupMerge()
return nil
@@ -1718,12 +1720,31 @@ func (u *User) replaceFsConfigPlaceholders(fsConfig vfs.Filesystem, replacer *st
return fsConfig
}
-func (u *User) mergeWithPrimaryGroup(group Group, replacer *strings.Replacer) {
+func (u *User) mergeCryptFsConfig(group *Group) {
+ if group.UserSettings.FsConfig.Provider == sdk.CryptedFilesystemProvider {
+ if u.FsConfig.CryptConfig.ReadBufferSize == 0 {
+ u.FsConfig.CryptConfig.ReadBufferSize = group.UserSettings.FsConfig.CryptConfig.ReadBufferSize
+ }
+ if u.FsConfig.CryptConfig.WriteBufferSize == 0 {
+ u.FsConfig.CryptConfig.WriteBufferSize = group.UserSettings.FsConfig.CryptConfig.WriteBufferSize
+ }
+ }
+}
+
+func (u *User) mergeWithPrimaryGroup(group *Group, replacer *strings.Replacer) {
if group.UserSettings.HomeDir != "" {
u.HomeDir = u.replacePlaceholder(group.UserSettings.HomeDir, replacer)
}
if group.UserSettings.FsConfig.Provider != 0 {
u.FsConfig = u.replaceFsConfigPlaceholders(group.UserSettings.FsConfig, replacer)
+ u.mergeCryptFsConfig(group)
+ } else {
+ if u.FsConfig.OSConfig.ReadBufferSize == 0 {
+ u.FsConfig.OSConfig.ReadBufferSize = group.UserSettings.FsConfig.OSConfig.ReadBufferSize
+ }
+ if u.FsConfig.OSConfig.WriteBufferSize == 0 {
+ u.FsConfig.OSConfig.WriteBufferSize = group.UserSettings.FsConfig.OSConfig.WriteBufferSize
+ }
}
if u.MaxSessions == 0 {
u.MaxSessions = group.UserSettings.MaxSessions
@@ -1748,11 +1769,11 @@ func (u *User) mergeWithPrimaryGroup(group Group, replacer *strings.Replacer) {
if u.ExpirationDate == 0 && group.UserSettings.ExpiresIn > 0 {
u.ExpirationDate = u.CreatedAt + int64(group.UserSettings.ExpiresIn)*86400000
}
- u.mergePrimaryGroupFilters(group.UserSettings.Filters, replacer)
+ u.mergePrimaryGroupFilters(&group.UserSettings.Filters, replacer)
u.mergeAdditiveProperties(group, sdk.GroupTypePrimary, replacer)
}
-func (u *User) mergePrimaryGroupFilters(filters sdk.BaseUserFilters, replacer *strings.Replacer) {
+func (u *User) mergePrimaryGroupFilters(filters *sdk.BaseUserFilters, replacer *strings.Replacer) {
if u.Filters.MaxUploadFileSize == 0 {
u.Filters.MaxUploadFileSize = filters.MaxUploadFileSize
}
@@ -1797,7 +1818,7 @@ func (u *User) mergePrimaryGroupFilters(filters sdk.BaseUserFilters, replacer *s
}
}
-func (u *User) mergeAdditiveProperties(group Group, groupType int, replacer *strings.Replacer) {
+func (u *User) mergeAdditiveProperties(group *Group, groupType int, replacer *strings.Replacer) {
u.mergeVirtualFolders(group, groupType, replacer)
u.mergePermissions(group, groupType, replacer)
u.mergeFilePatterns(group, groupType, replacer)
@@ -1811,7 +1832,7 @@ func (u *User) mergeAdditiveProperties(group Group, groupType int, replacer *str
u.Filters.TwoFactorAuthProtocols = append(u.Filters.TwoFactorAuthProtocols, group.UserSettings.Filters.TwoFactorAuthProtocols...)
}
-func (u *User) mergeVirtualFolders(group Group, groupType int, replacer *strings.Replacer) {
+func (u *User) mergeVirtualFolders(group *Group, groupType int, replacer *strings.Replacer) {
if len(group.VirtualFolders) > 0 {
folderPaths := make(map[string]bool)
for _, folder := range u.VirtualFolders {
@@ -1831,7 +1852,7 @@ func (u *User) mergeVirtualFolders(group Group, groupType int, replacer *strings
}
}
-func (u *User) mergePermissions(group Group, groupType int, replacer *strings.Replacer) {
+func (u *User) mergePermissions(group *Group, groupType int, replacer *strings.Replacer) {
for k, v := range group.UserSettings.Permissions {
if k == "/" {
if groupType == sdk.GroupTypePrimary {
@@ -1847,7 +1868,7 @@ func (u *User) mergePermissions(group Group, groupType int, replacer *strings.Re
}
}
-func (u *User) mergeFilePatterns(group Group, groupType int, replacer *strings.Replacer) {
+func (u *User) mergeFilePatterns(group *Group, groupType int, replacer *strings.Replacer) {
if len(group.UserSettings.Filters.FilePatterns) > 0 {
patternPaths := make(map[string]bool)
for _, pattern := range u.Filters.FilePatterns {
diff --git a/internal/ftpd/cryptfs_test.go b/internal/ftpd/cryptfs_test.go
index 11549354..950d99b9 100644
--- a/internal/ftpd/cryptfs_test.go
+++ b/internal/ftpd/cryptfs_test.go
@@ -136,6 +136,51 @@ func TestBasicFTPHandlingCryptFs(t *testing.T) {
50*time.Millisecond)
}
+func TestBufferedCryptFs(t *testing.T) {
+ u := getTestUserWithCryptFs()
+ u.FsConfig.CryptConfig.OSFsConfig = sdk.OSFsConfig{
+ ReadBufferSize: 1,
+ WriteBufferSize: 1,
+ }
+ user, _, err := httpdtest.AddUser(u, http.StatusCreated)
+ assert.NoError(t, err)
+ client, err := getFTPClient(user, true, nil)
+ if assert.NoError(t, err) {
+ testFilePath := filepath.Join(homeBasePath, testFileName)
+ testFileSize := int64(65535)
+ err = createTestFile(testFilePath, testFileSize)
+ assert.NoError(t, err)
+
+ err = checkBasicFTP(client)
+ assert.NoError(t, err)
+ err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0)
+ assert.NoError(t, err)
+ // overwrite an existing file
+ err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0)
+ assert.NoError(t, err)
+ localDownloadPath := filepath.Join(homeBasePath, testDLFileName)
+ err = ftpDownloadFile(testFileName, localDownloadPath, testFileSize, client, 0)
+ assert.NoError(t, err)
+ info, err := os.Stat(localDownloadPath)
+ if assert.NoError(t, err) {
+ assert.Equal(t, testFileSize, info.Size())
+ }
+ err = os.Remove(testFilePath)
+ assert.NoError(t, err)
+ err = os.Remove(localDownloadPath)
+ assert.NoError(t, err)
+ err = client.Quit()
+ assert.NoError(t, err)
+ }
+ _, err = httpdtest.RemoveUser(user, http.StatusOK)
+ assert.NoError(t, err)
+ err = os.RemoveAll(user.GetHomeDir())
+ assert.NoError(t, err)
+ assert.Eventually(t, func() bool { return len(common.Connections.GetStats("")) == 0 }, 1*time.Second, 50*time.Millisecond)
+ assert.Eventually(t, func() bool { return common.Connections.GetClientConnections() == 0 }, 1000*time.Millisecond,
+ 50*time.Millisecond)
+}
+
func TestZeroBytesTransfersCryptFs(t *testing.T) {
u := getTestUserWithCryptFs()
user, _, err := httpdtest.AddUser(u, http.StatusCreated)
diff --git a/internal/ftpd/ftpd_test.go b/internal/ftpd/ftpd_test.go
index cc935e63..5a9e68e7 100644
--- a/internal/ftpd/ftpd_test.go
+++ b/internal/ftpd/ftpd_test.go
@@ -2083,7 +2083,16 @@ func TestResume(t *testing.T) {
assert.NoError(t, err)
sftpUser, _, err := httpdtest.AddUser(getTestSFTPUser(), http.StatusCreated)
assert.NoError(t, err)
- for _, user := range []dataprovider.User{localUser, sftpUser} {
+ u = getTestUser()
+ u.FsConfig.OSConfig = sdk.OSFsConfig{
+ ReadBufferSize: 1,
+ WriteBufferSize: 1,
+ }
+ u.Username += "_buf"
+ u.HomeDir += "_buf"
+ bufferedUser, _, err := httpdtest.AddUser(u, http.StatusCreated)
+ assert.NoError(t, err)
+ for _, user := range []dataprovider.User{localUser, sftpUser, bufferedUser} {
client, err := getFTPClient(user, true, nil)
if assert.NoError(t, err) {
testFilePath := filepath.Join(homeBasePath, testFileName)
@@ -2166,6 +2175,10 @@ func TestResume(t *testing.T) {
assert.NoError(t, err)
err = os.RemoveAll(localUser.GetHomeDir())
assert.NoError(t, err)
+ _, err = httpdtest.RemoveUser(bufferedUser, http.StatusOK)
+ assert.NoError(t, err)
+ err = os.RemoveAll(bufferedUser.GetHomeDir())
+ assert.NoError(t, err)
}
//nolint:dupl
diff --git a/internal/ftpd/internal_test.go b/internal/ftpd/internal_test.go
index 6999182a..3d88e4cd 100644
--- a/internal/ftpd/internal_test.go
+++ b/internal/ftpd/internal_test.go
@@ -398,7 +398,7 @@ func (fs MockOsFs) Rename(source, target string) (int, int64, error) {
func newMockOsFs(err, statErr error, atomicUpload bool, connectionID, rootDir string) vfs.Fs {
return &MockOsFs{
- Fs: vfs.NewOsFs(connectionID, rootDir, ""),
+ Fs: vfs.NewOsFs(connectionID, rootDir, "", nil),
err: err,
statErr: statErr,
isAtomicUploadSupported: atomicUpload,
@@ -719,7 +719,7 @@ func TestUploadFileStatError(t *testing.T) {
user.Permissions["/"] = []string{dataprovider.PermAny}
mockCC := mockFTPClientContext{}
connID := fmt.Sprintf("%v", mockCC.ID())
- fs := vfs.NewOsFs(connID, user.HomeDir, "")
+ fs := vfs.NewOsFs(connID, user.HomeDir, "", nil)
connection := &Connection{
BaseConnection: common.NewBaseConnection(connID, common.ProtocolFTP, "", "", user),
clientContext: mockCC,
@@ -809,7 +809,7 @@ func TestUploadOverwriteErrors(t *testing.T) {
_, err = connection.handleFTPUploadToExistingFile(fs, os.O_TRUNC, filepath.Join(os.TempDir(), "sub", "file"),
filepath.Join(os.TempDir(), "sub", "file1"), 0, "/sub/file1")
assert.Error(t, err)
- fs = vfs.NewOsFs(connID, user.GetHomeDir(), "")
+ fs = vfs.NewOsFs(connID, user.GetHomeDir(), "", nil)
_, err = connection.handleFTPUploadToExistingFile(fs, 0, "missing1", "missing2", 0, "missing")
assert.Error(t, err)
}
diff --git a/internal/httpd/httpd_test.go b/internal/httpd/httpd_test.go
index 606091d6..659eab92 100644
--- a/internal/httpd/httpd_test.go
+++ b/internal/httpd/httpd_test.go
@@ -617,6 +617,10 @@ func TestBasicUserHandling(t *testing.T) {
assert.True(t, user.HasPassword)
user.Email = "invalid@email"
+ user.FsConfig.OSConfig = sdk.OSFsConfig{
+ ReadBufferSize: 1,
+ WriteBufferSize: 2,
+ }
_, body, err := httpdtest.UpdateUser(user, http.StatusBadRequest, "")
assert.NoError(t, err)
assert.Contains(t, string(body), "Validation error: email")
@@ -887,6 +891,12 @@ func TestGroupRelations(t *testing.T) {
_, _, err := httpdtest.AddFolder(vfs.BaseVirtualFolder{
Name: folderName2,
MappedPath: mappedPath2,
+ FsConfig: vfs.Filesystem{
+ OSConfig: sdk.OSFsConfig{
+ ReadBufferSize: 3,
+ WriteBufferSize: 5,
+ },
+ },
}, http.StatusCreated)
assert.NoError(t, err)
g1 := getTestGroup()
@@ -1145,21 +1155,47 @@ func TestGroupSettingsOverride(t *testing.T) {
folderName1 := filepath.Base(mappedPath1)
mappedPath2 := filepath.Join(os.TempDir(), util.GenerateUniqueID())
folderName2 := filepath.Base(mappedPath2)
+ mappedPath3 := filepath.Join(os.TempDir(), util.GenerateUniqueID())
+ folderName3 := filepath.Base(mappedPath3)
g1 := getTestGroup()
g1.Name += "_1"
g1.VirtualFolders = append(g1.VirtualFolders, vfs.VirtualFolder{
BaseVirtualFolder: vfs.BaseVirtualFolder{
Name: folderName1,
MappedPath: mappedPath1,
+ FsConfig: vfs.Filesystem{
+ OSConfig: sdk.OSFsConfig{
+ ReadBufferSize: 3,
+ WriteBufferSize: 5,
+ },
+ },
},
VirtualPath: "/vdir1",
})
+ g1.UserSettings.Permissions = map[string][]string{
+ "/dir1": {dataprovider.PermUpload},
+ "/dir2": {dataprovider.PermDownload, dataprovider.PermListItems},
+ }
+ g1.UserSettings.FsConfig.OSConfig = sdk.OSFsConfig{
+ ReadBufferSize: 6,
+ WriteBufferSize: 2,
+ }
g2 := getTestGroup()
g2.Name += "_2"
+ g2.UserSettings.Permissions = map[string][]string{
+ "/dir1": {dataprovider.PermAny},
+ "/dir3": {dataprovider.PermDownload, dataprovider.PermListItems, dataprovider.PermChtimes},
+ }
g2.VirtualFolders = append(g2.VirtualFolders, vfs.VirtualFolder{
BaseVirtualFolder: vfs.BaseVirtualFolder{
Name: folderName1,
MappedPath: mappedPath1,
+ FsConfig: vfs.Filesystem{
+ OSConfig: sdk.OSFsConfig{
+ ReadBufferSize: 3,
+ WriteBufferSize: 5,
+ },
+ },
},
VirtualPath: "/vdir2",
})
@@ -1170,6 +1206,19 @@ func TestGroupSettingsOverride(t *testing.T) {
},
VirtualPath: "/vdir3",
})
+ g2.VirtualFolders = append(g2.VirtualFolders, vfs.VirtualFolder{
+ BaseVirtualFolder: vfs.BaseVirtualFolder{
+ Name: folderName3,
+ MappedPath: mappedPath3,
+ FsConfig: vfs.Filesystem{
+ OSConfig: sdk.OSFsConfig{
+ ReadBufferSize: 1,
+ WriteBufferSize: 2,
+ },
+ },
+ },
+ VirtualPath: "/vdir4",
+ })
group1, resp, err := httpdtest.AddGroup(g1, http.StatusCreated)
assert.NoError(t, err, string(resp))
group2, resp, err := httpdtest.AddGroup(g2, http.StatusCreated)
@@ -1188,19 +1237,55 @@ func TestGroupSettingsOverride(t *testing.T) {
user, _, err := httpdtest.AddUser(u, http.StatusCreated)
assert.NoError(t, err)
assert.Len(t, user.VirtualFolders, 0)
+ assert.Len(t, user.Permissions, 1)
user, err = dataprovider.CheckUserAndPass(defaultUsername, defaultPassword, "", common.ProtocolHTTP)
assert.NoError(t, err)
- assert.Len(t, user.VirtualFolders, 3)
+
+ var folderNames []string
+ if assert.Len(t, user.VirtualFolders, 4) {
+ for _, f := range user.VirtualFolders {
+ if !util.Contains(folderNames, f.Name) {
+ folderNames = append(folderNames, f.Name)
+ }
+ switch f.Name {
+ case folderName1:
+ assert.Equal(t, mappedPath1, f.MappedPath)
+ assert.Equal(t, 3, f.BaseVirtualFolder.FsConfig.OSConfig.ReadBufferSize)
+ assert.Equal(t, 5, f.BaseVirtualFolder.FsConfig.OSConfig.WriteBufferSize)
+ assert.True(t, util.Contains([]string{"/vdir1", "/vdir2"}, f.VirtualPath))
+ case folderName2:
+ assert.Equal(t, mappedPath2, f.MappedPath)
+ assert.Equal(t, "/vdir3", f.VirtualPath)
+ assert.Equal(t, 0, f.BaseVirtualFolder.FsConfig.OSConfig.ReadBufferSize)
+ assert.Equal(t, 0, f.BaseVirtualFolder.FsConfig.OSConfig.WriteBufferSize)
+ case folderName3:
+ assert.Equal(t, mappedPath3, f.MappedPath)
+ assert.Equal(t, "/vdir4", f.VirtualPath)
+ assert.Equal(t, 1, f.BaseVirtualFolder.FsConfig.OSConfig.ReadBufferSize)
+ assert.Equal(t, 2, f.BaseVirtualFolder.FsConfig.OSConfig.WriteBufferSize)
+ }
+ }
+ }
+ assert.Len(t, folderNames, 3)
+ assert.Contains(t, folderNames, folderName1)
+ assert.Contains(t, folderNames, folderName2)
+ assert.Contains(t, folderNames, folderName3)
+ assert.Len(t, user.Permissions, 4)
+ assert.Equal(t, g1.UserSettings.Permissions["/dir1"], user.Permissions["/dir1"])
+ assert.Equal(t, g1.UserSettings.Permissions["/dir2"], user.Permissions["/dir2"])
+ assert.Equal(t, g2.UserSettings.Permissions["/dir3"], user.Permissions["/dir3"])
+ assert.Equal(t, g1.UserSettings.FsConfig.OSConfig.ReadBufferSize, user.FsConfig.OSConfig.ReadBufferSize)
+ assert.Equal(t, g1.UserSettings.FsConfig.OSConfig.WriteBufferSize, user.FsConfig.OSConfig.WriteBufferSize)
user, err = dataprovider.GetUserAfterIDPAuth(defaultUsername, "", common.ProtocolOIDC, nil)
assert.NoError(t, err)
- assert.Len(t, user.VirtualFolders, 3)
+ assert.Len(t, user.VirtualFolders, 4)
user1, user2, err := dataprovider.GetUserVariants(defaultUsername, "")
assert.NoError(t, err)
assert.Len(t, user1.VirtualFolders, 0)
- assert.Len(t, user2.VirtualFolders, 3)
+ assert.Len(t, user2.VirtualFolders, 4)
assert.Equal(t, int64(0), user1.ExpirationDate)
assert.Equal(t, int64(0), user2.ExpirationDate)
@@ -1226,7 +1311,7 @@ func TestGroupSettingsOverride(t *testing.T) {
assert.NoError(t, err)
user, err = dataprovider.CheckUserAndPass(defaultUsername, defaultPassword, "", common.ProtocolHTTP)
assert.NoError(t, err)
- assert.Len(t, user.VirtualFolders, 3)
+ assert.Len(t, user.VirtualFolders, 4)
assert.Equal(t, sdk.LocalFilesystemProvider, user.FsConfig.Provider)
assert.Equal(t, int64(0), user.DownloadBandwidth)
assert.Equal(t, int64(0), user.UploadBandwidth)
@@ -1272,7 +1357,7 @@ func TestGroupSettingsOverride(t *testing.T) {
assert.NoError(t, err)
user, err = dataprovider.CheckUserAndPass(defaultUsername, defaultPassword, "", common.ProtocolHTTP)
assert.NoError(t, err)
- assert.Len(t, user.VirtualFolders, 3)
+ assert.Len(t, user.VirtualFolders, 4)
assert.Equal(t, user.CreatedAt+int64(group1.UserSettings.ExpiresIn)*86400000, user.ExpirationDate)
assert.Equal(t, group1.UserSettings.Filters.PasswordStrength, user.Filters.PasswordStrength)
assert.Equal(t, sdk.SFTPFilesystemProvider, user.FsConfig.Provider)
@@ -1307,6 +1392,8 @@ func TestGroupSettingsOverride(t *testing.T) {
assert.NoError(t, err)
_, err = httpdtest.RemoveFolder(vfs.BaseVirtualFolder{Name: folderName2}, http.StatusOK)
assert.NoError(t, err)
+ _, err = httpdtest.RemoveFolder(vfs.BaseVirtualFolder{Name: folderName3}, http.StatusOK)
+ assert.NoError(t, err)
}
func TestConfigs(t *testing.T) {
@@ -12208,7 +12295,7 @@ func TestWebClientMaxConnections(t *testing.T) {
checkResponseCode(t, http.StatusOK, rr)
// now add a fake connection
- fs := vfs.NewOsFs("id", os.TempDir(), "")
+ fs := vfs.NewOsFs("id", os.TempDir(), "", nil)
connection := &httpd.Connection{
BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolHTTP, "", "", user),
}
@@ -12399,7 +12486,7 @@ func TestMaxSessions(t *testing.T) {
apiToken, err := getJWTAPIUserTokenFromTestServer(defaultUsername, defaultPassword)
assert.NoError(t, err)
// now add a fake connection
- fs := vfs.NewOsFs("id", os.TempDir(), "")
+ fs := vfs.NewOsFs("id", os.TempDir(), "", nil)
connection := &httpd.Connection{
BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolHTTP, "", "", user),
}
@@ -13508,7 +13595,7 @@ func TestShareMaxSessions(t *testing.T) {
rr = executeRequest(req)
checkResponseCode(t, http.StatusOK, rr)
// add a fake connection
- fs := vfs.NewOsFs("id", os.TempDir(), "")
+ fs := vfs.NewOsFs("id", os.TempDir(), "", nil)
connection := &httpd.Connection{
BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolHTTP, "", "", user),
}
@@ -15887,6 +15974,105 @@ func TestWebFilesAPI(t *testing.T) {
checkResponseCode(t, http.StatusNotFound, rr)
}
+func TestBufferedWebFilesAPI(t *testing.T) {
+ u := getTestUser()
+ u.FsConfig.OSConfig = sdk.OSFsConfig{
+ ReadBufferSize: 1,
+ WriteBufferSize: 1,
+ }
+ vdirPath := "/crypted"
+ mappedPath := filepath.Join(os.TempDir(), util.GenerateUniqueID())
+ folderName := filepath.Base(mappedPath)
+ u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{
+ BaseVirtualFolder: vfs.BaseVirtualFolder{
+ Name: folderName,
+ MappedPath: mappedPath,
+ FsConfig: vfs.Filesystem{
+ Provider: sdk.CryptedFilesystemProvider,
+ CryptConfig: vfs.CryptFsConfig{
+ OSFsConfig: sdk.OSFsConfig{
+ WriteBufferSize: 3,
+ ReadBufferSize: 2,
+ },
+ Passphrase: kms.NewPlainSecret(defaultPassword),
+ },
+ },
+ },
+ VirtualPath: vdirPath,
+ QuotaFiles: -1,
+ QuotaSize: -1,
+ })
+ user, _, err := httpdtest.AddUser(u, http.StatusCreated)
+ assert.NoError(t, err)
+ webAPIToken, err := getJWTAPIUserTokenFromTestServer(defaultUsername, defaultPassword)
+ assert.NoError(t, err)
+
+ body := new(bytes.Buffer)
+ writer := multipart.NewWriter(body)
+ part1, err := writer.CreateFormFile("filenames", "file1.txt")
+ assert.NoError(t, err)
+ _, err = part1.Write([]byte("file1 content"))
+ assert.NoError(t, err)
+ err = writer.Close()
+ assert.NoError(t, err)
+ reader := bytes.NewReader(body.Bytes())
+
+ req, err := http.NewRequest(http.MethodPost, userFilesPath, reader)
+ assert.NoError(t, err)
+ req.Header.Add("Content-Type", writer.FormDataContentType())
+ setBearerForReq(req, webAPIToken)
+ rr := executeRequest(req)
+ checkResponseCode(t, http.StatusCreated, rr)
+
+ _, err = reader.Seek(0, io.SeekStart)
+ assert.NoError(t, err)
+ req, err = http.NewRequest(http.MethodPost, userFilesPath+"?path="+url.QueryEscape(vdirPath), reader)
+ assert.NoError(t, err)
+ req.Header.Add("Content-Type", writer.FormDataContentType())
+ setBearerForReq(req, webAPIToken)
+ rr = executeRequest(req)
+ checkResponseCode(t, http.StatusCreated, rr)
+
+ req, err = http.NewRequest(http.MethodGet, userFilesPath+"?path=file1.txt", nil)
+ assert.NoError(t, err)
+ setBearerForReq(req, webAPIToken)
+ rr = executeRequest(req)
+ checkResponseCode(t, http.StatusOK, rr)
+ assert.Equal(t, "file1 content", rr.Body.String())
+
+ req, err = http.NewRequest(http.MethodGet, userFilesPath+"?path="+url.QueryEscape(vdirPath+"/file1.txt"), nil)
+ assert.NoError(t, err)
+ setBearerForReq(req, webAPIToken)
+ rr = executeRequest(req)
+ checkResponseCode(t, http.StatusOK, rr)
+ assert.Equal(t, "file1 content", rr.Body.String())
+
+ req, err = http.NewRequest(http.MethodGet, userFilesPath+"?path=file1.txt", nil)
+ assert.NoError(t, err)
+ req.Header.Set("Range", "bytes=2-")
+ setBearerForReq(req, webAPIToken)
+ rr = executeRequest(req)
+ checkResponseCode(t, http.StatusPartialContent, rr)
+ assert.Equal(t, "le1 content", rr.Body.String())
+
+ req, err = http.NewRequest(http.MethodGet, userFilesPath+"?path="+url.QueryEscape(vdirPath+"/file1.txt"), nil)
+ assert.NoError(t, err)
+ req.Header.Set("Range", "bytes=3-6")
+ setBearerForReq(req, webAPIToken)
+ rr = executeRequest(req)
+ checkResponseCode(t, http.StatusPartialContent, rr)
+ assert.Equal(t, "e1 c", rr.Body.String())
+
+ _, err = httpdtest.RemoveUser(user, http.StatusOK)
+ assert.NoError(t, err)
+ err = os.RemoveAll(user.GetHomeDir())
+ assert.NoError(t, err)
+ _, err = httpdtest.RemoveFolder(vfs.BaseVirtualFolder{Name: folderName}, http.StatusOK)
+ assert.NoError(t, err)
+ err = os.RemoveAll(mappedPath)
+ assert.NoError(t, err)
+}
+
func TestStartDirectory(t *testing.T) {
u := getTestUser()
u.Filters.StartDirectory = "/start/dir"
@@ -19009,6 +19195,8 @@ func TestWebUserAddMock(t *testing.T) {
form.Set("username", user.Username)
form.Set("email", user.Email)
form.Set("home_dir", user.HomeDir)
+ form.Set("osfs_read_buffer_size", "2")
+ form.Set("osfs_write_buffer_size", "3")
form.Set("password", user.Password)
form.Set("primary_group", group1.Name)
form.Set("secondary_groups", group2.Name)
@@ -19344,6 +19532,8 @@ func TestWebUserAddMock(t *testing.T) {
err = render.DecodeJSON(rr.Body, &newUser)
assert.NoError(t, err)
assert.Equal(t, user.UID, newUser.UID)
+ assert.Equal(t, 2, newUser.FsConfig.OSConfig.ReadBufferSize)
+ assert.Equal(t, 3, newUser.FsConfig.OSConfig.WriteBufferSize)
assert.Equal(t, user.UploadBandwidth, newUser.UploadBandwidth)
assert.Equal(t, user.DownloadBandwidth, newUser.DownloadBandwidth)
assert.Equal(t, user.UploadDataTransfer, newUser.UploadDataTransfer)
@@ -21088,6 +21278,8 @@ func TestWebUserCryptMock(t *testing.T) {
form.Set("denied_ip", "")
form.Set("fs_provider", "4")
form.Set("crypt_passphrase", "")
+ form.Set("cryptfs_read_buffer_size", "1")
+ form.Set("cryptfs_write_buffer_size", "2")
form.Set("pattern_path0", "/dir1")
form.Set("patterns0", "*.jpg,*.png")
form.Set("pattern_type0", "allowed")
@@ -21125,6 +21317,8 @@ func TestWebUserCryptMock(t *testing.T) {
assert.NotEmpty(t, updateUser.FsConfig.CryptConfig.Passphrase.GetPayload())
assert.Empty(t, updateUser.FsConfig.CryptConfig.Passphrase.GetKey())
assert.Empty(t, updateUser.FsConfig.CryptConfig.Passphrase.GetAdditionalData())
+ assert.Equal(t, 1, updateUser.FsConfig.CryptConfig.ReadBufferSize)
+ assert.Equal(t, 2, updateUser.FsConfig.CryptConfig.WriteBufferSize)
// now check that a redacted password is not saved
form.Set("crypt_passphrase", redactedSecret+" ")
b, contentType, _ = getMultipartFormData(form, "", "")
@@ -22637,6 +22831,8 @@ func TestAddWebFoldersMock(t *testing.T) {
form.Set("mapped_path", mappedPath)
form.Set("name", folderName)
form.Set("description", folderDesc)
+ form.Set("osfs_read_buffer_size", "3")
+ form.Set("osfs_write_buffer_size", "4")
b, contentType, err := getMultipartFormData(form, "", "")
assert.NoError(t, err)
req, err := http.NewRequest(http.MethodPost, webFolderPath, &b)
@@ -22690,6 +22886,8 @@ func TestAddWebFoldersMock(t *testing.T) {
assert.Equal(t, mappedPath, folder.MappedPath)
assert.Equal(t, folderName, folder.Name)
assert.Equal(t, folderDesc, folder.Description)
+ assert.Equal(t, 3, folder.FsConfig.OSConfig.ReadBufferSize)
+ assert.Equal(t, 4, folder.FsConfig.OSConfig.WriteBufferSize)
// cleanup
req, _ = http.NewRequest(http.MethodDelete, path.Join(folderPath, folderName), nil)
setBearerForReq(req, apiToken)
diff --git a/internal/httpd/webadmin.go b/internal/httpd/webadmin.go
index 022ab8b9..5ef82bae 100644
--- a/internal/httpd/webadmin.go
+++ b/internal/httpd/webadmin.go
@@ -1747,10 +1747,25 @@ func getAzureConfig(r *http.Request) (vfs.AzBlobFsConfig, error) {
return config, nil
}
+func getOsConfigFromPostFields(r *http.Request, readBufferField, writeBufferField string) sdk.OSFsConfig {
+ config := sdk.OSFsConfig{}
+ readBuffer, err := strconv.Atoi(r.Form.Get(readBufferField))
+ if err == nil {
+ config.ReadBufferSize = readBuffer
+ }
+ writeBuffer, err := strconv.Atoi(r.Form.Get(writeBufferField))
+ if err == nil {
+ config.WriteBufferSize = writeBuffer
+ }
+ return config
+}
+
func getFsConfigFromPostFields(r *http.Request) (vfs.Filesystem, error) {
var fs vfs.Filesystem
fs.Provider = sdk.GetProviderByName(r.Form.Get("fs_provider"))
switch fs.Provider {
+ case sdk.LocalFilesystemProvider:
+ fs.OSConfig = getOsConfigFromPostFields(r, "osfs_read_buffer_size", "osfs_write_buffer_size")
case sdk.S3FilesystemProvider:
config, err := getS3Config(r)
if err != nil {
@@ -1771,6 +1786,7 @@ func getFsConfigFromPostFields(r *http.Request) (vfs.Filesystem, error) {
fs.GCSConfig = config
case sdk.CryptedFilesystemProvider:
fs.CryptConfig.Passphrase = getSecretFromFormField(r, "crypt_passphrase")
+ fs.CryptConfig.OSFsConfig = getOsConfigFromPostFields(r, "cryptfs_read_buffer_size", "cryptfs_write_buffer_size")
case sdk.SFTPFilesystemProvider:
config, err := getSFTPConfig(r)
if err != nil {
diff --git a/internal/httpdtest/httpdtest.go b/internal/httpdtest/httpdtest.go
index b4b71589..e53a9763 100644
--- a/internal/httpdtest/httpdtest.go
+++ b/internal/httpdtest/httpdtest.go
@@ -2129,6 +2129,12 @@ func compareFsConfig(expected *vfs.Filesystem, actual *vfs.Filesystem) error {
if expected.Provider != actual.Provider {
return errors.New("fs provider mismatch")
}
+ if expected.OSConfig.ReadBufferSize != actual.OSConfig.ReadBufferSize {
+ return fmt.Errorf("read buffer size mismatch")
+ }
+ if expected.OSConfig.WriteBufferSize != actual.OSConfig.WriteBufferSize {
+ return fmt.Errorf("write buffer size mismatch")
+ }
if err := compareS3Config(expected, actual); err != nil {
return err
}
@@ -2141,6 +2147,12 @@ func compareFsConfig(expected *vfs.Filesystem, actual *vfs.Filesystem) error {
if err := checkEncryptedSecret(expected.CryptConfig.Passphrase, actual.CryptConfig.Passphrase); err != nil {
return err
}
+ if expected.CryptConfig.ReadBufferSize != actual.CryptConfig.ReadBufferSize {
+ return fmt.Errorf("crypt read buffer size mismatch")
+ }
+ if expected.CryptConfig.WriteBufferSize != actual.CryptConfig.WriteBufferSize {
+ return fmt.Errorf("crypt write buffer size mismatch")
+ }
if err := compareSFTPFsConfig(expected, actual); err != nil {
return err
}
diff --git a/internal/sftpd/internal_test.go b/internal/sftpd/internal_test.go
index 36caac36..baddaea9 100644
--- a/internal/sftpd/internal_test.go
+++ b/internal/sftpd/internal_test.go
@@ -149,7 +149,7 @@ func (fs MockOsFs) Rename(source, target string) (int, int64, error) {
func newMockOsFs(err, statErr error, atomicUpload bool, connectionID, rootDir string) vfs.Fs {
return &MockOsFs{
- Fs: vfs.NewOsFs(connectionID, rootDir, ""),
+ Fs: vfs.NewOsFs(connectionID, rootDir, "", nil),
err: err,
statErr: statErr,
isAtomicUploadSupported: atomicUpload,
@@ -183,7 +183,7 @@ func TestUploadResumeInvalidOffset(t *testing.T) {
Username: "testuser",
},
}
- fs := vfs.NewOsFs("", os.TempDir(), "")
+ fs := vfs.NewOsFs("", os.TempDir(), "", nil)
conn := common.NewBaseConnection("", common.ProtocolSFTP, "", "", user)
baseTransfer := common.NewBaseTransfer(file, conn, nil, file.Name(), file.Name(), testfile,
common.TransferUpload, 10, 0, 0, 0, false, fs, dataprovider.TransferQuota{})
@@ -214,7 +214,7 @@ func TestReadWriteErrors(t *testing.T) {
Username: "testuser",
},
}
- fs := vfs.NewOsFs("", os.TempDir(), "")
+ fs := vfs.NewOsFs("", os.TempDir(), "", nil)
conn := common.NewBaseConnection("", common.ProtocolSFTP, "", "", user)
baseTransfer := common.NewBaseTransfer(file, conn, nil, file.Name(), file.Name(), testfile, common.TransferDownload,
0, 0, 0, 0, false, fs, dataprovider.TransferQuota{})
@@ -288,7 +288,7 @@ func TestTransferCancelFn(t *testing.T) {
Username: "testuser",
},
}
- fs := vfs.NewOsFs("", os.TempDir(), "")
+ fs := vfs.NewOsFs("", os.TempDir(), "", nil)
conn := common.NewBaseConnection("", common.ProtocolSFTP, "", "", user)
baseTransfer := common.NewBaseTransfer(file, conn, cancelFn, file.Name(), file.Name(), testfile, common.TransferDownload,
0, 0, 0, 0, false, fs, dataprovider.TransferQuota{})
@@ -311,7 +311,7 @@ func TestTransferCancelFn(t *testing.T) {
func TestUploadFiles(t *testing.T) {
common.Config.UploadMode = common.UploadModeAtomic
- fs := vfs.NewOsFs("123", os.TempDir(), "")
+ fs := vfs.NewOsFs("123", os.TempDir(), "", nil)
u := dataprovider.User{}
c := Connection{
BaseConnection: common.NewBaseConnection("", common.ProtocolSFTP, "", "", u),
@@ -1213,7 +1213,7 @@ func TestSCPParseUploadMessage(t *testing.T) {
StdErrBuffer: bytes.NewBuffer(stdErrBuf),
ReadError: nil,
}
- fs := vfs.NewOsFs("", os.TempDir(), "")
+ fs := vfs.NewOsFs("", os.TempDir(), "", nil)
connection := &Connection{
BaseConnection: common.NewBaseConnection("", common.ProtocolSFTP, "", "", dataprovider.User{
BaseUser: sdk.BaseUser{
@@ -1470,7 +1470,7 @@ func TestSCPRecursiveDownloadErrors(t *testing.T) {
err := client.Close()
assert.NoError(t, err)
}()
- fs := vfs.NewOsFs("123", os.TempDir(), "")
+ fs := vfs.NewOsFs("123", os.TempDir(), "", nil)
connection := &Connection{
BaseConnection: common.NewBaseConnection("", common.ProtocolSCP, "", "", dataprovider.User{
BaseUser: sdk.BaseUser{
@@ -1593,7 +1593,7 @@ func TestSCPDownloadFileData(t *testing.T) {
ReadError: nil,
WriteError: writeErr,
}
- fs := vfs.NewOsFs("", os.TempDir(), "")
+ fs := vfs.NewOsFs("", os.TempDir(), "", nil)
connection := &Connection{
BaseConnection: common.NewBaseConnection("", common.ProtocolSCP, "", "", dataprovider.User{BaseUser: sdk.BaseUser{HomeDir: os.TempDir()}}),
channel: &mockSSHChannelReadErr,
@@ -1645,7 +1645,7 @@ func TestSCPUploadFiledata(t *testing.T) {
Username: "testuser",
},
}
- fs := vfs.NewOsFs("", os.TempDir(), "")
+ fs := vfs.NewOsFs("", os.TempDir(), "", nil)
connection := &Connection{
BaseConnection: common.NewBaseConnection("", common.ProtocolSCP, "", "", user),
channel: &mockSSHChannel,
@@ -1736,7 +1736,7 @@ func TestUploadError(t *testing.T) {
Username: "testuser",
},
}
- fs := vfs.NewOsFs("", os.TempDir(), "")
+ fs := vfs.NewOsFs("", os.TempDir(), "", nil)
connection := &Connection{
BaseConnection: common.NewBaseConnection("", common.ProtocolSCP, "", "", user),
}
diff --git a/internal/sftpd/sftpd_test.go b/internal/sftpd/sftpd_test.go
index 7a1c9e35..65180f01 100644
--- a/internal/sftpd/sftpd_test.go
+++ b/internal/sftpd/sftpd_test.go
@@ -1428,7 +1428,17 @@ func TestUploadResume(t *testing.T) {
u = getTestSFTPUser(usePubKey)
sftpUser, _, err := httpdtest.AddUser(u, http.StatusCreated)
assert.NoError(t, err)
- for _, user := range []dataprovider.User{localUser, sftpUser} {
+ u = getTestUser(usePubKey)
+ u.FsConfig.OSConfig = sdk.OSFsConfig{
+ WriteBufferSize: 1,
+ ReadBufferSize: 1,
+ }
+ u.Username += "_buffered"
+ u.HomeDir += "_with_buf"
+ bufferedUser, _, err := httpdtest.AddUser(u, http.StatusCreated)
+ assert.NoError(t, err)
+
+ for _, user := range []dataprovider.User{localUser, sftpUser, bufferedUser} {
conn, client, err := getSftpClient(user, usePubKey)
if assert.NoError(t, err) {
defer conn.Close()
@@ -1475,8 +1485,12 @@ func TestUploadResume(t *testing.T) {
assert.NoError(t, err)
_, err = httpdtest.RemoveUser(localUser, http.StatusOK)
assert.NoError(t, err)
+ _, err = httpdtest.RemoveUser(bufferedUser, http.StatusOK)
+ assert.NoError(t, err)
err = os.RemoveAll(localUser.GetHomeDir())
assert.NoError(t, err)
+ err = os.RemoveAll(bufferedUser.GetHomeDir())
+ assert.NoError(t, err)
}
func TestDirCommands(t *testing.T) {
@@ -5570,6 +5584,129 @@ func TestNestedVirtualFolders(t *testing.T) {
assert.NoError(t, err)
}
+func TestBufferedUser(t *testing.T) {
+ usePubKey := true
+ u := getTestUser(usePubKey)
+ u.QuotaFiles = 1000
+ u.FsConfig.OSConfig = sdk.OSFsConfig{
+ WriteBufferSize: 2,
+ ReadBufferSize: 1,
+ }
+ vdirPath := "/crypted"
+ mappedPath := filepath.Join(os.TempDir(), util.GenerateUniqueID())
+ folderName := filepath.Base(mappedPath)
+ u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{
+ BaseVirtualFolder: vfs.BaseVirtualFolder{
+ Name: folderName,
+ MappedPath: mappedPath,
+ FsConfig: vfs.Filesystem{
+ Provider: sdk.CryptedFilesystemProvider,
+ CryptConfig: vfs.CryptFsConfig{
+ OSFsConfig: sdk.OSFsConfig{
+ WriteBufferSize: 3,
+ ReadBufferSize: 2,
+ },
+ Passphrase: kms.NewPlainSecret(defaultPassword),
+ },
+ },
+ },
+ VirtualPath: vdirPath,
+ QuotaFiles: -1,
+ QuotaSize: -1,
+ })
+ user, _, err := httpdtest.AddUser(u, http.StatusCreated)
+ assert.NoError(t, err)
+ conn, client, err := getSftpClient(user, usePubKey)
+ if assert.NoError(t, err) {
+ defer conn.Close()
+ defer client.Close()
+
+ expectedQuotaSize := int64(0)
+ expectedQuotaFiles := 0
+ fileSize := int64(32768)
+ err = writeSFTPFile(testFileName, fileSize, client)
+ assert.NoError(t, err)
+ expectedQuotaSize += fileSize
+ expectedQuotaFiles++
+ err = writeSFTPFile(path.Join(vdirPath, testFileName), fileSize, client)
+ assert.NoError(t, err)
+ expectedQuotaSize += fileSize
+ expectedQuotaFiles++
+ user, _, err = httpdtest.GetUserByUsername(user.Username, http.StatusOK)
+ assert.NoError(t, err)
+ assert.Equal(t, expectedQuotaFiles, user.UsedQuotaFiles)
+ assert.Greater(t, user.UsedQuotaSize, expectedQuotaSize)
+ localDownloadPath := filepath.Join(homeBasePath, testDLFileName)
+ err = sftpDownloadFile(testFileName, localDownloadPath, fileSize, client)
+ assert.NoError(t, err)
+ err = sftpDownloadFile(path.Join(vdirPath, testFileName), localDownloadPath, fileSize, client)
+ assert.NoError(t, err)
+ err = os.Remove(localDownloadPath)
+ assert.NoError(t, err)
+ err = client.Remove(testFileName)
+ assert.NoError(t, err)
+ err = client.Remove(path.Join(vdirPath, testFileName))
+ assert.NoError(t, err)
+
+ data := []byte("test data")
+ f, err := client.OpenFile(testFileName, os.O_WRONLY|os.O_CREATE)
+ if assert.NoError(t, err) {
+ n, err := f.Write(data)
+ assert.NoError(t, err)
+ assert.Equal(t, len(data), n)
+ err = f.Truncate(2)
+ assert.NoError(t, err)
+ expectedQuotaSize := int64(2)
+ expectedQuotaFiles := 0
+ user, _, err = httpdtest.GetUserByUsername(user.Username, http.StatusOK)
+ assert.NoError(t, err)
+ assert.Equal(t, expectedQuotaFiles, user.UsedQuotaFiles)
+ assert.Equal(t, expectedQuotaSize, user.UsedQuotaSize)
+ _, err = f.Seek(expectedQuotaSize, io.SeekStart)
+ assert.NoError(t, err)
+ n, err = f.Write(data)
+ assert.NoError(t, err)
+ assert.Equal(t, len(data), n)
+ err = f.Truncate(5)
+ assert.NoError(t, err)
+ expectedQuotaSize = int64(5)
+ user, _, err = httpdtest.GetUserByUsername(user.Username, http.StatusOK)
+ assert.NoError(t, err)
+ assert.Equal(t, expectedQuotaFiles, user.UsedQuotaFiles)
+ assert.Equal(t, expectedQuotaSize, user.UsedQuotaSize)
+ _, err = f.Seek(expectedQuotaSize, io.SeekStart)
+ assert.NoError(t, err)
+ n, err = f.Write(data)
+ assert.NoError(t, err)
+ assert.Equal(t, len(data), n)
+ err = f.Close()
+ assert.NoError(t, err)
+ expectedQuotaSize = int64(5) + int64(len(data))
+ expectedQuotaFiles = 1
+ user, _, err = httpdtest.GetUserByUsername(user.Username, http.StatusOK)
+ assert.NoError(t, err)
+ assert.Equal(t, expectedQuotaFiles, user.UsedQuotaFiles)
+ assert.Equal(t, expectedQuotaSize, user.UsedQuotaSize)
+ }
+ // now truncate by path
+ err = client.Truncate(testFileName, 5)
+ assert.NoError(t, err)
+ user, _, err = httpdtest.GetUserByUsername(user.Username, http.StatusOK)
+ assert.NoError(t, err)
+ assert.Equal(t, 1, user.UsedQuotaFiles)
+ assert.Equal(t, int64(5), user.UsedQuotaSize)
+ }
+
+ _, err = httpdtest.RemoveUser(user, http.StatusOK)
+ assert.NoError(t, err)
+ err = os.RemoveAll(user.GetHomeDir())
+ assert.NoError(t, err)
+ _, err = httpdtest.RemoveFolder(vfs.BaseVirtualFolder{Name: folderName}, http.StatusOK)
+ assert.NoError(t, err)
+ err = os.RemoveAll(mappedPath)
+ assert.NoError(t, err)
+}
+
func TestTruncateQuotaLimits(t *testing.T) {
usePubKey := true
u := getTestUser(usePubKey)
@@ -8047,7 +8184,7 @@ func TestRootDirCommands(t *testing.T) {
func TestRelativePaths(t *testing.T) {
user := getTestUser(true)
var path, rel string
- filesystems := []vfs.Fs{vfs.NewOsFs("", user.GetHomeDir(), "")}
+ filesystems := []vfs.Fs{vfs.NewOsFs("", user.GetHomeDir(), "", nil)}
keyPrefix := strings.TrimPrefix(user.GetHomeDir(), "/") + "/"
s3config := vfs.S3FsConfig{
BaseS3FsConfig: sdk.BaseS3FsConfig{
@@ -8112,7 +8249,7 @@ func TestResolvePaths(t *testing.T) {
user := getTestUser(true)
var path, resolved string
var err error
- filesystems := []vfs.Fs{vfs.NewOsFs("", user.GetHomeDir(), "")}
+ filesystems := []vfs.Fs{vfs.NewOsFs("", user.GetHomeDir(), "", nil)}
keyPrefix := strings.TrimPrefix(user.GetHomeDir(), "/") + "/"
s3config := vfs.S3FsConfig{
BaseS3FsConfig: sdk.BaseS3FsConfig{
@@ -8175,8 +8312,8 @@ func TestVirtualRelativePaths(t *testing.T) {
})
err := os.MkdirAll(mappedPath, os.ModePerm)
assert.NoError(t, err)
- fsRoot := vfs.NewOsFs("", user.GetHomeDir(), "")
- fsVdir := vfs.NewOsFs("", mappedPath, vdirPath)
+ fsRoot := vfs.NewOsFs("", user.GetHomeDir(), "", nil)
+ fsVdir := vfs.NewOsFs("", mappedPath, vdirPath, nil)
rel := fsVdir.GetRelativePath(mappedPath)
assert.Equal(t, vdirPath, rel)
rel = fsRoot.GetRelativePath(filepath.Join(mappedPath, ".."))
diff --git a/internal/vfs/azblobfs.go b/internal/vfs/azblobfs.go
index 739d807b..18c19eca 100644
--- a/internal/vfs/azblobfs.go
+++ b/internal/vfs/azblobfs.go
@@ -517,7 +517,7 @@ func (*AzureBlobFs) isBadRequestError(err error) bool {
// CheckRootPath creates the specified local root directory if it does not exists
func (fs *AzureBlobFs) CheckRootPath(username string, uid int, gid int) bool {
// we need a local directory for temporary files
- osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, "")
+ osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, "", nil)
return osFs.CheckRootPath(username, uid, gid)
}
diff --git a/internal/vfs/cryptfs.go b/internal/vfs/cryptfs.go
index fc0560d7..2a15d213 100644
--- a/internal/vfs/cryptfs.go
+++ b/internal/vfs/cryptfs.go
@@ -15,6 +15,7 @@
package vfs
import (
+ "bufio"
"bytes"
"crypto/rand"
"crypto/sha256"
@@ -55,10 +56,12 @@ func NewCryptFs(connectionID, rootDir, mountPath string, config CryptFsConfig) (
}
fs := &CryptFs{
OsFs: &OsFs{
- name: cryptFsName,
- connectionID: connectionID,
- rootDir: rootDir,
- mountPath: getMountPath(mountPath),
+ name: cryptFsName,
+ connectionID: connectionID,
+ rootDir: rootDir,
+ mountPath: getMountPath(mountPath),
+ readBufferSize: config.OSFsConfig.ReadBufferSize * 1024 * 1024,
+ writeBufferSize: config.OSFsConfig.WriteBufferSize * 1024 * 1024,
},
masterKey: []byte(config.Passphrase.GetPayload()),
}
@@ -103,11 +106,11 @@ func (fs *CryptFs) Open(name string, offset int64) (File, *pipeat.PipeReaderAt,
var err error
if offset == 0 {
- n, err = sio.Decrypt(w, f, fs.getSIOConfig(key))
+ n, err = fs.decryptWrapper(w, f, fs.getSIOConfig(key))
} else {
var readerAt io.ReaderAt
var readed, written int
- buf := make([]byte, 65536)
+ buf := make([]byte, 65568)
wrapper := &cryptedFileWrapper{
File: f,
}
@@ -150,14 +153,8 @@ func (fs *CryptFs) Open(name string, offset int64) (File, *pipeat.PipeReaderAt,
}
// Create creates or opens the named file for writing
-func (fs *CryptFs) Create(name string, flag, _ int) (File, *PipeWriter, func(), error) {
- var err error
- var f *os.File
- if flag == 0 {
- f, err = os.Create(name)
- } else {
- f, err = os.OpenFile(name, flag, 0666)
- }
+func (fs *CryptFs) Create(name string, _, _ int) (File, *PipeWriter, func(), error) {
+ f, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
if err != nil {
return nil, nil, nil, err
}
@@ -192,7 +189,18 @@ func (fs *CryptFs) Create(name string, flag, _ int) (File, *PipeWriter, func(),
p := NewPipeWriter(w)
go func() {
- n, err := sio.Encrypt(f, r, fs.getSIOConfig(key))
+ var n int64
+ var err error
+ if fs.writeBufferSize <= 0 {
+ n, err = sio.Encrypt(f, r, fs.getSIOConfig(key))
+ } else {
+ bw := bufio.NewWriterSize(f, fs.writeBufferSize)
+ n, err = fs.encryptWrapper(bw, r, fs.getSIOConfig(key))
+ errFlush := bw.Flush()
+ if err == nil && errFlush != nil {
+ err = errFlush
+ }
+ }
errClose := f.Close()
if err == nil && errClose != nil {
err = errClose
@@ -311,6 +319,26 @@ func (fs *CryptFs) getFileAndEncryptionKey(name string) (*os.File, [32]byte, err
return f, key, err
}
+func (*CryptFs) encryptWrapper(dst io.Writer, src io.Reader, config sio.Config) (int64, error) {
+ encReader, err := sio.EncryptReader(src, config)
+ if err != nil {
+ return 0, err
+ }
+ return doCopy(dst, encReader, make([]byte, 65568))
+}
+
+func (fs *CryptFs) decryptWrapper(dst io.Writer, src io.Reader, config sio.Config) (int64, error) {
+ if fs.readBufferSize <= 0 {
+ return sio.Decrypt(dst, src, config)
+ }
+ br := bufio.NewReaderSize(src, fs.readBufferSize)
+ decReader, err := sio.DecryptReader(br, config)
+ if err != nil {
+ return 0, err
+ }
+ return doCopy(dst, decReader, make([]byte, 65568))
+}
+
func isZeroBytesDownload(f *os.File, offset int64) (bool, error) {
info, err := f.Stat()
if err != nil {
diff --git a/internal/vfs/filesystem.go b/internal/vfs/filesystem.go
index 10929ef8..a23afda7 100644
--- a/internal/vfs/filesystem.go
+++ b/internal/vfs/filesystem.go
@@ -26,6 +26,7 @@ import (
type Filesystem struct {
RedactedSecret string `json:"-"`
Provider sdk.FilesystemProvider `json:"provider"`
+ OSConfig sdk.OSFsConfig `json:"osconfig,omitempty"`
S3Config S3FsConfig `json:"s3config,omitempty"`
GCSConfig GCSFsConfig `json:"gcsconfig,omitempty"`
AzBlobConfig AzBlobFsConfig `json:"azblobconfig,omitempty"`
@@ -169,6 +170,7 @@ func (f *Filesystem) Validate(additionalData string) error {
if err := f.S3Config.ValidateAndEncryptCredentials(additionalData); err != nil {
return err
}
+ f.OSConfig = sdk.OSFsConfig{}
f.GCSConfig = GCSFsConfig{}
f.AzBlobConfig = AzBlobFsConfig{}
f.CryptConfig = CryptFsConfig{}
@@ -179,6 +181,7 @@ func (f *Filesystem) Validate(additionalData string) error {
if err := f.GCSConfig.ValidateAndEncryptCredentials(additionalData); err != nil {
return err
}
+ f.OSConfig = sdk.OSFsConfig{}
f.S3Config = S3FsConfig{}
f.AzBlobConfig = AzBlobFsConfig{}
f.CryptConfig = CryptFsConfig{}
@@ -189,6 +192,7 @@ func (f *Filesystem) Validate(additionalData string) error {
if err := f.AzBlobConfig.ValidateAndEncryptCredentials(additionalData); err != nil {
return err
}
+ f.OSConfig = sdk.OSFsConfig{}
f.S3Config = S3FsConfig{}
f.GCSConfig = GCSFsConfig{}
f.CryptConfig = CryptFsConfig{}
@@ -199,16 +203,18 @@ func (f *Filesystem) Validate(additionalData string) error {
if err := f.CryptConfig.ValidateAndEncryptCredentials(additionalData); err != nil {
return err
}
+ f.OSConfig = sdk.OSFsConfig{}
f.S3Config = S3FsConfig{}
f.GCSConfig = GCSFsConfig{}
f.AzBlobConfig = AzBlobFsConfig{}
f.SFTPConfig = SFTPFsConfig{}
f.HTTPConfig = HTTPFsConfig{}
- return nil
+ return validateOSFsConfig(&f.CryptConfig.OSFsConfig)
case sdk.SFTPFilesystemProvider:
if err := f.SFTPConfig.ValidateAndEncryptCredentials(additionalData); err != nil {
return err
}
+ f.OSConfig = sdk.OSFsConfig{}
f.S3Config = S3FsConfig{}
f.GCSConfig = GCSFsConfig{}
f.AzBlobConfig = AzBlobFsConfig{}
@@ -219,6 +225,7 @@ func (f *Filesystem) Validate(additionalData string) error {
if err := f.HTTPConfig.ValidateAndEncryptCredentials(additionalData); err != nil {
return err
}
+ f.OSConfig = sdk.OSFsConfig{}
f.S3Config = S3FsConfig{}
f.GCSConfig = GCSFsConfig{}
f.AzBlobConfig = AzBlobFsConfig{}
@@ -233,7 +240,7 @@ func (f *Filesystem) Validate(additionalData string) error {
f.CryptConfig = CryptFsConfig{}
f.SFTPConfig = SFTPFsConfig{}
f.HTTPConfig = HTTPFsConfig{}
- return nil
+ return validateOSFsConfig(&f.OSConfig)
}
}
@@ -293,6 +300,10 @@ func (f *Filesystem) GetACopy() Filesystem {
f.SetEmptySecretsIfNil()
fs := Filesystem{
Provider: f.Provider,
+ OSConfig: sdk.OSFsConfig{
+ ReadBufferSize: f.OSConfig.ReadBufferSize,
+ WriteBufferSize: f.OSConfig.WriteBufferSize,
+ },
S3Config: S3FsConfig{
BaseS3FsConfig: sdk.BaseS3FsConfig{
Bucket: f.S3Config.Bucket,
@@ -342,6 +353,10 @@ func (f *Filesystem) GetACopy() Filesystem {
SASURL: f.AzBlobConfig.SASURL.Clone(),
},
CryptConfig: CryptFsConfig{
+ OSFsConfig: sdk.OSFsConfig{
+ ReadBufferSize: f.CryptConfig.ReadBufferSize,
+ WriteBufferSize: f.CryptConfig.WriteBufferSize,
+ },
Passphrase: f.CryptConfig.Passphrase.Clone(),
},
SFTPConfig: SFTPFsConfig{
diff --git a/internal/vfs/folder.go b/internal/vfs/folder.go
index 4d131a6f..e3af0c9f 100644
--- a/internal/vfs/folder.go
+++ b/internal/vfs/folder.go
@@ -207,7 +207,7 @@ func (v *VirtualFolder) GetFilesystem(connectionID string, forbiddenSelfUsers []
case sdk.HTTPFilesystemProvider:
return NewHTTPFs(connectionID, v.MappedPath, v.VirtualPath, v.FsConfig.HTTPConfig)
default:
- return NewOsFs(connectionID, v.MappedPath, v.VirtualPath), nil
+ return NewOsFs(connectionID, v.MappedPath, v.VirtualPath, &v.FsConfig.OSConfig), nil
}
}
diff --git a/internal/vfs/gcsfs.go b/internal/vfs/gcsfs.go
index 016e5561..a55f6433 100644
--- a/internal/vfs/gcsfs.go
+++ b/internal/vfs/gcsfs.go
@@ -472,7 +472,7 @@ func (*GCSFs) IsNotSupported(err error) bool {
// CheckRootPath creates the specified local root directory if it does not exists
func (fs *GCSFs) CheckRootPath(username string, uid int, gid int) bool {
// we need a local directory for temporary files
- osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, "")
+ osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, "", nil)
return osFs.CheckRootPath(username, uid, gid)
}
diff --git a/internal/vfs/httpfs.go b/internal/vfs/httpfs.go
index 6bf5a56c..94cccb4d 100644
--- a/internal/vfs/httpfs.go
+++ b/internal/vfs/httpfs.go
@@ -523,7 +523,7 @@ func (*HTTPFs) IsNotSupported(err error) bool {
// CheckRootPath creates the specified local root directory if it does not exists
func (fs *HTTPFs) CheckRootPath(username string, uid int, gid int) bool {
// we need a local directory for temporary files
- osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, "")
+ osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, "", nil)
return osFs.CheckRootPath(username, uid, gid)
}
diff --git a/internal/vfs/osfs.go b/internal/vfs/osfs.go
index 5ee7b61c..5d721cd1 100644
--- a/internal/vfs/osfs.go
+++ b/internal/vfs/osfs.go
@@ -15,6 +15,7 @@
package vfs
import (
+ "bufio"
"errors"
"fmt"
"io"
@@ -30,6 +31,7 @@ import (
fscopy "github.com/otiai10/copy"
"github.com/pkg/sftp"
"github.com/rs/xid"
+ "github.com/sftpgo/sdk"
"github.com/drakkan/sftpgo/v2/internal/logger"
"github.com/drakkan/sftpgo/v2/internal/util"
@@ -54,16 +56,33 @@ type OsFs struct {
connectionID string
rootDir string
// if not empty this fs is mouted as virtual folder in the specified path
- mountPath string
+ mountPath string
+ localTempDir string
+ readBufferSize int
+ writeBufferSize int
}
// NewOsFs returns an OsFs object that allows to interact with local Os filesystem
-func NewOsFs(connectionID, rootDir, mountPath string) Fs {
+func NewOsFs(connectionID, rootDir, mountPath string, config *sdk.OSFsConfig) Fs {
+ var tempDir string
+ if tempPath != "" {
+ tempDir = tempPath
+ } else {
+ tempDir = filepath.Clean(os.TempDir())
+ }
+ var readBufferSize, writeBufferSize int
+ if config != nil {
+ readBufferSize = config.ReadBufferSize * 1024 * 1024
+ writeBufferSize = config.WriteBufferSize * 1024 * 1024
+ }
return &OsFs{
- name: osFsName,
- connectionID: connectionID,
- rootDir: rootDir,
- mountPath: getMountPath(mountPath),
+ name: osFsName,
+ connectionID: connectionID,
+ rootDir: rootDir,
+ mountPath: getMountPath(mountPath),
+ localTempDir: tempDir,
+ readBufferSize: readBufferSize,
+ writeBufferSize: writeBufferSize,
}
}
@@ -88,7 +107,7 @@ func (fs *OsFs) Lstat(name string) (os.FileInfo, error) {
}
// Open opens the named file for reading
-func (*OsFs) Open(name string, offset int64) (File, *pipeat.PipeReaderAt, func(), error) {
+func (fs *OsFs) Open(name string, offset int64) (File, *pipeat.PipeReaderAt, func(), error) {
f, err := os.Open(name)
if err != nil {
return nil, nil, nil, err
@@ -100,19 +119,65 @@ func (*OsFs) Open(name string, offset int64) (File, *pipeat.PipeReaderAt, func()
return nil, nil, nil, err
}
}
- return f, nil, nil, err
+ if fs.readBufferSize <= 0 {
+ return f, nil, nil, err
+ }
+ r, w, err := pipeat.PipeInDir(fs.localTempDir)
+ if err != nil {
+ f.Close()
+ return nil, nil, nil, err
+ }
+ go func() {
+ br := bufio.NewReaderSize(f, fs.readBufferSize)
+ n, err := doCopy(w, br, nil)
+ w.CloseWithError(err) //nolint:errcheck
+ f.Close()
+ fsLog(fs, logger.LevelDebug, "download completed, path: %q size: %v, err: %v", name, n, err)
+ }()
+
+ return nil, r, nil, nil
}
// Create creates or opens the named file for writing
-func (*OsFs) Create(name string, flag, _ int) (File, *PipeWriter, func(), error) {
- var err error
- var f *os.File
- if flag == 0 {
- f, err = os.Create(name)
- } else {
- f, err = os.OpenFile(name, flag, 0666)
+func (fs *OsFs) Create(name string, flag, _ int) (File, *PipeWriter, func(), error) {
+ if !fs.useWriteBuffering(flag) {
+ var err error
+ var f *os.File
+ if flag == 0 {
+ f, err = os.Create(name)
+ } else {
+ f, err = os.OpenFile(name, flag, 0666)
+ }
+ return f, nil, nil, err
}
- return f, nil, nil, err
+ f, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
+ if err != nil {
+ return nil, nil, nil, err
+ }
+ r, w, err := pipeat.PipeInDir(fs.localTempDir)
+ if err != nil {
+ f.Close()
+ return nil, nil, nil, err
+ }
+ p := NewPipeWriter(w)
+
+ go func() {
+ bw := bufio.NewWriterSize(f, fs.writeBufferSize)
+ n, err := doCopy(bw, r, nil)
+ errFlush := bw.Flush()
+ if err == nil && errFlush != nil {
+ err = errFlush
+ }
+ errClose := f.Close()
+ if err == nil && errClose != nil {
+ err = errClose
+ }
+ r.CloseWithError(err) //nolint:errcheck
+ p.Done(err)
+ fsLog(fs, logger.LevelDebug, "upload completed, path: %q, readed bytes: %v, err: %v", name, n, err)
+ }()
+
+ return nil, p, nil, nil
}
// Rename renames (moves) source to target
@@ -124,10 +189,16 @@ func (fs *OsFs) Rename(source, target string) (int, int64, error) {
if err != nil && isCrossDeviceError(err) {
fsLog(fs, logger.LevelError, "cross device error detected while renaming %q -> %q. Trying a copy and remove, this could take a long time",
source, target)
+ var readBufferSize uint
+ if fs.readBufferSize > 0 {
+ readBufferSize = uint(fs.readBufferSize)
+ }
+
err = fscopy.Copy(source, target, fscopy.Options{
OnSymlink: func(src string) fscopy.SymlinkAction {
return fscopy.Skip
},
+ CopyBufferSize: readBufferSize,
})
if err != nil {
fsLog(fs, logger.LevelError, "cross device copy error: %v", err)
@@ -509,3 +580,21 @@ func (*OsFs) Close() error {
func (*OsFs) GetAvailableDiskSize(dirName string) (*sftp.StatVFS, error) {
return getStatFS(dirName)
}
+
+func (fs *OsFs) useWriteBuffering(flag int) bool {
+ if fs.writeBufferSize <= 0 {
+ return false
+ }
+ if flag == 0 {
+ return true
+ }
+ if flag&os.O_TRUNC == 0 {
+ fsLog(fs, logger.LevelDebug, "truncate flag missing, buffering write not possible")
+ return false
+ }
+ if flag&os.O_RDWR != 0 {
+ fsLog(fs, logger.LevelDebug, "read and write flag found, buffering write not possible")
+ return false
+ }
+ return true
+}
diff --git a/internal/vfs/s3fs.go b/internal/vfs/s3fs.go
index 970fc45a..fc86b389 100644
--- a/internal/vfs/s3fs.go
+++ b/internal/vfs/s3fs.go
@@ -512,7 +512,7 @@ func (*S3Fs) IsNotSupported(err error) bool {
// CheckRootPath creates the specified local root directory if it does not exists
func (fs *S3Fs) CheckRootPath(username string, uid int, gid int) bool {
// we need a local directory for temporary files
- osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, "")
+ osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, "", nil)
return osFs.CheckRootPath(username, uid, gid)
}
diff --git a/internal/vfs/sftpfs.go b/internal/vfs/sftpfs.go
index 81a0bbd5..cba19b62 100644
--- a/internal/vfs/sftpfs.go
+++ b/internal/vfs/sftpfs.go
@@ -404,7 +404,7 @@ func (fs *SFTPFs) Create(name string, flag, _ int) (File, *PipeWriter, func(), e
bw := bufio.NewWriterSize(f, int(fs.config.BufferSize)*1024*1024)
// we don't use io.Copy since bufio.Writer implements io.WriterTo and
// so it calls the sftp.File WriteTo method without buffering
- n, err := fs.copy(bw, r)
+ n, err := doCopy(bw, r, nil)
errFlush := bw.Flush()
if err == nil && errFlush != nil {
err = errFlush
@@ -573,7 +573,7 @@ func (*SFTPFs) IsNotSupported(err error) bool {
// CheckRootPath creates the specified local root directory if it does not exists
func (fs *SFTPFs) CheckRootPath(username string, uid int, gid int) bool {
// local directory for temporary files in buffer mode
- osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, "")
+ osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, "", nil)
osFs.CheckRootPath(username, uid, gid)
if fs.config.Prefix == "/" {
return true
@@ -841,38 +841,6 @@ func (fs *SFTPFs) Close() error {
return nil
}
-func (fs *SFTPFs) copy(dst io.Writer, src io.Reader) (written int64, err error) {
- buf := make([]byte, 32768)
- for {
- nr, er := src.Read(buf)
- if nr > 0 {
- nw, ew := dst.Write(buf[0:nr])
- if nw < 0 || nr < nw {
- nw = 0
- if ew == nil {
- ew = errors.New("invalid write")
- }
- }
- written += int64(nw)
- if ew != nil {
- err = ew
- break
- }
- if nr != nw {
- err = io.ErrShortWrite
- break
- }
- }
- if er != nil {
- if er != io.EOF {
- err = er
- }
- break
- }
- }
- return written, err
-}
-
func (fs *SFTPFs) createConnection() error {
err := fs.conn.OpenConnection()
if err != nil {
diff --git a/internal/vfs/vfs.go b/internal/vfs/vfs.go
index 3022a97b..f8a8ad01 100644
--- a/internal/vfs/vfs.go
+++ b/internal/vfs/vfs.go
@@ -632,6 +632,7 @@ func (c *AzBlobFsConfig) validate() error {
// CryptFsConfig defines the configuration to store local files as encrypted
type CryptFsConfig struct {
+ sdk.OSFsConfig
Passphrase *kms.Secret `json:"passphrase,omitempty"`
}
@@ -757,21 +758,24 @@ func IsHTTPFs(fs Fs) bool {
return strings.HasPrefix(fs.Name(), httpFsName)
}
-// IsBufferedSFTPFs returns true if this is a buffered SFTP filesystem
-func IsBufferedSFTPFs(fs Fs) bool {
+// IsBufferedLocalOrSFTPFs returns true if this is a buffered SFTP or local filesystem
+func IsBufferedLocalOrSFTPFs(fs Fs) bool {
+ if osFs, ok := fs.(*OsFs); ok {
+ return osFs.writeBufferSize > 0
+ }
if !IsSFTPFs(fs) {
return false
}
return !fs.IsUploadResumeSupported()
}
-// IsLocalOrUnbufferedSFTPFs returns true if fs is local or SFTP with no buffer
-func IsLocalOrUnbufferedSFTPFs(fs Fs) bool {
- if IsLocalOsFs(fs) {
- return true
+// FsOpenReturnsFile returns true if fs.Open returns a *os.File handle
+func FsOpenReturnsFile(fs Fs) bool {
+ if osFs, ok := fs.(*OsFs); ok {
+ return osFs.readBufferSize == 0
}
- if IsSFTPFs(fs) {
- return fs.IsUploadResumeSupported()
+ if sftpFs, ok := fs.(*SFTPFs); ok {
+ return sftpFs.config.BufferSize == 0
}
return false
}
@@ -929,6 +933,50 @@ func fsMetadataCheck(fs fsMetadataChecker, storageID, keyPrefix string) error {
}
}
+func validateOSFsConfig(config *sdk.OSFsConfig) error {
+ if config.ReadBufferSize < 0 || config.ReadBufferSize > 10 {
+ return fmt.Errorf("invalid read buffer size must be between 0 and 10 MB")
+ }
+ if config.WriteBufferSize < 0 || config.WriteBufferSize > 10 {
+ return fmt.Errorf("invalid write buffer size must be between 0 and 10 MB")
+ }
+ return nil
+}
+
+func doCopy(dst io.Writer, src io.Reader, buf []byte) (written int64, err error) {
+ if buf == nil {
+ buf = make([]byte, 32768)
+ }
+ for {
+ nr, er := src.Read(buf)
+ if nr > 0 {
+ nw, ew := dst.Write(buf[0:nr])
+ if nw < 0 || nr < nw {
+ nw = 0
+ if ew == nil {
+ ew = errors.New("invalid write")
+ }
+ }
+ written += int64(nw)
+ if ew != nil {
+ err = ew
+ break
+ }
+ if nr != nw {
+ err = io.ErrShortWrite
+ break
+ }
+ }
+ if er != nil {
+ if er != io.EOF {
+ err = er
+ }
+ break
+ }
+ }
+ return written, err
+}
+
func getMountPath(mountPath string) string {
if mountPath == "/" {
return ""
diff --git a/internal/webdavd/file.go b/internal/webdavd/file.go
index 7f1cca9e..e4779f2d 100644
--- a/internal/webdavd/file.go
+++ b/internal/webdavd/file.go
@@ -282,7 +282,7 @@ func (f *webDavFile) updateTransferQuotaOnSeek() {
}
func (f *webDavFile) checkFile() error {
- if f.File == nil && vfs.IsLocalOrUnbufferedSFTPFs(f.Fs) {
+ if f.File == nil && vfs.FsOpenReturnsFile(f.Fs) {
file, _, _, err := f.Fs.Open(f.GetFsPath(), 0)
if err != nil {
f.Connection.Log(logger.LevelWarn, "could not open file %q for seeking: %v",
diff --git a/internal/webdavd/internal_test.go b/internal/webdavd/internal_test.go
index 3a812d90..5992a7dc 100644
--- a/internal/webdavd/internal_test.go
+++ b/internal/webdavd/internal_test.go
@@ -327,7 +327,7 @@ func (fs *MockOsFs) GetMimeType(_ string) (string, error) {
func newMockOsFs(atomicUpload bool, connectionID, rootDir string, reader *pipeat.PipeReaderAt, err error) vfs.Fs {
return &MockOsFs{
- Fs: vfs.NewOsFs(connectionID, rootDir, ""),
+ Fs: vfs.NewOsFs(connectionID, rootDir, "", nil),
isAtomicUploadSupported: atomicUpload,
reader: reader,
err: err,
@@ -484,7 +484,7 @@ func TestResolvePathErrors(t *testing.T) {
}
user.Permissions = make(map[string][]string)
user.Permissions["/"] = []string{dataprovider.PermAny}
- fs := vfs.NewOsFs("connID", user.HomeDir, "")
+ fs := vfs.NewOsFs("connID", user.HomeDir, "", nil)
connection := &Connection{
BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolWebDAV, "", "", user),
}
@@ -517,7 +517,7 @@ func TestResolvePathErrors(t *testing.T) {
if runtime.GOOS != "windows" {
user.HomeDir = filepath.Clean(os.TempDir())
connection.User = user
- fs := vfs.NewOsFs("connID", connection.User.HomeDir, "")
+ fs := vfs.NewOsFs("connID", connection.User.HomeDir, "", nil)
subDir := "sub"
testTxtFile := "file.txt"
err = os.MkdirAll(filepath.Join(os.TempDir(), subDir, subDir), os.ModePerm)
@@ -555,7 +555,7 @@ func TestFileAccessErrors(t *testing.T) {
}
user.Permissions = make(map[string][]string)
user.Permissions["/"] = []string{dataprovider.PermAny}
- fs := vfs.NewOsFs("connID", user.HomeDir, "")
+ fs := vfs.NewOsFs("connID", user.HomeDir, "", nil)
connection := &Connection{
BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolWebDAV, "", "", user),
}
@@ -648,7 +648,7 @@ func TestCheckRequestMethodWithPrefix(t *testing.T) {
},
},
}
- fs := vfs.NewOsFs("connID", user.HomeDir, "")
+ fs := vfs.NewOsFs("connID", user.HomeDir, "", nil)
connection := &Connection{
BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolWebDAV, "", "", user),
}
@@ -672,7 +672,7 @@ func TestContentType(t *testing.T) {
}
user.Permissions = make(map[string][]string)
user.Permissions["/"] = []string{dataprovider.PermAny}
- fs := vfs.NewOsFs("connID", user.HomeDir, "")
+ fs := vfs.NewOsFs("connID", user.HomeDir, "", nil)
connection := &Connection{
BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolWebDAV, "", "", user),
}
@@ -699,7 +699,7 @@ func TestContentType(t *testing.T) {
baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile+".unknown1",
common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{})
davFile = newWebDavFile(baseTransfer, nil, nil)
- davFile.Fs = vfs.NewOsFs("id", user.HomeDir, "")
+ davFile.Fs = vfs.NewOsFs("id", user.HomeDir, "", nil)
fi, err = davFile.Stat()
if assert.NoError(t, err) {
ctype, err := fi.(*webDavFileInfo).ContentType(ctx)
@@ -712,7 +712,7 @@ func TestContentType(t *testing.T) {
baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile,
common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{})
davFile = newWebDavFile(baseTransfer, nil, nil)
- davFile.Fs = vfs.NewOsFs("id", user.HomeDir, "")
+ davFile.Fs = vfs.NewOsFs("id", user.HomeDir, "", nil)
fi, err = davFile.Stat()
if assert.NoError(t, err) {
ctype, err := fi.(*webDavFileInfo).ContentType(ctx)
@@ -727,7 +727,7 @@ func TestContentType(t *testing.T) {
baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile+".custom",
common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{})
davFile = newWebDavFile(baseTransfer, nil, nil)
- davFile.Fs = vfs.NewOsFs("id", user.HomeDir, "")
+ davFile.Fs = vfs.NewOsFs("id", user.HomeDir, "", nil)
fi, err = davFile.Stat()
if assert.NoError(t, err) {
ctype, err := fi.(*webDavFileInfo).ContentType(ctx)
@@ -781,7 +781,7 @@ func TestTransferReadWriteErrors(t *testing.T) {
}
user.Permissions = make(map[string][]string)
user.Permissions["/"] = []string{dataprovider.PermAny}
- fs := vfs.NewOsFs("connID", user.HomeDir, "")
+ fs := vfs.NewOsFs("connID", user.HomeDir, "", nil)
connection := &Connection{
BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolWebDAV, "", "", user),
}
@@ -937,7 +937,7 @@ func TestTransferSeek(t *testing.T) {
assert.True(t, fs.IsNotExist(err))
davFile.Connection.RemoveTransfer(davFile.BaseTransfer)
- fs = vfs.NewOsFs(fs.ConnectionID(), user.GetHomeDir(), "")
+ fs = vfs.NewOsFs(fs.ConnectionID(), user.GetHomeDir(), "", nil)
baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath+"1", testFilePath+"1", testFile,
common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{AllowedTotalSize: 100})
davFile = newWebDavFile(baseTransfer, nil, nil)
diff --git a/internal/webdavd/webdavd_test.go b/internal/webdavd/webdavd_test.go
index 01109a3b..699f5de3 100644
--- a/internal/webdavd/webdavd_test.go
+++ b/internal/webdavd/webdavd_test.go
@@ -52,6 +52,7 @@ import (
"github.com/drakkan/sftpgo/v2/internal/kms"
"github.com/drakkan/sftpgo/v2/internal/logger"
"github.com/drakkan/sftpgo/v2/internal/sftpd"
+ "github.com/drakkan/sftpgo/v2/internal/util"
"github.com/drakkan/sftpgo/v2/internal/vfs"
"github.com/drakkan/sftpgo/v2/internal/webdavd"
)
@@ -720,6 +721,70 @@ func TestBasicHandlingCryptFs(t *testing.T) {
1*time.Second, 100*time.Millisecond)
}
+func TestBufferedUser(t *testing.T) {
+ u := getTestUser()
+ u.FsConfig.OSConfig = sdk.OSFsConfig{
+ WriteBufferSize: 2,
+ ReadBufferSize: 1,
+ }
+ vdirPath := "/crypted"
+ mappedPath := filepath.Join(os.TempDir(), util.GenerateUniqueID())
+ folderName := filepath.Base(mappedPath)
+ u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{
+ BaseVirtualFolder: vfs.BaseVirtualFolder{
+ Name: folderName,
+ MappedPath: mappedPath,
+ FsConfig: vfs.Filesystem{
+ Provider: sdk.CryptedFilesystemProvider,
+ CryptConfig: vfs.CryptFsConfig{
+ OSFsConfig: sdk.OSFsConfig{
+ WriteBufferSize: 3,
+ ReadBufferSize: 2,
+ },
+ Passphrase: kms.NewPlainSecret(defaultPassword),
+ },
+ },
+ },
+ VirtualPath: vdirPath,
+ QuotaFiles: -1,
+ QuotaSize: -1,
+ })
+ user, _, err := httpdtest.AddUser(u, http.StatusCreated)
+ assert.NoError(t, err)
+
+ client := getWebDavClient(user, false, nil)
+ assert.NoError(t, checkBasicFunc(client))
+
+ testFilePath := filepath.Join(homeBasePath, testFileName)
+ testFileSize := int64(65535)
+ err = createTestFile(testFilePath, testFileSize)
+ assert.NoError(t, err)
+ err = uploadFileWithRawClient(testFilePath, testFileName,
+ user.Username, defaultPassword, false, testFileSize, client)
+ assert.NoError(t, err)
+ err = uploadFileWithRawClient(testFilePath, path.Join(vdirPath, testFileName),
+ user.Username, defaultPassword, false, testFileSize, client)
+ assert.NoError(t, err)
+ localDownloadPath := filepath.Join(homeBasePath, testDLFileName)
+ err = downloadFile(testFileName, localDownloadPath, testFileSize, client)
+ assert.NoError(t, err)
+ err = downloadFile(path.Join(vdirPath, testFileName), localDownloadPath, testFileSize, client)
+ assert.NoError(t, err)
+
+ err = os.Remove(testFilePath)
+ assert.NoError(t, err)
+ err = os.Remove(localDownloadPath)
+ assert.NoError(t, err)
+ _, err = httpdtest.RemoveUser(user, http.StatusOK)
+ assert.NoError(t, err)
+ err = os.RemoveAll(user.GetHomeDir())
+ assert.NoError(t, err)
+ _, err = httpdtest.RemoveFolder(vfs.BaseVirtualFolder{Name: folderName}, http.StatusOK)
+ assert.NoError(t, err)
+ err = os.RemoveAll(mappedPath)
+ assert.NoError(t, err)
+}
+
func TestLoginEmptyPassword(t *testing.T) {
u := getTestUser()
u.Password = ""
@@ -1497,7 +1562,7 @@ func TestMaxConnections(t *testing.T) {
client := getWebDavClient(user, true, nil)
assert.NoError(t, checkBasicFunc(client))
// now add a fake connection
- fs := vfs.NewOsFs("id", os.TempDir(), "")
+ fs := vfs.NewOsFs("id", os.TempDir(), "", nil)
connection := &webdavd.Connection{
BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolWebDAV, "", "", user),
}
@@ -1576,7 +1641,7 @@ func TestMaxSessions(t *testing.T) {
client := getWebDavClient(user, false, nil)
assert.NoError(t, checkBasicFunc(client))
// now add a fake connection
- fs := vfs.NewOsFs("id", os.TempDir(), "")
+ fs := vfs.NewOsFs("id", os.TempDir(), "", nil)
connection := &webdavd.Connection{
BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolWebDAV, "", "", user),
}
diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml
index 70c5b6f5..f98ad7e6 100644
--- a/openapi/openapi.yaml
+++ b/openapi/openapi.yaml
@@ -5729,11 +5729,34 @@ components:
use_emulator:
type: boolean
description: Azure Blob Storage configuration details
+ OSFsConfig:
+ type: object
+ properties:
+ read_buffer_size:
+ type: integer
+ minimum: 0
+ maximum: 10
+ description: 'The read buffer size, as MB, to use for downloads. 0 means no buffering, that's fine in most use cases.'
+ write_buffer_size:
+ type: integer
+ minimum: 0
+ maximum: 10
+ description: 'The write buffer size, as MB, to use for uploads. 0 means no buffering, that's fine in most use cases.'
CryptFsConfig:
type: object
properties:
passphrase:
$ref: '#/components/schemas/Secret'
+ read_buffer_size:
+ type: integer
+ minimum: 0
+ maximum: 10
+ description: 'The read buffer size, as MB, to use for downloads. 0 means no buffering, that's fine in most use cases.'
+ write_buffer_size:
+ type: integer
+ minimum: 0
+ maximum: 10
+ description: 'The write buffer size, as MB, to use for uploads. 0 means no buffering, that's fine in most use cases.'
description: Crypt filesystem configuration details
SFTPFsConfig:
type: object
@@ -5804,6 +5827,8 @@ components:
properties:
provider:
$ref: '#/components/schemas/FsProviders'
+ osconfig:
+ $ref: '#/components/schemas/OSFsConfig'
s3config:
$ref: '#/components/schemas/S3Config'
gcsconfig:
diff --git a/templates/webadmin/fsconfig.html b/templates/webadmin/fsconfig.html
index cb2a6cdd..724f176e 100644
--- a/templates/webadmin/fsconfig.html
+++ b/templates/webadmin/fsconfig.html
@@ -53,6 +53,25 @@ along with this program. If not, see .
{{end}}
+
+
+