add hide policy to pattern filters

Disallowed files/dirs can be completly hidden. This may cause performance
issues for large directories

Signed-off-by: Nicola Murino <nicola.murino@gmail.com>

common/connection.go

@@ -242,7 +242,7 @@ func (c *BaseConnection) ListDir(virtualPath string) ([]os.FileInfo, error) {
 		c.Log(logger.LevelDebug, "error listing directory: %+v", err)
 		return nil, c.GetFsError(fs, err)
 	}
-	return c.User.AddVirtualDirs(files, virtualPath), nil
+	return c.User.FilterListDir(files, virtualPath), nil
 }
 
 // CheckParentDirs tries to create the specified directory and any missing parent dirs
@@ -254,7 +254,7 @@ func (c *BaseConnection) CheckParentDirs(virtualPath string) error {
 	if fs.HasVirtualFolders() {
 		return nil
 	}
-	if _, err := c.DoStat(virtualPath, 0); !c.IsNotExistError(err) {
+	if _, err := c.DoStat(virtualPath, 0, false); !c.IsNotExistError(err) {
 		return err
 	}
 	dirs := util.GetDirsForVirtualPath(virtualPath)
@@ -275,10 +275,15 @@ func (c *BaseConnection) CheckParentDirs(virtualPath string) error {
 }
 
 // CreateDir creates a new directory at the specified fsPath
-func (c *BaseConnection) CreateDir(virtualPath string) error {
+func (c *BaseConnection) CreateDir(virtualPath string, checkFilePatterns bool) error {
 	if !c.User.HasPerm(dataprovider.PermCreateDirs, path.Dir(virtualPath)) {
 		return c.GetPermissionDeniedError()
 	}
+	if checkFilePatterns {
+		if ok, _ := c.User.IsFileAllowed(virtualPath); !ok {
+			return c.GetPermissionDeniedError()
+		}
+	}
 	if c.User.IsVirtualFolder(virtualPath) {
 		c.Log(logger.LevelWarn, "mkdir not allowed %#v is a virtual folder", virtualPath)
 		return c.GetPermissionDeniedError()
@@ -304,9 +309,9 @@ func (c *BaseConnection) IsRemoveFileAllowed(virtualPath string) error {
 	if !c.User.HasAnyPerm([]string{dataprovider.PermDeleteFiles, dataprovider.PermDelete}, path.Dir(virtualPath)) {
 		return c.GetPermissionDeniedError()
 	}
-	if !c.User.IsFileAllowed(virtualPath) {
+	if ok, policy := c.User.IsFileAllowed(virtualPath); !ok {
 		c.Log(logger.LevelDebug, "removing file %#v is not allowed", virtualPath)
-		return c.GetPermissionDeniedError()
+		return c.GetErrorForDeniedFile(policy)
 	}
 	return nil
 }
@@ -368,6 +373,10 @@ func (c *BaseConnection) IsRemoveDirAllowed(fs vfs.Fs, fsPath, virtualPath strin
 	if !c.User.HasAnyPerm([]string{dataprovider.PermDeleteDirs, dataprovider.PermDelete}, path.Dir(virtualPath)) {
 		return c.GetPermissionDeniedError()
 	}
+	if ok, policy := c.User.IsFileAllowed(virtualPath); !ok {
+		c.Log(logger.LevelDebug, "removing directory %#v is not allowed", virtualPath)
+		return c.GetErrorForDeniedFile(policy)
+	}
 	return nil
 }
 
@@ -488,16 +497,25 @@ func (c *BaseConnection) CreateSymlink(virtualSourcePath, virtualTargetPath stri
 		return c.GetFsError(fs, err)
 	}
 	if fs.GetRelativePath(fsSourcePath) == "/" {
-		c.Log(logger.LevelWarn, "symlinking root dir is not allowed")
+		c.Log(logger.LevelError, "symlinking root dir is not allowed")
 		return c.GetPermissionDeniedError()
 	}
 	if fs.GetRelativePath(fsTargetPath) == "/" {
-		c.Log(logger.LevelWarn, "symlinking to root dir is not allowed")
+		c.Log(logger.LevelError, "symlinking to root dir is not allowed")
 		return c.GetPermissionDeniedError()
 	}
 	if !c.User.HasPerm(dataprovider.PermCreateSymlinks, path.Dir(virtualTargetPath)) {
 		return c.GetPermissionDeniedError()
 	}
+	ok, policy := c.User.IsFileAllowed(virtualSourcePath)
+	if !ok && policy == sdk.DenyPolicyHide {
+		c.Log(logger.LevelError, "symlink source path %#v is not allowed", virtualSourcePath)
+		return c.GetNotExistError()
+	}
+	if ok, _ = c.User.IsFileAllowed(virtualTargetPath); !ok {
+		c.Log(logger.LevelError, "symlink target path %#v is not allowed", virtualTargetPath)
+		return c.GetPermissionDeniedError()
+	}
 	if err := fs.Symlink(fsSourcePath, fsTargetPath); err != nil {
 		c.Log(logger.LevelError, "failed to create symlink %#v -> %#v: %+v", fsSourcePath, fsTargetPath, err)
 		return c.GetFsError(fs, err)
@@ -518,13 +536,19 @@ func (c *BaseConnection) getPathForSetStatPerms(fs vfs.Fs, fsPath, virtualPath s
 }
 
 // DoStat execute a Stat if mode = 0, Lstat if mode = 1
-func (c *BaseConnection) DoStat(virtualPath string, mode int) (os.FileInfo, error) {
+func (c *BaseConnection) DoStat(virtualPath string, mode int, checkFilePatterns bool) (os.FileInfo, error) {
 	// for some vfs we don't create intermediary folders so we cannot simply check
 	// if virtualPath is a virtual folder
 	vfolders := c.User.GetVirtualFoldersInPath(path.Dir(virtualPath))
 	if _, ok := vfolders[virtualPath]; ok {
 		return vfs.NewFileInfo(virtualPath, true, 0, time.Now(), false), nil
 	}
+	if checkFilePatterns {
+		ok, policy := c.User.IsFileAllowed(virtualPath)
+		if !ok && policy == sdk.DenyPolicyHide {
+			return nil, c.GetNotExistError()
+		}
+	}
 
 	var info os.FileInfo
 
@@ -549,9 +573,9 @@ func (c *BaseConnection) DoStat(virtualPath string, mode int) (os.FileInfo, erro
 }
 
 func (c *BaseConnection) createDirIfMissing(name string) error {
-	_, err := c.DoStat(name, 0)
+	_, err := c.DoStat(name, 0, false)
 	if c.IsNotExistError(err) {
-		return c.CreateDir(name)
+		return c.CreateDir(name, false)
 	}
 	return err
 }
@@ -625,6 +649,9 @@ func (c *BaseConnection) handleChtimes(fs vfs.Fs, fsPath, pathForPerms string, a
 
 // SetStat set StatAttributes for the specified fsPath
 func (c *BaseConnection) SetStat(virtualPath string, attributes *StatAttributes) error {
+	if ok, policy := c.User.IsFileAllowed(virtualPath); !ok {
+		return c.GetErrorForDeniedFile(policy)
+	}
 	fs, fsPath, err := c.GetFsAndResolvedPath(virtualPath)
 	if err != nil {
 		return err
@@ -799,12 +826,12 @@ func (c *BaseConnection) isRenamePermitted(fsSrc, fsDst vfs.Fs, fsSourcePath, fs
 		c.Log(logger.LevelWarn, "renaming a virtual folder is not allowed")
 		return false
 	}
-	if !c.User.IsFileAllowed(virtualSourcePath) || !c.User.IsFileAllowed(virtualTargetPath) {
+	isSrcAllowed, _ := c.User.IsFileAllowed(virtualSourcePath)
-		if fi != nil && fi.Mode().IsRegular() {
+	isDstAllowed, _ := c.User.IsFileAllowed(virtualTargetPath)
-			c.Log(logger.LevelDebug, "renaming file is not allowed, source: %#v target: %#v",
+	if !isSrcAllowed || !isDstAllowed {
-				virtualSourcePath, virtualTargetPath)
+		c.Log(logger.LevelDebug, "renaming source: %#v to target: %#v not allowed", virtualSourcePath,
-			return false
+			virtualTargetPath)
-		}
+		return false
 	}
 	return c.hasRenamePerms(virtualSourcePath, virtualTargetPath, fi)
 }
@@ -1141,6 +1168,16 @@ func (c *BaseConnection) IsNotExistError(err error) bool {
 	}
 }
 
+// GetErrorForDeniedFile return permission denied or not exist error based on the specified policy
+func (c *BaseConnection) GetErrorForDeniedFile(policy int) error {
+	switch policy {
+	case sdk.DenyPolicyHide:
+		return c.GetNotExistError()
+	default:
+		return c.GetPermissionDeniedError()
+	}
+}
+
 // GetPermissionDeniedError returns an appropriate permission denied error for the connection protocol
 func (c *BaseConnection) GetPermissionDeniedError() error {
 	switch c.protocol {

common/protocol_test.go

@@ -482,6 +482,100 @@ func TestPermissionErrors(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestHiddenPatternFilter(t *testing.T) {
+	deniedDir := "/denied_hidden"
+	u := getTestUser()
+	u.Filters.FilePatterns = []sdk.PatternsFilter{
+		{
+			Path:           deniedDir,
+			DeniedPatterns: []string{"*.txt", "beta*"},
+			DenyPolicy:     sdk.DenyPolicyHide,
+		},
+	}
+	user, _, err := httpdtest.AddUser(u, http.StatusCreated)
+	assert.NoError(t, err)
+
+	dirName := "beta"
+	testFile := filepath.Join(u.GetHomeDir(), deniedDir, "file.txt")
+	testFile1 := filepath.Join(u.GetHomeDir(), deniedDir, "beta.txt")
+	err = os.MkdirAll(filepath.Join(u.GetHomeDir(), deniedDir), os.ModePerm)
+	assert.NoError(t, err)
+	err = os.WriteFile(testFile, testFileContent, os.ModePerm)
+	assert.NoError(t, err)
+	err = os.WriteFile(testFile1, testFileContent, os.ModePerm)
+	assert.NoError(t, err)
+	err = os.MkdirAll(filepath.Join(u.GetHomeDir(), deniedDir, dirName), os.ModePerm)
+	assert.NoError(t, err)
+
+	conn, client, err := getSftpClient(user)
+	if assert.NoError(t, err) {
+		defer conn.Close()
+		defer client.Close()
+
+		files, err := client.ReadDir(deniedDir)
+		assert.NoError(t, err)
+		assert.Len(t, files, 0)
+		err = client.Remove(path.Join(deniedDir, filepath.Base(testFile)))
+		assert.ErrorIs(t, err, os.ErrNotExist)
+		err = client.Chtimes(path.Join(deniedDir, filepath.Base(testFile)), time.Now(), time.Now())
+		assert.ErrorIs(t, err, os.ErrNotExist)
+		_, err = client.Stat(path.Join(deniedDir, filepath.Base(testFile1)))
+		assert.ErrorIs(t, err, os.ErrNotExist)
+		err = client.RemoveDirectory(path.Join(deniedDir, dirName))
+		assert.ErrorIs(t, err, os.ErrNotExist)
+		err = client.Rename(path.Join(deniedDir, dirName), path.Join(deniedDir, "newname"))
+		assert.ErrorIs(t, err, os.ErrPermission)
+		err = client.Mkdir(path.Join(deniedDir, "beta1"))
+		assert.ErrorIs(t, err, os.ErrPermission)
+		err = writeSFTPFile(path.Join(deniedDir, "afile.txt"), 1024, client)
+		assert.ErrorIs(t, err, os.ErrPermission)
+		err = client.Symlink(path.Join(deniedDir, dirName), dirName)
+		assert.ErrorIs(t, err, os.ErrNotExist)
+		err = writeSFTPFile(path.Join(deniedDir, testFileName), 1024, client)
+		assert.NoError(t, err)
+		err = client.Symlink(path.Join(deniedDir, testFileName), path.Join(deniedDir, "symlink.txt"))
+		assert.ErrorIs(t, err, os.ErrPermission)
+		files, err = client.ReadDir(deniedDir)
+		assert.NoError(t, err)
+		assert.Len(t, files, 1)
+	}
+	_, err = httpdtest.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	u.Filters.FilePatterns = []sdk.PatternsFilter{
+		{
+			Path:           deniedDir,
+			DeniedPatterns: []string{"*.txt", "beta*"},
+			DenyPolicy:     sdk.DenyPolicyDefault,
+		},
+	}
+	user, _, err = httpdtest.AddUser(u, http.StatusCreated)
+	assert.NoError(t, err)
+	conn, client, err = getSftpClient(user)
+	if assert.NoError(t, err) {
+		defer conn.Close()
+		defer client.Close()
+
+		files, err := client.ReadDir(deniedDir)
+		assert.NoError(t, err)
+		assert.Len(t, files, 4)
+		_, err = client.Stat(path.Join(deniedDir, filepath.Base(testFile)))
+		assert.NoError(t, err)
+		err = client.Chtimes(path.Join(deniedDir, filepath.Base(testFile)), time.Now(), time.Now())
+		assert.ErrorIs(t, err, os.ErrPermission)
+		err = client.Mkdir(path.Join(deniedDir, "beta2"))
+		assert.ErrorIs(t, err, os.ErrPermission)
+		err = writeSFTPFile(path.Join(deniedDir, "afile2.txt"), 1024, client)
+		assert.ErrorIs(t, err, os.ErrPermission)
+		err = client.Symlink(path.Join(deniedDir, testFileName), path.Join(deniedDir, "link.txt"))
+		assert.ErrorIs(t, err, os.ErrPermission)
+	}
+
+	_, err = httpdtest.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+}
+
 func TestFileNotAllowedErrors(t *testing.T) {
 	deniedDir := "/denied"
 	u := getTestUser()
@@ -503,9 +597,7 @@ func TestFileNotAllowedErrors(t *testing.T) {
 		err = os.WriteFile(testFile, testFileContent, os.ModePerm)
 		assert.NoError(t, err)
 		err = client.Remove(path.Join(deniedDir, "file.txt"))
-		// the sftp client will try to remove the path as directory after receiving
+		assert.ErrorIs(t, err, os.ErrPermission)
-		// a permission denied error, so we get a generic failure here
-		assert.Error(t, err)
 		err = client.Rename(path.Join(deniedDir, "file.txt"), path.Join(deniedDir, "file1.txt"))
 		assert.ErrorIs(t, err, os.ErrPermission)
 	}
@@ -1946,8 +2038,9 @@ func TestDirs(t *testing.T) {
 		defer client.Close()
 		info, err := client.ReadDir("/")
 		if assert.NoError(t, err) {
-			assert.Len(t, info, 1)
+			if assert.Len(t, info, 1) {
-			assert.Equal(t, "path", info[0].Name())
+				assert.Equal(t, "path", info[0].Name())
+			}
 		}
 		fi, err := client.Stat(path.Dir(vdirPath))
 		if assert.NoError(t, err) {
@@ -2061,7 +2154,7 @@ func TestResolvePathError(t *testing.T) {
 	testPath := "apath"
 	_, err := conn.ListDir(testPath)
 	assert.Error(t, err)
-	err = conn.CreateDir(testPath)
+	err = conn.CreateDir(testPath, true)
 	assert.Error(t, err)
 	err = conn.RemoveDir(testPath)
 	assert.Error(t, err)
@@ -2069,7 +2162,7 @@ func TestResolvePathError(t *testing.T) {
 	assert.Error(t, err)
 	err = conn.CreateSymlink(testPath, testPath+".sym")
 	assert.Error(t, err)
-	_, err = conn.DoStat(testPath, 0)
+	_, err = conn.DoStat(testPath, 0, false)
 	assert.Error(t, err)
 	err = conn.SetStat(testPath, &common.StatAttributes{
 		Atime: time.Now(),

dataprovider/dataprovider.go

@@ -1691,6 +1691,9 @@ func validateFiltersPatternExtensions(user *User) error {
 		if len(f.AllowedPatterns) == 0 && len(f.DeniedPatterns) == 0 {
 			return util.NewValidationError(fmt.Sprintf("empty file patterns filter for path %#v", f.Path))
 		}
+		if f.DenyPolicy < sdk.DenyPolicyDefault || f.DenyPolicy > sdk.DenyPolicyHide {
+			return util.NewValidationError(fmt.Sprintf("invalid deny policy %v for path %#v", f.DenyPolicy, f.Path))
+		}
 		f.Path = cleanedPath
 		allowed := make([]string, 0, len(f.AllowedPatterns))
 		denied := make([]string, 0, len(f.DeniedPatterns))
@@ -1708,8 +1711,8 @@ func validateFiltersPatternExtensions(user *User) error {
 			}
 			denied = append(denied, strings.ToLower(pattern))
 		}
-		f.AllowedPatterns = allowed
+		f.AllowedPatterns = util.RemoveDuplicates(allowed)
-		f.DeniedPatterns = denied
+		f.DeniedPatterns = util.RemoveDuplicates(denied)
 		filters = append(filters, f)
 		filteredPaths = append(filteredPaths, cleanedPath)
 	}

dataprovider/user.go

@@ -549,36 +549,55 @@ func (u *User) GetVirtualFoldersInPath(virtualPath string) map[string]bool {
 	return result
 }
 
-// AddVirtualDirs adds virtual folders, if defined, to the given files list
+// FilterListDir adds virtual folders and remove hidden items from the given files list
-func (u *User) AddVirtualDirs(list []os.FileInfo, virtualPath string) []os.FileInfo {
+func (u *User) FilterListDir(dirContents []os.FileInfo, virtualPath string) []os.FileInfo {
-	if len(u.VirtualFolders) == 0 {
+	filter := u.getPatternsFilterForPath(virtualPath)
-		return list
+	if len(u.VirtualFolders) == 0 && filter.DenyPolicy != sdk.DenyPolicyHide {
+		return dirContents
 	}
 
 	vdirs := make(map[string]bool)
 	for dir := range u.GetVirtualFoldersInPath(virtualPath) {
-		vdirs[path.Base(dir)] = true
+		dirName := path.Base(dir)
-	}
+		if filter.DenyPolicy == sdk.DenyPolicyHide {
-	if len(vdirs) == 0 {
+			if !filter.CheckAllowed(dirName) {
-		return list
+				continue
+			}
+		}
+		vdirs[dirName] = true
 	}
 
-	for index := range list {
+	validIdx := 0
+	for index, fi := range dirContents {
 		for dir := range vdirs {
-			if list[index].Name() == dir {
+			if fi.Name() == dir {
-				if !list[index].IsDir() {
+				if !fi.IsDir() {
-					list[index] = vfs.NewFileInfo(dir, true, 0, time.Now(), false)
+					fi = vfs.NewFileInfo(dir, true, 0, time.Now(), false)
+					dirContents[index] = fi
 				}
 				delete(vdirs, dir)
 			}
 		}
+		if filter.DenyPolicy == sdk.DenyPolicyHide {
+			if filter.CheckAllowed(fi.Name()) {
+				dirContents[validIdx] = fi
+				validIdx++
+			}
+		}
+	}
+
+	if filter.DenyPolicy == sdk.DenyPolicyHide {
+		for idx := validIdx; idx < len(dirContents); idx++ {
+			dirContents[idx] = nil
+		}
+		dirContents = dirContents[:validIdx]
 	}
 
 	for dir := range vdirs {
 		fi := vfs.NewFileInfo(dir, true, 0, time.Now(), false)
-		list = append(list, fi)
+		dirContents = append(dirContents, fi)
 	}
-	return list
+	return dirContents
 }
 
 // IsMappedPath returns true if the specified filesystem path has a virtual folder mapping.
@@ -801,29 +820,26 @@ func (u *User) GetFlatFilePatterns() []sdk.PatternsFilter {
 			result = append(result, sdk.PatternsFilter{
 				Path:            pattern.Path,
 				AllowedPatterns: pattern.AllowedPatterns,
+				DenyPolicy:      pattern.DenyPolicy,
 			})
 		}
 		if len(pattern.DeniedPatterns) > 0 {
 			result = append(result, sdk.PatternsFilter{
 				Path:           pattern.Path,
 				DeniedPatterns: pattern.DeniedPatterns,
+				DenyPolicy:     pattern.DenyPolicy,
 			})
 		}
 	}
 	return result
 }
 
-// IsFileAllowed returns true if the specified file is allowed by the file restrictions filters
+func (u *User) getPatternsFilterForPath(virtualPath string) sdk.PatternsFilter {
-func (u *User) IsFileAllowed(virtualPath string) bool {
-	return u.isFilePatternAllowed(virtualPath)
-}
-
-func (u *User) isFilePatternAllowed(virtualPath string) bool {
-	if len(u.Filters.FilePatterns) == 0 {
-		return true
-	}
-	dirsForPath := util.GetDirsForVirtualPath(path.Dir(virtualPath))
 	var filter sdk.PatternsFilter
+	if len(u.Filters.FilePatterns) == 0 {
+		return filter
+	}
+	dirsForPath := util.GetDirsForVirtualPath(virtualPath)
 	for _, dir := range dirsForPath {
 		for _, f := range u.Filters.FilePatterns {
 			if f.Path == dir {
@@ -835,23 +851,14 @@ func (u *User) isFilePatternAllowed(virtualPath string) bool {
 			break
 		}
 	}
-	if filter.Path != "" {
+	return filter
-		toMatch := strings.ToLower(path.Base(virtualPath))
+}
-		for _, denied := range filter.DeniedPatterns {
+
-			matched, err := path.Match(denied, toMatch)
+// IsFileAllowed returns true if the specified file is allowed by the file restrictions filters.
-			if err != nil || matched {
+// The second parameter returned is the deny policy
-				return false
+func (u *User) IsFileAllowed(virtualPath string) (bool, int) {
-			}
+	filter := u.getPatternsFilterForPath(path.Dir(virtualPath))
-		}
+	return filter.CheckAllowed(path.Base(virtualPath)), filter.DenyPolicy
-		for _, allowed := range filter.AllowedPatterns {
-			matched, err := path.Match(allowed, toMatch)
-			if err == nil && matched {
-				return true
-			}
-		}
-		return len(filter.AllowedPatterns) == 0
-	}
-	return true
 }
 
 // CanManageMFA returns true if the user can add a multi-factor authentication configuration

ftpd/handler.go

@@ -83,7 +83,7 @@ func (c *Connection) Create(name string) (afero.File, error) {
 func (c *Connection) Mkdir(name string, perm os.FileMode) error {
 	c.UpdateLastActivity()
 
-	return c.CreateDir(name)
+	return c.CreateDir(name, true)
 }
 
 // MkdirAll is not implemented, we don't need it
@@ -145,7 +145,7 @@ func (c *Connection) Stat(name string) (os.FileInfo, error) {
 		return nil, c.GetPermissionDeniedError()
 	}
 
-	fi, err := c.DoStat(name, 0)
+	fi, err := c.DoStat(name, 0, true)
 	if err != nil {
 		return nil, err
 	}
@@ -319,9 +319,9 @@ func (c *Connection) downloadFile(fs vfs.Fs, fsPath, ftpPath string, offset int6
 		return nil, c.GetPermissionDeniedError()
 	}
 
-	if !c.User.IsFileAllowed(ftpPath) {
+	if ok, policy := c.User.IsFileAllowed(ftpPath); !ok {
 		c.Log(logger.LevelWarn, "reading file %#v is not allowed", ftpPath)
-		return nil, c.GetPermissionDeniedError()
+		return nil, c.GetErrorForDeniedFile(policy)
 	}
 
 	if err := common.ExecutePreAction(c.BaseConnection, common.OperationPreDownload, fsPath, ftpPath, 0, 0); err != nil {
@@ -344,7 +344,7 @@ func (c *Connection) downloadFile(fs vfs.Fs, fsPath, ftpPath string, offset int6
 }
 
 func (c *Connection) uploadFile(fs vfs.Fs, fsPath, ftpPath string, flags int) (ftpserver.FileTransfer, error) {
-	if !c.User.IsFileAllowed(ftpPath) {
+	if ok, _ := c.User.IsFileAllowed(ftpPath); !ok {
 		c.Log(logger.LevelWarn, "writing file %#v is not allowed", ftpPath)
 		return nil, ftpserver.ErrFileNameNotAllowed
 	}

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/Azure/azure-storage-blob-go v0.14.0
 	github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962
 	github.com/alexedwards/argon2id v0.0.0-20211130144151-3585854a6387
-	github.com/aws/aws-sdk-go v1.42.33
+	github.com/aws/aws-sdk-go v1.42.35
 	github.com/cockroachdb/cockroach-go/v2 v2.2.5
 	github.com/eikenb/pipeat v0.0.0-20210603033007-44fc3ffce52b
 	github.com/fclairamb/ftpserverlib v0.17.0
@@ -39,7 +39,7 @@ require (
 	github.com/rs/cors v1.8.2
 	github.com/rs/xid v1.3.0
 	github.com/rs/zerolog v1.26.2-0.20211219225053-665519c4da50
-	github.com/sftpgo/sdk v0.0.0-20220110174344-ecf586dd8941
+	github.com/sftpgo/sdk v0.0.0-20220115154521-b31d253a0bea
 	github.com/shirou/gopsutil/v3 v3.21.13-0.20220106132423-a3ae4bc40d26
 	github.com/spf13/afero v1.8.0
 	github.com/spf13/cobra v1.3.0
@@ -54,7 +54,7 @@ require (
 	gocloud.dev v0.24.0
 	golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3
 	golang.org/x/net v0.0.0-20220111093109-d55c255bac03
-	golang.org/x/sys v0.0.0-20220111092808-5a964db01320
+	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9
 	golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11
 	google.golang.org/api v0.65.0
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
@@ -76,7 +76,7 @@ require (
 	github.com/fatih/color v1.13.0 // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
-	github.com/goccy/go-json v0.9.1 // indirect
+	github.com/goccy/go-json v0.9.3 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/go-cmp v0.5.6 // indirect
@@ -126,7 +126,7 @@ require (
 	golang.org/x/tools v0.1.8 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220112215332-a9c7c0acf9f2 // indirect
+	google.golang.org/genproto v0.0.0-20220114231437-d2e6a121cae0 // indirect
 	google.golang.org/grpc v1.43.0 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/ini.v1 v1.66.2 // indirect

go.sum

@@ -141,8 +141,8 @@ github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgI
 github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/aws/aws-sdk-go v1.40.34/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
-github.com/aws/aws-sdk-go v1.42.33 h1:YlwikF3suaqs6XXwCQAnQ1xDXv0olmYRqD4W+lXcfF8=
+github.com/aws/aws-sdk-go v1.42.35 h1:N4N9buNs4YlosI9N0+WYrq8cIZwdgv34yRbxzZlTvFs=
-github.com/aws/aws-sdk-go v1.42.33/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc=
+github.com/aws/aws-sdk-go v1.42.35/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc=
 github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4=
 github.com/aws/aws-sdk-go-v2/config v1.7.0/go.mod h1:w9+nMZ7soXCe5nT46Ri354SNhXDQ6v+V5wqDjnZE+GY=
 github.com/aws/aws-sdk-go-v2/credentials v1.4.0/go.mod h1:dgGR+Qq7Wjcd4AOAW5Rf5Tnv3+x7ed6kETXyS9WCuAY=
@@ -283,8 +283,9 @@ github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22
 github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
 github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
 github.com/goccy/go-json v0.7.6/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
-github.com/goccy/go-json v0.9.1 h1:xurvfvj3gq6SWUkkZ0opoUDTms7jif41uZ9z9s+hVlY=
 github.com/goccy/go-json v0.9.1/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/goccy/go-json v0.9.3 h1:VYKeLtdIQXWaeTZy5JNGZbVui5ck7Vf5MlWEcflqz0s=
+github.com/goccy/go-json v0.9.3/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
 github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
@@ -688,8 +689,8 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo=
 github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY=
-github.com/sftpgo/sdk v0.0.0-20220110174344-ecf586dd8941 h1:CxKFDSYekL6+dOZ9rSglYGwcXyhM4Aki6yDsdiPlJ5Y=
+github.com/sftpgo/sdk v0.0.0-20220115154521-b31d253a0bea h1:ouwL3x9tXiAXIhdXtJGONd905f1dBLu3HhfFoaTq24k=
-github.com/sftpgo/sdk v0.0.0-20220110174344-ecf586dd8941/go.mod h1:Bhgac6kiwIziILXLzH4wepT8lQXyhF83poDXqZorN6Q=
+github.com/sftpgo/sdk v0.0.0-20220115154521-b31d253a0bea/go.mod h1:Bhgac6kiwIziILXLzH4wepT8lQXyhF83poDXqZorN6Q=
 github.com/shirou/gopsutil/v3 v3.21.13-0.20220106132423-a3ae4bc40d26 h1:nkvraEu1xs6D3AimiR9SkIOCG6lVvVZRfwbbQ7fX1DY=
 github.com/shirou/gopsutil/v3 v3.21.13-0.20220106132423-a3ae4bc40d26/go.mod h1:BToYZVTlSVlfazpDDYFnsVZLaoRG+g8ufT6fPQLdJzA=
 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
@@ -942,8 +943,9 @@ golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220111092808-5a964db01320 h1:0jf+tOCoZ3LyutmCOWpVni1chK4VfFLhRsDK7MhqGRY=
 golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
+golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1160,8 +1162,8 @@ google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ6
 google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20220112215332-a9c7c0acf9f2 h1:z+R4M/SuyaRsj1zu3WC+nIQyfSrSIpuDcY01/R3uCtg=
+google.golang.org/genproto v0.0.0-20220114231437-d2e6a121cae0 h1:aCsSLXylHWFno0r4S3joLpiaWayvqd2Mn4iSvx4WZZc=
-google.golang.org/genproto v0.0.0-20220112215332-a9c7c0acf9f2/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/genproto v0.0.0-20220114231437-d2e6a121cae0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=

httpd/api_http_user.go

@@ -92,7 +92,7 @@ func createUserDir(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	}
-	err = connection.CreateDir(name)
+	err = connection.CreateDir(name, true)
 	if err != nil {
 		sendAPIResponse(w, r, err, fmt.Sprintf("Unable to create directory %#v", name), getMappedStatusCode(err))
 		return

httpd/handler.go

@@ -66,7 +66,7 @@ func (c *Connection) Stat(name string, mode int) (os.FileInfo, error) {
 		return nil, c.GetPermissionDeniedError()
 	}
 
-	fi, err := c.DoStat(name, mode)
+	fi, err := c.DoStat(name, mode, true)
 	if err != nil {
 		return nil, err
 	}
@@ -89,9 +89,9 @@ func (c *Connection) getFileReader(name string, offset int64, method string) (io
 		return nil, c.GetPermissionDeniedError()
 	}
 
-	if !c.User.IsFileAllowed(name) {
+	if ok, policy := c.User.IsFileAllowed(name); !ok {
 		c.Log(logger.LevelWarn, "reading file %#v is not allowed", name)
-		return nil, c.GetPermissionDeniedError()
+		return nil, c.GetErrorForDeniedFile(policy)
 	}
 
 	fs, p, err := c.GetFsAndResolvedPath(name)
@@ -120,7 +120,7 @@ func (c *Connection) getFileReader(name string, offset int64, method string) (io
 func (c *Connection) getFileWriter(name string) (io.WriteCloser, error) {
 	c.UpdateLastActivity()
 
-	if !c.User.IsFileAllowed(name) {
+	if ok, _ := c.User.IsFileAllowed(name); !ok {
 		c.Log(logger.LevelWarn, "writing file %#v is not allowed", name)
 		return nil, c.GetPermissionDeniedError()
 	}

httpd/httpd_test.go

@@ -1482,6 +1482,15 @@ func TestAddUserInvalidFilters(t *testing.T) {
 	}
 	_, _, err = httpdtest.AddUser(u, http.StatusBadRequest)
 	assert.NoError(t, err)
+	u.Filters.FilePatterns = []sdk.PatternsFilter{
+		{
+			Path:            "/subdir",
+			AllowedPatterns: []string{"*.*"},
+			DenyPolicy:      100,
+		},
+	}
+	_, _, err = httpdtest.AddUser(u, http.StatusBadRequest)
+	assert.NoError(t, err)
 	u.Filters.DeniedProtocols = []string{"invalid"}
 	_, _, err = httpdtest.AddUser(u, http.StatusBadRequest)
 	assert.NoError(t, err)
@@ -2087,6 +2096,7 @@ func TestUpdateUser(t *testing.T) {
 		Path:            "/subdir",
 		AllowedPatterns: []string{"*.zip", "*.rar"},
 		DeniedPatterns:  []string{"*.jpg", "*.png"},
+		DenyPolicy:      sdk.DenyPolicyHide,
 	})
 	user.Filters.MaxUploadFileSize = 4096
 	user.UploadBandwidth = 1024
@@ -13061,6 +13071,7 @@ func TestWebUserAddMock(t *testing.T) {
 	form.Set("pattern_path0", "/dir2")
 	form.Set("patterns0", "*.jpg,*.png")
 	form.Set("pattern_type0", "allowed")
+	form.Set("pattern_policy0", "1")
 	form.Set("pattern_path1", "/dir1")
 	form.Set("patterns1", "*.png")
 	form.Set("pattern_type1", "allowed")
@@ -13294,23 +13305,25 @@ func TestWebUserAddMock(t *testing.T) {
 	}
 	assert.Len(t, newUser.Filters.FilePatterns, 3)
 	for _, filter := range newUser.Filters.FilePatterns {
-		if filter.Path == "/dir1" {
+		switch filter.Path {
+		case "/dir1":
 			assert.Len(t, filter.DeniedPatterns, 1)
 			assert.Len(t, filter.AllowedPatterns, 1)
 			assert.True(t, util.IsStringInSlice("*.png", filter.AllowedPatterns))
 			assert.True(t, util.IsStringInSlice("*.zip", filter.DeniedPatterns))
-		}
+			assert.Equal(t, sdk.DenyPolicyDefault, filter.DenyPolicy)
-		if filter.Path == "/dir2" {
+		case "/dir2":
 			assert.Len(t, filter.DeniedPatterns, 1)
 			assert.Len(t, filter.AllowedPatterns, 2)
 			assert.True(t, util.IsStringInSlice("*.jpg", filter.AllowedPatterns))
 			assert.True(t, util.IsStringInSlice("*.png", filter.AllowedPatterns))
 			assert.True(t, util.IsStringInSlice("*.mkv", filter.DeniedPatterns))
-		}
+			assert.Equal(t, sdk.DenyPolicyHide, filter.DenyPolicy)
-		if filter.Path == "/dir3" {
+		case "/dir3":
 			assert.Len(t, filter.DeniedPatterns, 1)
 			assert.Len(t, filter.AllowedPatterns, 0)
 			assert.True(t, util.IsStringInSlice("*.rar", filter.DeniedPatterns))
+			assert.Equal(t, sdk.DenyPolicyDefault, filter.DenyPolicy)
 		}
 	}
 	if assert.Len(t, newUser.Filters.BandwidthLimits, 2) {
@@ -14140,9 +14153,11 @@ func TestWebUserS3Mock(t *testing.T) {
 	form.Set("pattern_path0", "/dir1")
 	form.Set("patterns0", "*.jpg,*.png")
 	form.Set("pattern_type0", "allowed")
+	form.Set("pattern_policy0", "0")
 	form.Set("pattern_path1", "/dir2")
 	form.Set("patterns1", "*.zip")
 	form.Set("pattern_type1", "denied")
+	form.Set("pattern_policy1", "1")
 	form.Set("max_upload_file_size", "0")
 	form.Set("s3_force_path_style", "checked")
 	form.Set("description", user.Description)
@@ -14221,7 +14236,16 @@ func TestWebUserS3Mock(t *testing.T) {
 	assert.Equal(t, updateUser.FsConfig.S3Config.DownloadPartSize, user.FsConfig.S3Config.DownloadPartSize)
 	assert.Equal(t, updateUser.FsConfig.S3Config.DownloadConcurrency, user.FsConfig.S3Config.DownloadConcurrency)
 	assert.True(t, updateUser.FsConfig.S3Config.ForcePathStyle)
-	assert.Equal(t, 2, len(updateUser.Filters.FilePatterns))
+	if assert.Equal(t, 2, len(updateUser.Filters.FilePatterns)) {
+		for _, filter := range updateUser.Filters.FilePatterns {
+			switch filter.Path {
+			case "/dir1":
+				assert.Equal(t, sdk.DenyPolicyDefault, filter.DenyPolicy)
+			case "/dir2":
+				assert.Equal(t, sdk.DenyPolicyHide, filter.DenyPolicy)
+			}
+		}
+	}
 	assert.Equal(t, sdkkms.SecretStatusSecretBox, updateUser.FsConfig.S3Config.AccessSecret.GetStatus())
 	assert.NotEmpty(t, updateUser.FsConfig.S3Config.AccessSecret.GetPayload())
 	assert.Empty(t, updateUser.FsConfig.S3Config.AccessSecret.GetKey())

httpd/webadmin.go

@@ -797,11 +797,20 @@ func getBandwidthLimitsFromPostFields(r *http.Request) ([]sdk.BandwidthLimit, er
 	return result, nil
 }
 
+func getPatterDenyPolicyFromString(policy string) int {
+	denyPolicy := sdk.DenyPolicyDefault
+	if policy == "1" {
+		denyPolicy = sdk.DenyPolicyHide
+	}
+	return denyPolicy
+}
+
 func getFilePatternsFromPostField(r *http.Request) []sdk.PatternsFilter {
 	var result []sdk.PatternsFilter
 
 	allowedPatterns := make(map[string][]string)
 	deniedPatterns := make(map[string][]string)
+	patternPolicies := make(map[string]string)
 
 	for k := range r.Form {
 		if strings.HasPrefix(k, "pattern_path") {
@@ -810,12 +819,16 @@ func getFilePatternsFromPostField(r *http.Request) []sdk.PatternsFilter {
 			filters := strings.TrimSpace(r.Form.Get(fmt.Sprintf("patterns%v", idx)))
 			filters = strings.ReplaceAll(filters, " ", "")
 			patternType := r.Form.Get(fmt.Sprintf("pattern_type%v", idx))
+			patternPolicy := r.Form.Get(fmt.Sprintf("pattern_policy%v", idx))
 			if p != "" && filters != "" {
 				if patternType == "allowed" {
 					allowedPatterns[p] = append(allowedPatterns[p], strings.Split(filters, ",")...)
 				} else {
 					deniedPatterns[p] = append(deniedPatterns[p], strings.Split(filters, ",")...)
 				}
+				if patternPolicy != "" && patternPolicy != "0" {
+					patternPolicies[p] = patternPolicy
+				}
 			}
 		}
 	}
@@ -823,11 +836,12 @@ func getFilePatternsFromPostField(r *http.Request) []sdk.PatternsFilter {
 	for dirAllowed, allowPatterns := range allowedPatterns {
 		filter := sdk.PatternsFilter{
 			Path:            dirAllowed,
-			AllowedPatterns: util.RemoveDuplicates(allowPatterns),
+			AllowedPatterns: allowPatterns,
+			DenyPolicy:      getPatterDenyPolicyFromString(patternPolicies[dirAllowed]),
 		}
 		for dirDenied, denPatterns := range deniedPatterns {
 			if dirAllowed == dirDenied {
-				filter.DeniedPatterns = util.RemoveDuplicates(denPatterns)
+				filter.DeniedPatterns = denPatterns
 				break
 			}
 		}
@@ -845,6 +859,7 @@ func getFilePatternsFromPostField(r *http.Request) []sdk.PatternsFilter {
 			result = append(result, sdk.PatternsFilter{
 				Path:           dirDenied,
 				DeniedPatterns: denPatterns,
+				DenyPolicy:     getPatterDenyPolicyFromString(patternPolicies[dirDenied]),
 			})
 		}
 	}

httpdtest/httpdtest.go

@@ -1542,7 +1542,7 @@ func compareUserFilePatternsFilters(expected *dataprovider.User, actual *datapro
 	for _, f := range expected.Filters.FilePatterns {
 		found := false
 		for _, f1 := range actual.Filters.FilePatterns {
-			if path.Clean(f.Path) == path.Clean(f1.Path) {
+			if path.Clean(f.Path) == path.Clean(f1.Path) && f.DenyPolicy == f1.DenyPolicy {
 				if !checkFilterMatch(f.AllowedPatterns, f1.AllowedPatterns) ||
 					!checkFilterMatch(f.DeniedPatterns, f1.DeniedPatterns) {
 					return errors.New("file patterns contents mismatch")

openapi/openapi.yaml

@@ -4373,7 +4373,7 @@ components:
           type: array
           items:
             type: string
-          description: 'list of, case insensitive, allowed shell like file patterns.'
+          description: 'list of, case insensitive, allowed shell like patterns.'
           example:
             - '*.jpg'
             - a*b?.png
@@ -4381,9 +4381,18 @@ components:
           type: array
           items:
             type: string
-          description: 'list of, case insensitive, denied shell like file patterns. Denied patterns are evaluated before the allowed ones'
+          description: 'list of, case insensitive, denied shell like patterns. Denied patterns are evaluated before the allowed ones'
           example:
             - '*.zip'
+        deny_policy:
+          type: integer
+          enum:
+            - 0
+            - 1
+          description: |
+            Deny policies
+              * `0` - default policy. Denied files/directories matching the filters are visible in directory listing but cannot be uploaded/downloaded/overwritten/renamed
+              * `1` - deny policy hide. This policy applies the same restrictions as the default one and denied files/directories matching the filters will also be hidden in directory listing. This mode may cause performance issues for large directories
     HooksFilter:
       type: object
       properties:

sftpd/handler.go

@@ -63,9 +63,9 @@ func (c *Connection) Fileread(request *sftp.Request) (io.ReaderAt, error) {
 		return nil, sftp.ErrSSHFxPermissionDenied
 	}
 
-	if !c.User.IsFileAllowed(request.Filepath) {
+	if ok, policy := c.User.IsFileAllowed(request.Filepath); !ok {
 		c.Log(logger.LevelWarn, "reading file %#v is not allowed", request.Filepath)
-		return nil, sftp.ErrSSHFxPermissionDenied
+		return nil, c.GetErrorForDeniedFile(policy)
 	}
 
 	fs, p, err := c.GetFsAndResolvedPath(request.Filepath)
@@ -104,9 +104,9 @@ func (c *Connection) Filewrite(request *sftp.Request) (io.WriterAt, error) {
 func (c *Connection) handleFilewrite(request *sftp.Request) (sftp.WriterAtReaderAt, error) {
 	c.UpdateLastActivity()
 
-	if !c.User.IsFileAllowed(request.Filepath) {
+	if ok, _ := c.User.IsFileAllowed(request.Filepath); !ok {
 		c.Log(logger.LevelWarn, "writing file %#v is not allowed", request.Filepath)
-		return nil, sftp.ErrSSHFxPermissionDenied
+		return nil, c.GetPermissionDeniedError()
 	}
 
 	fs, p, err := c.GetFsAndResolvedPath(request.Filepath)
@@ -175,7 +175,7 @@ func (c *Connection) Filecmd(request *sftp.Request) error {
 	case "Rmdir":
 		return c.RemoveDir(request.Filepath)
 	case "Mkdir":
-		err := c.CreateDir(request.Filepath)
+		err := c.CreateDir(request.Filepath, true)
 		if err != nil {
 			return err
 		}
@@ -214,7 +214,7 @@ func (c *Connection) Filelist(request *sftp.Request) (sftp.ListerAt, error) {
 			return nil, sftp.ErrSSHFxPermissionDenied
 		}
 
-		s, err := c.DoStat(request.Filepath, 0)
+		s, err := c.DoStat(request.Filepath, 0, true)
 		if err != nil {
 			return nil, err
 		}
@@ -255,7 +255,7 @@ func (c *Connection) Lstat(request *sftp.Request) (sftp.ListerAt, error) {
 		return nil, sftp.ErrSSHFxPermissionDenied
 	}
 
-	s, err := c.DoStat(request.Filepath, 1)
+	s, err := c.DoStat(request.Filepath, 1, true)
 	if err != nil {
 		return nil, err
 	}

sftpd/scp.go

@@ -148,7 +148,7 @@ func (c *scpCommand) handleCreateDir(fs vfs.Fs, dirPath string) error {
 		return common.ErrPermissionDenied
 	}
 
-	info, err := c.connection.DoStat(dirPath, 1)
+	info, err := c.connection.DoStat(dirPath, 1, true)
 	if err == nil && info.IsDir() {
 		return nil
 	}
@@ -276,9 +276,9 @@ func (c *scpCommand) handleUpload(uploadFilePath string, sizeToRead int64) error
 		return err
 	}
 
-	if !c.connection.User.IsFileAllowed(uploadFilePath) {
+	if ok, _ := c.connection.User.IsFileAllowed(uploadFilePath); !ok {
 		c.connection.Log(logger.LevelWarn, "writing file %#v is not allowed", uploadFilePath)
-		c.sendErrorMessage(fs, common.ErrPermissionDenied)
+		c.sendErrorMessage(fs, c.connection.GetPermissionDeniedError())
 		return common.ErrPermissionDenied
 	}
 
@@ -372,7 +372,7 @@ func (c *scpCommand) handleRecursiveDownload(fs vfs.Fs, dirPath, virtualPath str
 			c.sendErrorMessage(fs, err)
 			return err
 		}
-		files = c.connection.User.AddVirtualDirs(files, fs.GetRelativePath(dirPath))
+		files = c.connection.User.FilterListDir(files, fs.GetRelativePath(dirPath))
 		var dirs []string
 		for _, file := range files {
 			filePath := fs.GetRelativePath(fs.Join(dirPath, file.Name()))
@@ -509,9 +509,9 @@ func (c *scpCommand) handleDownload(filePath string) error {
 		return common.ErrPermissionDenied
 	}
 
-	if !c.connection.User.IsFileAllowed(filePath) {
+	if ok, policy := c.connection.User.IsFileAllowed(filePath); !ok {
 		c.connection.Log(logger.LevelWarn, "reading file %#v is not allowed", filePath)
-		c.sendErrorMessage(fs, common.ErrPermissionDenied)
+		c.sendErrorMessage(fs, c.connection.GetErrorForDeniedFile(policy))
 		return common.ErrPermissionDenied
 	}
 

sftpd/sftpd_test.go

@@ -7423,9 +7423,12 @@ func TestFilterFilePatterns(t *testing.T) {
 		},
 	}
 	user.Filters = filters
-	assert.True(t, user.IsFileAllowed("/test/test.jPg"))
+	ok, _ := user.IsFileAllowed("/test/test.jPg")
-	assert.False(t, user.IsFileAllowed("/test/test.pdf"))
+	assert.True(t, ok)
-	assert.True(t, user.IsFileAllowed("/test.pDf"))
+	ok, _ = user.IsFileAllowed("/test/test.pdf")
+	assert.False(t, ok)
+	ok, _ = user.IsFileAllowed("/test.pDf")
+	assert.True(t, ok)
 
 	filters.FilePatterns = append(filters.FilePatterns, sdk.PatternsFilter{
 		Path:            "/",
@@ -7433,19 +7436,26 @@ func TestFilterFilePatterns(t *testing.T) {
 		DeniedPatterns:  []string{"*.gz"},
 	})
 	user.Filters = filters
-	assert.False(t, user.IsFileAllowed("/test1/test.gz"))
+	ok, _ = user.IsFileAllowed("/test1/test.gz")
-	assert.True(t, user.IsFileAllowed("/test1/test.zip"))
+	assert.False(t, ok)
-	assert.False(t, user.IsFileAllowed("/test/sub/test.pdf"))
+	ok, _ = user.IsFileAllowed("/test1/test.zip")
-	assert.False(t, user.IsFileAllowed("/test1/test.png"))
+	assert.True(t, ok)
+	ok, _ = user.IsFileAllowed("/test/sub/test.pdf")
+	assert.False(t, ok)
+	ok, _ = user.IsFileAllowed("/test1/test.png")
+	assert.False(t, ok)
 
 	filters.FilePatterns = append(filters.FilePatterns, sdk.PatternsFilter{
 		Path:           "/test/sub",
 		DeniedPatterns: []string{"*.tar"},
 	})
 	user.Filters = filters
-	assert.False(t, user.IsFileAllowed("/test/sub/sub/test.tar"))
+	ok, _ = user.IsFileAllowed("/test/sub/sub/test.tar")
-	assert.True(t, user.IsFileAllowed("/test/sub/test.gz"))
+	assert.False(t, ok)
-	assert.False(t, user.IsFileAllowed("/test/test.zip"))
+	ok, _ = user.IsFileAllowed("/test/sub/test.gz")
+	assert.True(t, ok)
+	ok, _ = user.IsFileAllowed("/test/test.zip")
+	assert.False(t, ok)
 }
 
 func TestUserAllowedLoginMethods(t *testing.T) {

sftpd/ssh_cmd.go

@@ -161,7 +161,7 @@ func (c *sshCommand) handleSFTPGoCopy() error {
 			return c.sendErrorResponse(err)
 		}
 	} else if fi.Mode().IsRegular() {
-		if !c.connection.User.IsFileAllowed(sshDestPath) {
+		if ok, _ := c.connection.User.IsFileAllowed(sshDestPath); !ok {
 			err := errors.New("unsupported copy destination: this file is not allowed")
 			return c.sendErrorResponse(err)
 		}
@@ -282,9 +282,9 @@ func (c *sshCommand) handleHashCommands() error {
 		response = fmt.Sprintf("%x  -\n", h.Sum(nil))
 	} else {
 		sshPath := c.getDestPath()
-		if !c.connection.User.IsFileAllowed(sshPath) {
+		if ok, policy := c.connection.User.IsFileAllowed(sshPath); !ok {
 			c.connection.Log(logger.LevelInfo, "hash not allowed for file %#v", sshPath)
-			return c.sendErrorResponse(c.connection.GetPermissionDeniedError())
+			return c.sendErrorResponse(c.connection.GetErrorForDeniedFile(policy))
 		}
 		fs, fsPath, err := c.connection.GetFsAndResolvedPath(sshPath)
 		if err != nil {

templates/webadmin/user.html

@@ -385,18 +385,19 @@
 
                             <div class="card bg-light mb-3">
                                 <div class="card-header">
-                                    <b>Per-directory file patterns</b>
+                                    <b>Per-directory pattern restrictions</b>
                                 </div>
                                 <div class="card-body">
-                                    <h6 class="card-title mb-4">Comma separated denied or allowed files, based on shell patterns</h6>
+                                    <h6 class="card-title mb-4">Comma separated denied or allowed files/directories, based on shell patterns.</h6>
+                                    <p class="card-text">Denied entries are visible in directory listing by default, you can hide them by setting the "Hidden" policy, but please be aware that this may cause performance issues for large directories</p>
                                     <div class="form-group row">
                                         <div class="col-md-12 form_field_patterns_outer">
                                             {{range $idx, $pattern := .User.GetFlatFilePatterns -}}
                                             <div class="row form_field_patterns_outer_row">
-                                                <div class="form-group col-md-4">
+                                                <div class="form-group col-md-3">
                                                     <input type="text" class="form-control" id="idPatternPath{{$idx}}" name="pattern_path{{$idx}}" placeholder="directory path, i.e. /dir" value="{{$pattern.Path}}" maxlength="255">
                                                 </div>
-                                                <div class="form-group col-md-5">
+                                                <div class="form-group col-md-4">
                                                     <input type="text" class="form-control" id="idPatterns{{$idx}}" name="patterns{{$idx}}" placeholder="*.zip,?.txt" value="{{$pattern.GetCommaSeparatedPatterns}}" maxlength="255">
                                                 </div>
                                                 <div class="form-group col-md-2">
@@ -405,6 +406,12 @@
                                                         <option value="allowed" {{if $pattern.IsAllowed}}selected{{end}}>Allowed</option>
                                                     </select>
                                                 </div>
+                                                <div class="form-group col-md-2">
+                                                    <select class="form-control" id="idPatternPolicy{{$idx}}" name="pattern_policy{{$idx}}">
+                                                        <option value="0" {{if eq $pattern.DenyPolicy 0}}selected{{end}}>Visible</option>
+                                                        <option value="1" {{if eq $pattern.DenyPolicy 1}}selected{{end}}>Hidden</option>
+                                                    </select>
+                                                </div>
                                                 <div class="form-group col-md-1">
                                                     <button class="btn btn-circle btn-danger remove_pattern_btn_frm_field">
                                                         <i class="fas fa-trash"></i>
@@ -413,10 +420,10 @@
                                             </div>
                                             {{else}}
                                             <div class="row form_field_patterns_outer_row">
-                                                <div class="form-group col-md-4">
+                                                <div class="form-group col-md-3">
                                                     <input type="text" class="form-control" id="idPatternPath0" name="pattern_path0" placeholder="directory path, i.e. /dir" value="" maxlength="255">
                                                 </div>
-                                                <div class="form-group col-md-5">
+                                                <div class="form-group col-md-4">
                                                     <input type="text" class="form-control" id="idPatterns0" name="patterns0" placeholder="*.zip,?.txt" value="" maxlength="255">
                                                 </div>
                                                 <div class="form-group col-md-2">
@@ -425,6 +432,12 @@
                                                         <option value="allowed">Allowed</option>
                                                     </select>
                                                 </div>
+                                                <div class="form-group col-md-2">
+                                                    <select class="form-control" id="idPatternPolicy0" name="pattern_policy0">
+                                                        <option value="0">Visible</option>
+                                                        <option value="1">Hidden</option>
+                                                    </select>
+                                                </div>
                                                 <div class="form-group col-md-1">
                                                     <button class="btn btn-circle btn-danger remove_pattern_btn_frm_field">
                                                         <i class="fas fa-trash"></i>
@@ -956,10 +969,10 @@
             }
             $(".form_field_patterns_outer").append(`
                     <div class="row form_field_patterns_outer_row">
-                        <div class="form-group col-md-4">
+                        <div class="form-group col-md-3">
                             <input type="text" class="form-control" id="idPatternPath${index}" name="pattern_path${index}" placeholder="directory path, i.e. /dir" value="" maxlength="255">
                         </div>
-                        <div class="form-group col-md-5">
+                        <div class="form-group col-md-4">
                             <input type="text" class="form-control" id="idPatterns${index}" name="patterns${index}" placeholder="*.zip,?.txt" value="" maxlength="255">
                         </div>
                         <div class="form-group col-md-2">
@@ -968,6 +981,12 @@
                                 <option value="allowed">Allowed</option>
                             </select>
                         </div>
+                        <div class="form-group col-md-2">
+                            <select class="form-control" id="idPatternPolicy${index}" name="pattern_policy${index}">
+                                <option value="0">Visible</option>
+                                <option value="1">Hidden</option>
+                            </select>
+                        </div>
                         <div class="form-group col-md-1">
                             <button class="btn btn-circle btn-danger remove_pattern_btn_frm_field">
                                 <i class="fas fa-trash"></i>

webdavd/file.go

@@ -143,9 +143,9 @@ func (f *webDavFile) Read(p []byte) (n int, err error) {
 			return 0, f.Connection.GetPermissionDeniedError()
 		}
 
-		if !f.Connection.User.IsFileAllowed(f.GetVirtualPath()) {
+		if ok, policy := f.Connection.User.IsFileAllowed(f.GetVirtualPath()); !ok {
 			f.Connection.Log(logger.LevelWarn, "reading file %#v is not allowed", f.GetVirtualPath())
-			return 0, f.Connection.GetPermissionDeniedError()
+			return 0, f.Connection.GetErrorForDeniedFile(policy)
 		}
 		err := common.ExecutePreAction(f.Connection, common.OperationPreDownload, f.GetFsPath(), f.GetVirtualPath(), 0, 0)
 		if err != nil {

webdavd/handler.go

@@ -62,7 +62,7 @@ func (c *Connection) Mkdir(ctx context.Context, name string, perm os.FileMode) e
 	c.UpdateLastActivity()
 
 	name = util.CleanPath(name)
-	return c.CreateDir(name)
+	return c.CreateDir(name, true)
 }
 
 // Rename renames a file or a directory
@@ -85,7 +85,7 @@ func (c *Connection) Stat(ctx context.Context, name string) (os.FileInfo, error)
 		return nil, c.GetPermissionDeniedError()
 	}
 
-	fi, err := c.DoStat(name, 0)
+	fi, err := c.DoStat(name, 0, true)
 	if err != nil {
 		return nil, err
 	}
@@ -156,7 +156,7 @@ func (c *Connection) getFile(fs vfs.Fs, fsPath, virtualPath string) (webdav.File
 }
 
 func (c *Connection) putFile(fs vfs.Fs, fsPath, virtualPath string) (webdav.File, error) {
-	if !c.User.IsFileAllowed(virtualPath) {
+	if ok, _ := c.User.IsFileAllowed(virtualPath); !ok {
 		c.Log(logger.LevelWarn, "writing file %#v is not allowed", virtualPath)
 		return nil, c.GetPermissionDeniedError()
 	}