webclient: allow to download multiple files as zip

docs/full-configuration.md

@@ -149,7 +149,7 @@ The configuration file contains the following sections:
     - `client_auth_type`, integer. Set to `1` to require a client certificate and verify it. Set to `2` to request a client certificate during the TLS handshake and verify it if given, in this mode the client is allowed not to send a certificate. At least one certification authority must be defined in order to verify client certificates. If no certification authority is defined, this setting is ignored. Default: 0.
     - `tls_cipher_suites`, list of strings. List of supported cipher suites for TLS version 1.2. If empty, a default list of secure cipher suites is used, with a preference order based on hardware performance. Note that TLS 1.3 ciphersuites are not configurable. The supported ciphersuites names are defined [here](https://github.com/golang/go/blob/master/src/crypto/tls/cipher_suites.go#L52). Any invalid name will be silently ignored. The order matters, the ciphers listed first will be the preferred ones. Default: empty.
     - `prefix`, string. Prefix for WebDAV resources, if empty WebDAV resources will be available at the `/` URI. If defined it must be an absolute URI, for example `/dav`. Default: "".
-    - `proxy_allowed`, list of IP addresses and IP ranges allowed to set `X-Forwarded-For`, `X-Real-IP` headers. Any of the indicated headers, if set on requests from a connection address not in this list, will be silently ignored. Default: empty.
+    - `proxy_allowed`, list of IP addresses and IP ranges allowed to set `X-Forwarded-For`, `X-Real-IP`, `CF-Connecting-IP`, `True-Client-IP` headers. Any of the indicated headers, if set on requests from a connection address not in this list, will be silently ignored. Default: empty.
   - `bind_port`, integer. Deprecated, please use `bindings`.
   - `bind_address`, string. Deprecated, please use `bindings`.
   - `certificate_file`, string. Certificate for WebDAV over HTTPS. This can be an absolute path or a path relative to the config dir.
@@ -220,7 +220,7 @@ The configuration file contains the following sections:
     - `enable_https`, boolean. Set to `true` and provide both a certificate and a key file to enable HTTPS connection for this binding. Default `false`.
     - `client_auth_type`, integer. Set to `1` to require client certificate authentication in addition to JWT/Web authentication. You need to define at least a certificate authority for this to work. Default: 0.
     - `tls_cipher_suites`, list of strings. List of supported cipher suites for TLS version 1.2. If empty, a default list of secure cipher suites is used, with a preference order based on hardware performance. Note that TLS 1.3 ciphersuites are not configurable. The supported ciphersuites names are defined [here](https://github.com/golang/go/blob/master/src/crypto/tls/cipher_suites.go#L52). Any invalid name will be silently ignored. The order matters, the ciphers listed first will be the preferred ones. Default: empty.
-    - `proxy_allowed`, list of IP addresses and IP ranges allowed to set `X-Forwarded-For`, `X-Real-IP`, `X-Forwarded-Proto` headers. Any of the indicated headers, if set on requests from a connection address not in this list, will be silently ignored. Default: empty.
+    - `proxy_allowed`, list of IP addresses and IP ranges allowed to set `X-Forwarded-For`, `X-Real-IP`, `X-Forwarded-Proto`, `CF-Connecting-IP`, `True-Client-IP` headers. Any of the indicated headers, if set on requests from a connection address not in this list, will be silently ignored. Default: empty.
   - `bind_port`, integer. Deprecated, please use `bindings`.
   - `bind_address`, string. Deprecated, please use `bindings`. Leave blank to listen on all available network interfaces. On \*NIX you can specify an absolute path to listen on a Unix-domain socket. Default: ""
   - `templates_path`, string. Path to the HTML web templates. This can be an absolute path or a path relative to the config dir

docs/web-client.md

@@ -4,7 +4,8 @@ SFTPGo provides a basic front-end web interface for your users. It allows end-us
 
 The web interface can be globally disabled within the `httpd` configuration via the `enable_web_client` key or on a per-user basis by adding `HTTP` to the denied protocols.
 Public keys management can be disabled, per-user, using a specific permission.
+The web client allows you to download multiple files or folders as a single zip file, any non regular files (for example symlinks) will be silently ignored.
 
-With the default `httpd` configuration, the web admin is available at the following URL:
+With the default `httpd` configuration, the web client is available at the following URL:
 
 [http://127.0.0.1:8080/web/client](http://127.0.0.1:8080/web/client)

go.mod

@@ -8,10 +8,10 @@ require (
 	github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962
 	github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect
 	github.com/alexedwards/argon2id v0.0.0-20210511081203-7d35d68092b8
-	github.com/aws/aws-sdk-go v1.38.49
+	github.com/aws/aws-sdk-go v1.38.51
 	github.com/cockroachdb/cockroach-go/v2 v2.1.1
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect
-	github.com/eikenb/pipeat v0.0.0-20200430215831-470df5986b6d
+	github.com/eikenb/pipeat v0.0.0-20210528001815-f1fcb2512a3a
 	github.com/fclairamb/ftpserverlib v0.13.1
 	github.com/frankban/quicktest v1.13.0 // indirect
 	github.com/go-chi/chi/v5 v5.0.3
@@ -25,6 +25,7 @@ require (
 	github.com/grandcat/zeroconf v1.0.0
 	github.com/hashicorp/go-retryablehttp v0.7.0
 	github.com/jlaffaye/ftp v0.0.0-20201112195030-9aae4d151126
+	github.com/klauspost/compress v1.12.3
 	github.com/klauspost/cpuid/v2 v2.0.6 // indirect
 	github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect
 	github.com/lestrrat-go/jwx v1.2.0

go.sum

@@ -127,8 +127,8 @@ github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpi
 github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.30.27/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
-github.com/aws/aws-sdk-go v1.38.49 h1:E31vxjCe6a5I+mJLmUGaZobiWmg9KdWaud9IfceYeYQ=
-github.com/aws/aws-sdk-go v1.38.49/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
+github.com/aws/aws-sdk-go v1.38.51 h1:aKQmbVbwOCuQSd8+fm/MR3bq0QOsu9Q7S+/QEND36oQ=
+github.com/aws/aws-sdk-go v1.38.51/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@@ -215,8 +215,8 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
-github.com/eikenb/pipeat v0.0.0-20200430215831-470df5986b6d h1:8RvCRWer7TB2n+DKhW4uW15hRiqPmabSnSyYhju/Nuw=
-github.com/eikenb/pipeat v0.0.0-20200430215831-470df5986b6d/go.mod h1:+JPhBw5JdJrSF80r6xsSg1TYHjyAGxYs4X24VyUdMZU=
+github.com/eikenb/pipeat v0.0.0-20210528001815-f1fcb2512a3a h1:+btFAKG3kNCqm1DMKDGaWkolX/4aytcbvnfdgt6z+UI=
+github.com/eikenb/pipeat v0.0.0-20210528001815-f1fcb2512a3a/go.mod h1:+JPhBw5JdJrSF80r6xsSg1TYHjyAGxYs4X24VyUdMZU=
 github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -544,6 +544,8 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
+github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU=
+github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.6 h1:dQ5ueTiftKxp0gyjKSx5+8BtPWkyQbd95m8Gys/RarI=
 github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=

httpd/api_utils.go

@@ -3,14 +3,19 @@ package httpd
 import (
 	"context"
 	"errors"
+	"io"
 	"net/http"
 	"os"
+	"path"
 	"strconv"
+	"strings"
 
 	"github.com/go-chi/render"
+	"github.com/klauspost/compress/zip"
 
 	"github.com/drakkan/sftpgo/common"
 	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/logger"
 )
 
 func sendAPIResponse(w http.ResponseWriter, r *http.Request, err error, message string, code int) {
@@ -90,3 +95,74 @@ func getSearchFilters(w http.ResponseWriter, r *http.Request) (int, int, string,
 
 	return limit, offset, order, err
 }
+
+func renderCompressedFiles(w http.ResponseWriter, conn *Connection, baseDir string, files []string) {
+	w.Header().Set("Content-Type", "application/zip")
+	w.Header().Set("Accept-Ranges", "none")
+	w.Header().Set("Content-Transfer-Encoding", "binary")
+	w.WriteHeader(http.StatusOK)
+
+	wr := zip.NewWriter(w)
+
+	for _, file := range files {
+		fullPath := path.Join(baseDir, file)
+		if err := addZipEntry(wr, conn, fullPath, baseDir); err != nil {
+			panic(http.ErrAbortHandler)
+		}
+	}
+	if err := wr.Close(); err != nil {
+		conn.Log(logger.LevelWarn, "unable to close zip file: %v", err)
+		panic(http.ErrAbortHandler)
+	}
+}
+
+func addZipEntry(wr *zip.Writer, conn *Connection, entryPath, baseDir string) error {
+	info, err := conn.Stat(entryPath, 1)
+	if err != nil {
+		conn.Log(logger.LevelDebug, "unable to add zip entry %#v, stat error: %v", entryPath, err)
+		return err
+	}
+	if info.IsDir() {
+		_, err := wr.Create(getZipEntryName(entryPath, baseDir) + "/")
+		if err != nil {
+			conn.Log(logger.LevelDebug, "unable to create zip entry %#v: %v", entryPath, err)
+			return err
+		}
+		contents, err := conn.ReadDir(entryPath)
+		if err != nil {
+			conn.Log(logger.LevelDebug, "unable to add zip entry %#v, read dir error: %v", entryPath, err)
+			return err
+		}
+		for _, info := range contents {
+			fullPath := path.Join(entryPath, info.Name())
+			if err := addZipEntry(wr, conn, fullPath, baseDir); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+	if !info.Mode().IsRegular() {
+		// we only allow regular files
+		conn.Log(logger.LevelDebug, "skipping zip entry for non regular file %#v", entryPath)
+		return nil
+	}
+	reader, err := conn.getFileReader(entryPath, 0, http.MethodGet)
+	if err != nil {
+		conn.Log(logger.LevelDebug, "unable to add zip entry %#v, cannot open file: %v", entryPath, err)
+		return err
+	}
+	defer reader.Close()
+
+	f, err := wr.Create(getZipEntryName(entryPath, baseDir))
+	if err != nil {
+		conn.Log(logger.LevelDebug, "unable to create zip entry %#v: %v", entryPath, err)
+		return err
+	}
+	_, err = io.Copy(f, reader)
+	return err
+}
+
+func getZipEntryName(entryPath, baseDir string) string {
+	entryPath = strings.TrimPrefix(entryPath, baseDir)
+	return strings.TrimPrefix(entryPath, "/")
+}

httpd/httpd.go

@@ -75,6 +75,7 @@ const (
 	webClientLoginPathDefault       = "/web/client/login"
 	webClientFilesPathDefault       = "/web/client/files"
 	webClientDirContentsPathDefault = "/web/client/listdir"
+	webClientDownloadPathDefault    = "/web/client/download"
 	webClientCredentialsPathDefault = "/web/client/credentials"
 	webChangeClientPwdPathDefault   = "/web/client/changepwd"
 	webChangeClientKeysPathDefault  = "/web/client/managekeys"
@@ -120,6 +121,7 @@ var (
 	webClientLoginPath       string
 	webClientFilesPath       string
 	webClientDirContentsPath string
+	webClientDownloadPath    string
 	webClientCredentialsPath string
 	webChangeClientPwdPath   string
 	webChangeClientKeysPath  string
@@ -415,6 +417,7 @@ func updateWebClientURLs(baseURL string) {
 	webClientLoginPath = path.Join(baseURL, webClientLoginPathDefault)
 	webClientFilesPath = path.Join(baseURL, webClientFilesPathDefault)
 	webClientDirContentsPath = path.Join(baseURL, webClientDirContentsPathDefault)
+	webClientDownloadPath = path.Join(baseURL, webClientDownloadPathDefault)
 	webClientCredentialsPath = path.Join(baseURL, webClientCredentialsPathDefault)
 	webChangeClientPwdPath = path.Join(baseURL, webChangeClientPwdPathDefault)
 	webChangeClientKeysPath = path.Join(baseURL, webChangeClientKeysPathDefault)

httpd/httpd_test.go

@@ -92,6 +92,7 @@ const (
 	webClientLoginPath        = "/web/client/login"
 	webClientFilesPath        = "/web/client/files"
 	webClientDirContentsPath  = "/web/client/listdir"
+	webClientDownloadPath     = "/web/client/download"
 	webClientCredentialsPath  = "/web/client/credentials"
 	webChangeClientPwdPath    = "/web/client/changepwd"
 	webChangeClientKeysPath   = "/web/client/managekeys"
@@ -4576,6 +4577,12 @@ func TestWebClientLoginMock(t *testing.T) {
 	checkResponseCode(t, http.StatusInternalServerError, rr)
 	assert.Contains(t, rr.Body.String(), "unable to retrieve your user")
 
+	req, _ = http.NewRequest(http.MethodGet, webClientDownloadPath, nil)
+	setJWTCookieForReq(req, webToken)
+	rr = executeRequest(req)
+	checkResponseCode(t, http.StatusInternalServerError, rr)
+	assert.Contains(t, rr.Body.String(), "Unable to retrieve your user")
+
 	csrfToken, err := getCSRFToken(httpBaseURL + webClientLoginPath)
 	assert.NoError(t, err)
 	form := make(url.Values)
@@ -4993,6 +5000,24 @@ func TestWebGetFiles(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Len(t, dirContents, 1)
 
+	req, _ = http.NewRequest(http.MethodGet, webClientDownloadPath+"?path="+url.QueryEscape("/")+"&files="+
+		url.QueryEscape(fmt.Sprintf(`["%v","%v","%v"]`, testFileName, testDir, testFileName+extensions[2])), nil)
+	setJWTCookieForReq(req, webToken)
+	rr = executeRequest(req)
+	checkResponseCode(t, http.StatusOK, rr)
+
+	req, _ = http.NewRequest(http.MethodGet, webClientDownloadPath+"?path="+url.QueryEscape("/")+"&files="+
+		url.QueryEscape(fmt.Sprintf(`["%v"]`, testDir)), nil)
+	setJWTCookieForReq(req, webToken)
+	rr = executeRequest(req)
+	checkResponseCode(t, http.StatusOK, rr)
+
+	req, _ = http.NewRequest(http.MethodGet, webClientDownloadPath+"?path="+url.QueryEscape("/")+"&files=notalist", nil)
+	setJWTCookieForReq(req, webToken)
+	rr = executeRequest(req)
+	checkResponseCode(t, http.StatusInternalServerError, rr)
+	assert.Contains(t, rr.Body.String(), "Unable to get files list")
+
 	req, _ = http.NewRequest(http.MethodGet, webClientDirContentsPath+"?path=/", nil)
 	setJWTCookieForReq(req, webToken)
 	rr = executeRequest(req)
@@ -5102,6 +5127,28 @@ func TestWebGetFiles(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestCompressionErrorMock(t *testing.T) {
+	user, _, err := httpdtest.AddUser(getTestUser(), http.StatusCreated)
+	assert.NoError(t, err)
+
+	defer func() {
+		rcv := recover()
+		assert.Equal(t, http.ErrAbortHandler, rcv)
+		_, err := httpdtest.RemoveUser(user, http.StatusOK)
+		assert.NoError(t, err)
+		err = os.RemoveAll(user.GetHomeDir())
+		assert.NoError(t, err)
+	}()
+
+	webToken, err := getJWTWebClientTokenFromTestServer(defaultUsername, defaultPassword)
+	assert.NoError(t, err)
+
+	req, _ := http.NewRequest(http.MethodGet, webClientDownloadPath+"?path="+url.QueryEscape("/")+"&files="+
+		url.QueryEscape(`["missing"]`), nil)
+	setJWTCookieForReq(req, webToken)
+	executeRequest(req)
+}
+
 func TestGetFilesSFTPBackend(t *testing.T) {
 	user, _, err := httpdtest.AddUser(getTestUser(), http.StatusCreated)
 	assert.NoError(t, err)

httpd/internal_test.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/jwtauth/v5"
+	"github.com/klauspost/compress/zip"
 	"github.com/lestrrat-go/jwx/jwa"
 	"github.com/lestrrat-go/jwx/jwt"
 	"github.com/rs/xid"
@@ -263,6 +264,19 @@ xr5cb9VBRBtB9aOKVfuRhpatAfS2Pzm2Htae9lFn7slGPUmu2hkjDw==
 -----END RSA PRIVATE KEY-----`
 )
 
+type failingWriter struct {
+}
+
+func (r *failingWriter) Write(p []byte) (n int, err error) {
+	return 0, errors.New("write error")
+}
+
+func (r *failingWriter) WriteHeader(statusCode int) {}
+
+func (r *failingWriter) Header() http.Header {
+	return make(http.Header)
+}
+
 func TestShouldBind(t *testing.T) {
 	c := Conf{
 		Bindings: []Binding{
@@ -1088,6 +1102,119 @@ func TestProxyHeaders(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestRecoverer(t *testing.T) {
+	recoveryPath := "/recovery"
+	b := Binding{
+		Address:         "",
+		Port:            8080,
+		EnableWebAdmin:  true,
+		EnableWebClient: false,
+	}
+	server := newHttpdServer(b, "../static")
+	server.initializeRouter()
+	server.router.Get(recoveryPath, func(w http.ResponseWriter, r *http.Request) {
+		panic("panic")
+	})
+	testServer := httptest.NewServer(server.router)
+	defer testServer.Close()
+
+	req, err := http.NewRequest(http.MethodGet, recoveryPath, nil)
+	assert.NoError(t, err)
+	rr := httptest.NewRecorder()
+	testServer.Config.Handler.ServeHTTP(rr, req)
+	assert.Equal(t, http.StatusInternalServerError, rr.Code, rr.Body.String())
+
+	server.router = chi.NewRouter()
+	server.router.Use(recoverer)
+	server.router.Get(recoveryPath, func(w http.ResponseWriter, r *http.Request) {
+		panic("panic")
+	})
+	testServer = httptest.NewServer(server.router)
+	defer testServer.Close()
+
+	req, err = http.NewRequest(http.MethodGet, recoveryPath, nil)
+	assert.NoError(t, err)
+	rr = httptest.NewRecorder()
+	testServer.Config.Handler.ServeHTTP(rr, req)
+	assert.Equal(t, http.StatusInternalServerError, rr.Code, rr.Body.String())
+}
+
+func TestCompressorAbortHandler(t *testing.T) {
+	defer func() {
+		rcv := recover()
+		assert.Equal(t, http.ErrAbortHandler, rcv)
+	}()
+
+	connection := &Connection{
+		BaseConnection: common.NewBaseConnection(xid.New().String(), common.ProtocolHTTP, dataprovider.User{}),
+		request:        nil,
+	}
+	renderCompressedFiles(&failingWriter{}, connection, "", nil)
+}
+
+func TestZipErrors(t *testing.T) {
+	user := dataprovider.User{
+		HomeDir: filepath.Clean(os.TempDir()),
+	}
+	user.Permissions = make(map[string][]string)
+	user.Permissions["/"] = []string{dataprovider.PermAny}
+	connection := &Connection{
+		BaseConnection: common.NewBaseConnection(xid.New().String(), common.ProtocolHTTP, user),
+		request:        nil,
+	}
+
+	testDir := filepath.Join(os.TempDir(), "testDir")
+	err := os.MkdirAll(testDir, os.ModePerm)
+	assert.NoError(t, err)
+
+	wr := zip.NewWriter(&failingWriter{})
+	err = wr.Close()
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "write error")
+	}
+
+	err = addZipEntry(wr, connection, "/"+filepath.Base(testDir), "/")
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "write error")
+	}
+
+	testFilePath := filepath.Join(testDir, "ziptest.zip")
+	err = os.WriteFile(testFilePath, utils.GenerateRandomBytes(65535), os.ModePerm)
+	assert.NoError(t, err)
+	err = addZipEntry(wr, connection, path.Join("/", filepath.Base(testDir), filepath.Base(testFilePath)),
+		"/"+filepath.Base(testDir))
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "write error")
+	}
+
+	connection.User.Permissions["/"] = []string{dataprovider.PermListItems}
+	err = addZipEntry(wr, connection, path.Join("/", filepath.Base(testDir), filepath.Base(testFilePath)),
+		"/"+filepath.Base(testDir))
+	assert.ErrorIs(t, err, os.ErrPermission)
+
+	// creating a virtual folder to a missing path stat is ok but readdir fails
+	user.VirtualFolders = append(user.VirtualFolders, vfs.VirtualFolder{
+		BaseVirtualFolder: vfs.BaseVirtualFolder{
+			MappedPath: filepath.Join(os.TempDir(), "mapped"),
+		},
+		VirtualPath: "/vpath",
+	})
+	connection.User = user
+	wr = zip.NewWriter(bytes.NewBuffer(make([]byte, 0)))
+	err = addZipEntry(wr, connection, user.VirtualFolders[0].VirtualPath, "/")
+	assert.Error(t, err)
+
+	user.Filters.FilePatterns = append(user.Filters.FilePatterns, dataprovider.PatternsFilter{
+		Path:           "/",
+		DeniedPatterns: []string{"*.zip"},
+	})
+	err = addZipEntry(wr, connection, "/"+filepath.Base(testDir), "/")
+	assert.ErrorIs(t, err, os.ErrPermission)
+
+	err = os.RemoveAll(testDir)
+	assert.NoError(t, err)
+}
+
 func TestWebAdminRedirect(t *testing.T) {
 	b := Binding{
 		Address:         "",
@@ -1312,6 +1439,8 @@ func TestHTTPDFile(t *testing.T) {
 	assert.Error(t, err)
 	err = httpdFile.Close()
 	assert.ErrorIs(t, err, common.ErrTransferClosed)
+	err = os.Remove(p)
+	assert.NoError(t, err)
 }
 
 func TestChangeUserPwd(t *testing.T) {
@@ -1359,6 +1488,13 @@ func TestGetFilesInvalidClaims(t *testing.T) {
 	handleClientGetDirContents(rr, req)
 	assert.Equal(t, http.StatusForbidden, rr.Code)
 	assert.Contains(t, rr.Body.String(), "invalid token claims")
+
+	rr = httptest.NewRecorder()
+	req, _ = http.NewRequest(http.MethodGet, webClientDownloadPath, nil)
+	req.Header.Set("Cookie", fmt.Sprintf("jwt=%v", token["access_token"]))
+	handleWebClientDownload(rr, req)
+	assert.Equal(t, http.StatusForbidden, rr.Code)
+	assert.Contains(t, rr.Body.String(), "Invalid token claims")
 }
 
 func TestManageKeysInvalidClaims(t *testing.T) {

httpd/middleware.go

@@ -3,7 +3,9 @@ package httpd
 import (
 	"errors"
 	"net/http"
+	"runtime/debug"
 
+	"github.com/go-chi/chi/v5/middleware"
 	"github.com/go-chi/jwtauth/v5"
 	"github.com/lestrrat-go/jwx/jwt"
 
@@ -177,3 +179,28 @@ func verifyCSRFHeader(next http.Handler) http.Handler {
 		next.ServeHTTP(w, r)
 	})
 }
+
+func recoverer(next http.Handler) http.Handler {
+	fn := func(w http.ResponseWriter, r *http.Request) {
+		defer func() {
+			if rvr := recover(); rvr != nil {
+				if rvr == http.ErrAbortHandler {
+					panic(rvr)
+				}
+
+				logEntry := middleware.GetLogEntry(r)
+				if logEntry != nil {
+					logEntry.Panic(rvr, debug.Stack())
+				} else {
+					middleware.PrintPrettyStack(rvr)
+				}
+
+				w.WriteHeader(http.StatusInternalServerError)
+			}
+		}()
+
+		next.ServeHTTP(w, r)
+	}
+
+	return http.HandlerFunc(fn)
+}

httpd/server.go

@@ -462,7 +462,7 @@ func (s *httpdServer) initializeRouter() {
 
 	s.router.Use(middleware.RequestID)
 	s.router.Use(logger.NewStructuredLogger(logger.GetLogger()))
-	s.router.Use(middleware.Recoverer)
+	s.router.Use(recoverer)
 	s.router.Use(s.checkConnection)
 	s.router.Use(middleware.GetHead)
 	s.router.Use(middleware.StripSlashes)
@@ -574,6 +574,7 @@ func (s *httpdServer) initializeRouter() {
 			router.Get(webClientLogoutPath, handleWebClientLogout)
 			router.With(s.refreshCookie).Get(webClientFilesPath, handleClientGetFiles)
 			router.With(compressor.Handler, s.refreshCookie).Get(webClientDirContentsPath, handleClientGetDirContents)
+			router.With(s.refreshCookie).Get(webClientDownloadPath, handleWebClientDownload)
 			router.With(s.refreshCookie).Get(webClientCredentialsPath, handleClientGetCredentials)
 			router.Post(webChangeClientPwdPath, handleWebClientChangePwdPost)
 			router.With(checkClientPerm(dataprovider.WebClientPubKeyChangeDisabled)).

httpd/webclient.go

@@ -1,6 +1,7 @@
 package httpd
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"html/template"
@@ -79,11 +80,12 @@ type dirMapping struct {
 
 type filesPage struct {
 	baseClientPage
-	CurrentDir string
-	ReadDirURL string
-	Files      []os.FileInfo
-	Error      string
-	Paths      []dirMapping
+	CurrentDir  string
+	ReadDirURL  string
+	DownloadURL string
+	Files       []os.FileInfo
+	Error       string
+	Paths       []dirMapping
 }
 
 type clientMessagePage struct {
@@ -219,6 +221,7 @@ func renderFilesPage(w http.ResponseWriter, r *http.Request, files []os.FileInfo
 		Files:          files,
 		Error:          error,
 		CurrentDir:     url.QueryEscape(dirName),
+		DownloadURL:    webClientDownloadPath,
 		ReadDirURL:     webClientDirContentsPath,
 	}
 	paths := []dirMapping{}
@@ -269,6 +272,43 @@ func handleWebClientLogout(w http.ResponseWriter, r *http.Request) {
 	http.Redirect(w, r, webClientLoginPath, http.StatusFound)
 }
 
+func handleWebClientDownload(w http.ResponseWriter, r *http.Request) {
+	claims, err := getTokenClaims(r)
+	if err != nil || claims.Username == "" {
+		renderClientMessagePage(w, r, "Invalid token claims", "", http.StatusForbidden, nil, "")
+		return
+	}
+
+	user, err := dataprovider.UserExists(claims.Username)
+	if err != nil {
+		renderClientMessagePage(w, r, "Unable to retrieve your user", "", http.StatusInternalServerError, nil, "")
+		return
+	}
+
+	connection := &Connection{
+		BaseConnection: common.NewBaseConnection(xid.New().String(), common.ProtocolHTTP, user),
+		request:        r,
+	}
+	common.Connections.Add(connection)
+	defer common.Connections.Remove(connection.GetID())
+
+	name := "/"
+	if _, ok := r.URL.Query()["path"]; ok {
+		name = utils.CleanPath(r.URL.Query().Get("path"))
+	}
+
+	files := r.URL.Query().Get("files")
+	var filesList []string
+	err = json.Unmarshal([]byte(files), &filesList)
+	if err != nil {
+		renderClientMessagePage(w, r, "Unable to get files list", "", http.StatusInternalServerError, err, "")
+		return
+	}
+
+	w.Header().Set("Content-Disposition", "attachment; filename=\"sftpgo-download.zip\"")
+	renderCompressedFiles(w, connection, name, filesList)
+}
+
 func handleClientGetDirContents(w http.ResponseWriter, r *http.Request) {
 	claims, err := getTokenClaims(r)
 	if err != nil || claims.Username == "" {

sftpd/ssh_cmd.go

@@ -174,7 +174,11 @@ func (c *sshCommand) handleSFTPGoCopy() error {
 		return c.sendErrorResponse(err)
 	}
 	c.connection.Log(logger.LevelDebug, "start copy %#v -> %#v", fsSourcePath, fsDestPath)
-	err = fscopy.Copy(fsSourcePath, fsDestPath)
+	err = fscopy.Copy(fsSourcePath, fsDestPath, fscopy.Options{
+		OnSymlink: func(src string) fscopy.SymlinkAction {
+			return fscopy.Skip
+		},
+	})
 	if err != nil {
 		return c.sendErrorResponse(c.connection.GetFsError(fsSrc, err))
 	}

templates/webclient/files.html

@@ -7,7 +7,13 @@
 <link href="{{.StaticURL}}/vendor/datatables/buttons.bootstrap4.min.css" rel="stylesheet">
 <link href="{{.StaticURL}}/vendor/datatables/fixedHeader.bootstrap4.min.css" rel="stylesheet">
 <link href="{{.StaticURL}}/vendor/datatables/responsive.bootstrap4.min.css" rel="stylesheet">
-<link href="{{.StaticURL}}/vendor/datatables/select.bootstrap4.min.css" rel="stylesheet">
+<link href="{{.StaticURL}}/vendor/datatables/dataTables.checkboxes.css" rel="stylesheet">
+<style>
+    div.dataTables_wrapper span.selected-info,
+    div.dataTables_wrapper span.selected-item {
+        margin-left: 0.5em;
+    }
+</style>
 {{end}}
 
 {{define "page_body"}}
@@ -29,6 +35,7 @@
             <table class="table table-hover nowrap" id="dataTable" width="100%" cellspacing="0">
                 <thead>
                     <tr>
+                        <th></th>
                         <th>Type</th>
                         <th>Name</th>
                         <th>Size</th>
@@ -49,97 +56,108 @@
 <script src="{{.StaticURL}}/vendor/datatables/dataTables.fixedHeader.min.js"></script>
 <script src="{{.StaticURL}}/vendor/datatables/dataTables.responsive.min.js"></script>
 <script src="{{.StaticURL}}/vendor/datatables/responsive.bootstrap4.min.js"></script>
-<script src="{{.StaticURL}}/vendor/datatables/dataTables.select.min.js"></script>
+<script src="{{.StaticURL}}/vendor/datatables/dataTables.checkboxes.min.js"></script>
 <script type="text/javascript">
 
     function getIconForFile(filename) {
-            var extension = filename.slice((filename.lastIndexOf(".") - 1 >>> 0) + 2).toLowerCase();
-
-            switch (extension) {
-                case "doc":
-                case "docx":
-                case "odt":
-                    return "far fa-file-word";
-                case "ppt":
-                case "pptx":
-                    return "far fa-file-powerpoint";
-                case "xls":
-                case "xlsx":
-                case "ods":
-                    return "far fa-file-excel";
-                case "pdf":
-                    return "far fa-file-pdf";
-                case "webm":
-                case "mkv":
-                case "flv":
-                case "vob":
-                case "ogv":
-                case "ogg":
-                case "avi":
-                case "ts":
-                case "mov":
-                case "wmv":
-                case "asf":
-                case "mpeg":
-                case "mpv":
-                case "3gp":
-                    return "far fa-file-video";
-                case "jpeg":
-                case "jpg":
-                case "png":
-                case "gif":
-                case "webp":
-                case "tiff":
-                case "psd":
-                case "bmp":
-                case "svg":
-                case "jp2":
-                    return "far fa-file-image";
-                case "go":
-                case "sh":
-                case "java":
-                case "php":
-                case "cs":
-                case "asp":
-                case "aspx":
-                case "css":
-                case "html":
-                case "js":
-                case "py":
-                case "rb":
-                case "cgi":
-                case "c":
-                case "cpp":
-                case "h":
-                case "hpp":
-                case "kt":
-                case "ktm":
-                case "kts":
-                case "swift":
-                case "r":
-                    return "far fa-file-code";
-                case "zip":
-                case "rar":
-                case "tar":
-                case "gz":
-                case "bz2":
-                case "zstd":
-                case "zst":
-                case "sz":
-                case "lz":
-                case "lz4":
-                case "xz":
-                    return "far fa-file-archive";
-                case "txt":
-                case "json":
-                case "yaml":
-                case "toml":
-                    return "far fa-file-alt";
-                default:
-                    return "far fa-file";
-            }
+        var extension = filename.slice((filename.lastIndexOf(".") - 1 >>> 0) + 2).toLowerCase();
+        switch (extension) {
+            case "doc":
+            case "docx":
+            case "odt":
+            case "wps":
+                return "far fa-file-word";
+            case "ppt":
+            case "pptx":
+                return "far fa-file-powerpoint";
+            case "xls":
+            case "xlsx":
+            case "ods":
+                return "far fa-file-excel";
+            case "pdf":
+                return "far fa-file-pdf";
+            case "webm":
+            case "mkv":
+            case "flv":
+            case "vob":
+            case "ogv":
+            case "ogg":
+            case "avi":
+            case "ts":
+            case "mov":
+            case "wmv":
+            case "asf":
+            case "mpeg":
+            case "mpv":
+            case "3gp":
+                return "far fa-file-video";
+            case "jpeg":
+            case "jpg":
+            case "png":
+            case "gif":
+            case "webp":
+            case "tiff":
+            case "psd":
+            case "bmp":
+            case "svg":
+            case "jp2":
+                return "far fa-file-image";
+            case "go":
+            case "sh":
+            case "bat":
+            case "java":
+            case "php":
+            case "cs":
+            case "asp":
+            case "aspx":
+            case "css":
+            case "html":
+            case "xhtml":
+            case "htm":
+            case "js":
+            case "jsp":
+            case "py":
+            case "rb":
+            case "cgi":
+            case "c":
+            case "cpp":
+            case "h":
+            case "hpp":
+            case "kt":
+            case "ktm":
+            case "kts":
+            case "swift":
+            case "r":
+                return "far fa-file-code";
+            case "zip":
+            case "zipx":
+            case "rar":
+            case "tar":
+            case "gz":
+            case "bz2":
+            case "zstd":
+            case "zst":
+            case "sz":
+            case "lz":
+            case "lz4":
+            case "xz":
+            case "jar":
+                return "far fa-file-archive";
+            case "txt":
+            case "rtf":
+            case "json":
+            case "xml":
+            case "yaml":
+            case "toml":
+            case "log":
+            case "csv":
+            case "ini":
+            case "cfg":
+                return "far fa-file-alt";
+            default:
+                return "far fa-file";
         }
-
+    }
 
     $(document).ready(function () {
         $.fn.dataTable.ext.buttons.refresh = {
@@ -151,6 +169,24 @@
             }
         };
 
+        $.fn.dataTable.ext.buttons.download = {
+            text: '<i class="fas fa-download"></i>',
+            name: 'download',
+            titleAttr: "Download",
+            action: function (e, dt, node, config) {
+                var filesArray = [];
+                var selected = dt.column(0).checkboxes.selected();
+                for (i = 0; i < selected.length; i++) {
+                    filesArray.push(selected[i]);
+                }
+                var files = fixedEncodeURIComponent(JSON.stringify(filesArray));
+                var downloadURL = '{{.DownloadURL}}';
+                var currentDir = '{{.CurrentDir}}';
+                window.location = `${downloadURL}?path=${currentDir}&files=${files}`;
+            },
+            enabled: false
+        };
+
         var table = $('#dataTable').DataTable({
             "ajax": {
                 "url": "{{.ReadDirURL}}?path={{.CurrentDir}}",
@@ -174,21 +210,20 @@
             "deferRender": true,
             "processing": true,
             "columns": [
+                { "data": "name" },
                 { "data": "type" },
                 {
                     "data": "name",
-                    "render": function(data, type, row){
+                    "render": function (data, type, row) {
                         if (type === 'display') {
-                            if (row["type"] == "1"){
+                            if (row["type"] == "1") {
                                 return `<i class="fas fa-folder"></i>&nbsp;<a href="${row['url']}">${data}</a>`;
-                            } else {
-                                if (row["size"] == ""){
-                                    return `<i class="fas fa-external-link-alt"></i>&nbsp;<a href="${row['url']}">${data}</a>`;
-                                } else {
-                                    var icon = getIconForFile(data);
-                                    return `<i class="${icon}"></i>&nbsp;<a href="${row['url']}">${data}</a>`;
-                                }
                             }
+                            if (row["size"] == "") {
+                                return `<i class="fas fa-external-link-alt"></i>&nbsp;<a href="${row['url']}">${data}</a>`;
+                            }
+                            var icon = getIconForFile(data);
+                            return `<i class="${icon}"></i>&nbsp;<a href="${row['url']}">${data}</a>`;
                         }
                         return data;
                     }
@@ -201,11 +236,30 @@
             "columnDefs": [
                 {
                     "targets": [0],
+                    "checkboxes": {
+                        "selectCallback": function (nodes, selected) {
+                            var selectedItems = table.column(0).checkboxes.selected().length;
+                            var selectedText = "";
+                            if (selectedItems == 1) {
+                                selectedText = "1 item selected";
+                            } else if (selectedItems > 1) {
+                                selectedText = `${selectedItems} items selected`;
+                            }
+                            table.button('download:name').enable(selectedItems > 0);
+                            $('#dataTable_info').find('span').remove();
+                            $("#dataTable_info").append('<span class="selected-info"><span class="selected-item">' + selectedText + '</span></span>');
+                        }
+                    },
+                    "orderable": false,
+                    "searchable": false
+                },
+                {
+                    "targets": [1],
                     "visible": false,
                     "searchable": false
                 },
                 {
-                    "targets": [2,3],
+                    "targets": [3, 4],
                     "searchable": false
                 }
             ],
@@ -217,32 +271,19 @@
                 "loadingRecords": "",
                 "emptyTable": "No files or folders"
             },
-            /*"select": {
-                "style":    'single',
-                "blurable": true
-            },*/
-            "initComplete": function(settings, json) {
-                table.button().add(0,'refresh');
-                table.button().add(0,'pageLength');
+            "initComplete": function (settings, json) {
+                table.button().add(0, 'refresh');
+                table.button().add(0, 'pageLength');
+                table.button().add(0, 'download');
                 table.buttons().container().appendTo('#dataTable_wrapper .col-md-6:eq(0)');
             },
-            "orderFixed": [ 0, 'asc' ],
-            "order": [[1, 'asc']]
+            "orderFixed": [1, 'asc'],
+            "order": [[2, 'asc']]
         });
 
-        new $.fn.dataTable.FixedHeader( table );
-
+        new $.fn.dataTable.FixedHeader(table);
         $.fn.dataTable.ext.errMode = 'none';
 
-        /*table.on('select', function (e, dt, type, indexes) {
-            if (type === 'row') {
-                var rows = table.rows(indexes).nodes().to$();
-                $.each(rows, function() {
-                    if ($(this).hasClass('ignoreselection')) table.row($(this)).deselect();
-                })
-            }
-        });*/
-
     });
 </script>
 

utils/utils.go

@@ -40,8 +40,10 @@ const (
 )
 
 var (
-	xForwardedFor = http.CanonicalHeaderKey("X-Forwarded-For")
-	xRealIP       = http.CanonicalHeaderKey("X-Real-IP")
+	xForwardedFor  = http.CanonicalHeaderKey("X-Forwarded-For")
+	xRealIP        = http.CanonicalHeaderKey("X-Real-IP")
+	cfConnectingIP = http.CanonicalHeaderKey("CF-Connecting-IP")
+	trueClientIP   = http.CanonicalHeaderKey("True-Client-IP")
 )
 
 // IsStringInSlice searches a string in a slice and returns true if the string is found
@@ -530,6 +532,10 @@ func GetRealIP(r *http.Request) string {
 
 	if xrip := r.Header.Get(xRealIP); xrip != "" {
 		ip = xrip
+	} else if clientIP := r.Header.Get(trueClientIP); clientIP != "" {
+		ip = clientIP
+	} else if clientIP := r.Header.Get(cfConnectingIP); clientIP != "" {
+		ip = clientIP
 	} else if xff := r.Header.Get(xForwardedFor); xff != "" {
 		i := strings.Index(xff, ", ")
 		if i == -1 {

vfs/osfs.go

@@ -104,7 +104,11 @@ func (fs *OsFs) Rename(source, target string) error {
 	if err != nil && isCrossDeviceError(err) {
 		fsLog(fs, logger.LevelWarn, "cross device error detected while renaming %#v -> %#v. Trying a copy and remove, this could take a long time",
 			source, target)
-		err = fscopy.Copy(source, target)
+		err = fscopy.Copy(source, target, fscopy.Options{
+			OnSymlink: func(src string) fscopy.SymlinkAction {
+				return fscopy.Skip
+			},
+		})
 		if err != nil {
 			fsLog(fs, logger.LevelDebug, "cross device copy error: %v", err)
 			return err

webdavd/internal_test.go

@@ -432,10 +432,18 @@ func TestRemoteAddress(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Empty(t, req.RemoteAddr)
 
-	req.Header.Set("X-Forwarded-For", remoteAddr1)
+	req.Header.Set("True-Client-IP", remoteAddr1)
 	ip := utils.GetRealIP(req)
 	assert.Equal(t, remoteAddr1, ip)
-	// this will be ignore, remoteAddr1 is not allowed to se this header
+	req.Header.Del("True-Client-IP")
+	req.Header.Set("CF-Connecting-IP", remoteAddr1)
+	ip = utils.GetRealIP(req)
+	assert.Equal(t, remoteAddr1, ip)
+	req.Header.Del("CF-Connecting-IP")
+	req.Header.Set("X-Forwarded-For", remoteAddr1)
+	ip = utils.GetRealIP(req)
+	assert.Equal(t, remoteAddr1, ip)
+	// this will be ignored, remoteAddr1 is not allowed to se this header
 	req.Header.Set("X-Forwarded-For", remoteAddr2)
 	req.RemoteAddr = remoteAddr1
 	ip = server.checkRemoteAddress(req)