update deps and backport some fixes from main branch

.github/workflows/release.yml

@@ -5,7 +5,7 @@ on:
     tags: 'v*'
 
 env:
-  GO_VERSION: 1.16.6
+  GO_VERSION: 1.16.7
 
 jobs:
   prepare-sources-with-deps:

cmd/portable.go

@@ -1,3 +1,4 @@
+//go:build !noportable
 // +build !noportable
 
 package cmd

cmd/portable_disabled.go

@@ -1,3 +1,4 @@
+//go:build noportable
 // +build noportable
 
 package cmd

common/common.go

@@ -409,9 +409,8 @@ func (c *Configuration) IsAtomicUploadEnabled() bool {
 }
 
 // GetProxyListener returns a wrapper for the given listener that supports the
-// HAProxy Proxy Protocol or nil if the proxy protocol is not configured
+// HAProxy Proxy Protocol
 func (c *Configuration) GetProxyListener(listener net.Listener) (*proxyproto.Listener, error) {
-	var proxyListener *proxyproto.Listener
 	var err error
 	if c.ProxyProtocol > 0 {
 		var policyFunc func(upstream net.Addr) (proxyproto.Policy, error)
@@ -433,12 +432,13 @@ func (c *Configuration) GetProxyListener(listener net.Listener) (*proxyproto.Lis
 				}
 			}
 		}
-		proxyListener = &proxyproto.Listener{
+		return &proxyproto.Listener{
 			Listener:          listener,
 			Policy:            policyFunc,
+			ReadHeaderTimeout: 5 * time.Second,
+		}, nil
 	}
-	}
-	return proxyListener, nil
+	return nil, errors.New("proxy protocol not configured")
 }
 
 // ExecuteStartupHook runs the startup hook if defined

config/config_linux.go

@@ -1,3 +1,4 @@
+//go:build linux
 // +build linux
 
 package config

config/config_nolinux.go

@@ -1,3 +1,4 @@
+//go:build !linux
 // +build !linux
 
 package config

dataprovider/bolt.go

@@ -1,3 +1,4 @@
+//go:build !nobolt
 // +build !nobolt
 
 package dataprovider

dataprovider/bolt_disabled.go

@@ -1,3 +1,4 @@
+//go:build nobolt
 // +build nobolt
 
 package dataprovider

dataprovider/mysql.go

@@ -1,3 +1,4 @@
+//go:build !nomysql
 // +build !nomysql
 
 package dataprovider

dataprovider/mysql_disabled.go

@@ -1,3 +1,4 @@
+//go:build nomysql
 // +build nomysql
 
 package dataprovider

dataprovider/pgsql.go

@@ -1,3 +1,4 @@
+//go:build !nopgsql
 // +build !nopgsql
 
 package dataprovider

dataprovider/pgsql_disabled.go

@@ -1,3 +1,4 @@
+//go:build nopgsql
 // +build nopgsql
 
 package dataprovider

dataprovider/sqlite.go

@@ -1,3 +1,4 @@
+//go:build !nosqlite
 // +build !nosqlite
 
 package dataprovider

dataprovider/sqlite_disabled.go

@@ -1,3 +1,4 @@
+//go:build nosqlite
 // +build nosqlite
 
 package dataprovider

ftpd/ftpd_test.go

@@ -2275,13 +2275,13 @@ func TestActiveModeDisabled(t *testing.T) {
 	if assert.NoError(t, err) {
 		code, response, err := client.SendCustomCommand("PORT 10,2,0,2,4,31")
 		assert.NoError(t, err)
-		assert.Equal(t, ftp.StatusCommandOK, code)
-		assert.Equal(t, "PORT command successful", response)
+		assert.Equal(t, ftp.StatusBadArguments, code)
+		assert.Equal(t, "Your request does not meet the configured security requirements", response)
 
 		code, response, err = client.SendCustomCommand("EPRT |1|132.235.1.2|6275|")
 		assert.NoError(t, err)
-		assert.Equal(t, ftp.StatusCommandOK, code)
-		assert.Equal(t, "EPRT command successful", response)
+		assert.Equal(t, ftp.StatusBadArguments, code)
+		assert.Equal(t, "Your request does not meet the configured security requirements", response)
 
 		err = client.Quit()
 		assert.NoError(t, err)

ftpd/internal_test.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/eikenb/pipeat"
+	ftpserver "github.com/fclairamb/ftpserverlib"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -250,6 +251,7 @@ xr5cb9VBRBtB9aOKVfuRhpatAfS2Pzm2Htae9lFn7slGPUmu2hkjDw==
 )
 
 type mockFTPClientContext struct {
+	lastDataChannel ftpserver.DataChannel
 }
 
 func (cc mockFTPClientContext) Path() string {
@@ -294,6 +296,10 @@ func (cc mockFTPClientContext) GetLastCommand() string {
 	return ""
 }
 
+func (cc mockFTPClientContext) GetLastDataChannel() ftpserver.DataChannel {
+	return cc.lastDataChannel
+}
+
 // MockOsFs mockable OsFs
 type MockOsFs struct {
 	vfs.Fs

ftpd/server.go

@@ -94,7 +94,7 @@ func (s *Server) GetSettings() (*ftpserver.Settings, error) {
 		}
 	}
 	var ftpListener net.Listener
-	if common.Config.ProxyProtocol > 0 && s.binding.ApplyProxyConfig {
+	if s.binding.HasProxy() {
 		listener, err := net.Listen("tcp", s.binding.GetAddress())
 		if err != nil {
 			logger.Warn(logSender, "", "error starting listener on address %v: %v", s.binding.GetAddress(), err)
@@ -105,6 +105,9 @@ func (s *Server) GetSettings() (*ftpserver.Settings, error) {
 			logger.Warn(logSender, "", "error enabling proxy listener: %v", err)
 			return nil, err
 		}
+		if s.binding.TLSMode == 2 && s.tlsConfig != nil {
+			ftpListener = tls.NewListener(ftpListener, s.tlsConfig)
+		}
 	}
 
 	if s.binding.TLSMode < 0 || s.binding.TLSMode > 2 {
@@ -197,6 +200,14 @@ func (s *Server) AuthUser(cc ftpserver.ClientContext, username, password string)
 	return connection, nil
 }
 
+// WrapPassiveListener implements the MainDriverExtensionPassiveWrapper interface
+func (s *Server) WrapPassiveListener(listener net.Listener) (net.Listener, error) {
+	if s.binding.HasProxy() {
+		return common.Config.GetProxyListener(listener)
+	}
+	return listener, nil
+}
+
 // VerifyConnection checks whether a user should be authenticated using a client certificate without prompting for a password
 func (s *Server) VerifyConnection(cc ftpserver.ClientContext, user string, tlsConn *tls.Conn) (ftpserver.ClientDriver, error) {
 	if !s.binding.isMutualTLSEnabled() {

go.mod

@@ -3,43 +3,40 @@ module github.com/drakkan/sftpgo
 go 1.16
 
 require (
-	cloud.google.com/go/storage v1.16.0
+	cloud.google.com/go/storage v1.16.1
 	github.com/Azure/azure-storage-blob-go v0.14.0
 	github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962
-	github.com/StackExchange/wmi v1.2.0 // indirect
 	github.com/alexedwards/argon2id v0.0.0-20210511081203-7d35d68092b8
-	github.com/aws/aws-sdk-go v1.40.1
+	github.com/aws/aws-sdk-go v1.40.39
 	github.com/cockroachdb/cockroach-go/v2 v2.1.1
 	github.com/eikenb/pipeat v0.0.0-20210603033007-44fc3ffce52b
-	github.com/fclairamb/ftpserverlib v0.14.0
-	github.com/frankban/quicktest v1.13.0 // indirect
-	github.com/go-chi/chi/v5 v5.0.3
+	github.com/fclairamb/ftpserverlib v0.15.0
+	github.com/frankban/quicktest v1.13.1 // indirect
+	github.com/go-chi/chi/v5 v5.0.4
 	github.com/go-chi/jwtauth/v5 v5.0.1
 	github.com/go-chi/render v1.0.1
-	github.com/go-ole/go-ole v1.2.5 // indirect
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/grandcat/zeroconf v1.0.0
 	github.com/hashicorp/go-retryablehttp v0.7.0
 	github.com/jlaffaye/ftp v0.0.0-20201112195030-9aae4d151126
-	github.com/klauspost/compress v1.13.1
-	github.com/klauspost/cpuid/v2 v2.0.8 // indirect
-	github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect
-	github.com/lestrrat-go/jwx v1.2.4
-	github.com/lib/pq v1.10.2
+	github.com/klauspost/compress v1.13.5
+	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/lestrrat-go/jwx v1.2.6
+	github.com/lib/pq v1.10.3
 	github.com/mattn/go-sqlite3 v1.14.8
 	github.com/miekg/dns v1.1.43 // indirect
 	github.com/minio/sio v0.3.0
 	github.com/otiai10/copy v1.6.0
-	github.com/pires/go-proxyproto v0.6.0
-	github.com/pkg/sftp v1.13.2
+	github.com/pires/go-proxyproto v0.6.1
+	github.com/pkg/sftp v1.13.3
 	github.com/prometheus/client_golang v1.11.0
-	github.com/prometheus/common v0.29.0 // indirect
+	github.com/prometheus/common v0.30.0 // indirect
 	github.com/rs/cors v1.8.0
 	github.com/rs/xid v1.3.0
-	github.com/rs/zerolog v1.23.0
+	github.com/rs/zerolog v1.25.0
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/shirou/gopsutil/v3 v3.21.6
+	github.com/shirou/gopsutil/v3 v3.21.8
 	github.com/spf13/afero v1.6.0
 	github.com/spf13/cobra v1.2.1
 	github.com/spf13/viper v1.8.1
@@ -48,19 +45,21 @@ require (
 	github.com/yl2chen/cidranger v1.0.2
 	go.etcd.io/bbolt v1.3.6
 	go.uber.org/automaxprocs v1.4.0
-	gocloud.dev v0.23.0
-	gocloud.dev/secrets/hashivault v0.23.0
-	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a
-	golang.org/x/net v0.0.0-20210614182718-04defd469f4e
-	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c
-	golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6
-	google.golang.org/api v0.50.0
-	google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea // indirect
+	gocloud.dev v0.24.0
+	gocloud.dev/secrets/hashivault v0.24.0
+	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
+	golang.org/x/net v0.0.0-20210907225631-ff17edfbf26d
+	golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365
+	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac
+	google.golang.org/api v0.56.0
+	google.golang.org/genproto v0.0.0-20210909144509-af19c3c38cb7 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )
 
 replace (
+	github.com/eikenb/pipeat => github.com/drakkan/pipeat v0.0.0-20210805162858-70e57fa8a639
+	github.com/fclairamb/ftpserverlib => github.com/drakkan/ftpserverlib v0.0.0-20210805132427-425f32d9dc15
 	github.com/jlaffaye/ftp => github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9
-	golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20210515063737-edf1d3b63536
-	golang.org/x/net => github.com/drakkan/net v0.0.0-20210615043241-a7f9e02422df
+	golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20210908103413-a132997f748f
+	golang.org/x/net => github.com/drakkan/net v0.0.0-20210908102438-2debf45fec0b
 )

kms/aws.go

@@ -1,3 +1,4 @@
+//go:build !noawskms
 // +build !noawskms
 
 package kms

kms/aws_disabled.go

@@ -1,3 +1,4 @@
+//go:build noawskms
 // +build noawskms
 
 package kms

kms/gcp.go

@@ -1,3 +1,4 @@
+//go:build !nogcpkms
 // +build !nogcpkms
 
 package kms

kms/gcp_disabled.go

@@ -1,3 +1,4 @@
+//go:build nogcpkms
 // +build nogcpkms
 
 package kms

kms/vault.go

@@ -1,3 +1,4 @@
+//go:build !novaultkms
 // +build !novaultkms
 
 package kms

kms/vault_disabled.go

@@ -1,3 +1,4 @@
+//go:build novaultkms
 // +build novaultkms
 
 package kms

logger/journald.go

@@ -1,3 +1,4 @@
+//go:build linux
 // +build linux
 
 package logger

logger/journald_nolinux.go

@@ -1,3 +1,4 @@
+//go:build !linux
 // +build !linux
 
 package logger

metrics/metrics.go

@@ -1,3 +1,4 @@
+//go:build !nometrics
 // +build !nometrics
 
 // Package metrics provides Prometheus metrics support

service/service_portable.go

@@ -1,3 +1,4 @@
+//go:build !noportable
 // +build !noportable
 
 package service

service/signals_unix.go

@@ -1,3 +1,4 @@
+//go:build !windows
 // +build !windows
 
 package service

sftpd/cmd_unix.go

@@ -1,3 +1,4 @@
+//go:build !windows
 // +build !windows
 
 package sftpd

sftpd/internal_unix_test.go

@@ -1,3 +1,4 @@
+//go:build !windows
 // +build !windows
 
 package sftpd

sftpd/server.go

@@ -241,17 +241,15 @@ func (c *Configuration) Initialize(configDir string) error {
 				return
 			}
 
-			if binding.ApplyProxyConfig {
+			if binding.ApplyProxyConfig && common.Config.ProxyProtocol > 0 {
 				proxyListener, err := common.Config.GetProxyListener(listener)
 				if err != nil {
 					logger.Warn(logSender, "", "error enabling proxy listener: %v", err)
 					exitChannel <- err
 					return
 				}
-				if proxyListener != nil {
 				listener = proxyListener
 			}
-			}
 
 			exitChannel <- c.serve(listener, serverConfig)
 		}(binding)

vfs/azblobfs.go

@@ -1,3 +1,4 @@
+//go:build !noazblob
 // +build !noazblob
 
 package vfs

vfs/azblobfs_disabled.go

@@ -1,3 +1,4 @@
+//go:build noazblob
 // +build noazblob
 
 package vfs

vfs/gcsfs.go

@@ -1,3 +1,4 @@
+//go:build !nogcs
 // +build !nogcs
 
 package vfs

vfs/gcsfs_disabled.go

@@ -1,3 +1,4 @@
+//go:build nogcs
 // +build nogcs
 
 package vfs

vfs/s3fs.go

@@ -1,3 +1,4 @@
+//go:build !nos3
 // +build !nos3
 
 package vfs

vfs/s3fs_disabled.go

@@ -1,3 +1,4 @@
+//go:build nos3
 // +build nos3
 
 package vfs

vfs/statvfs_fallback.go

@@ -1,3 +1,4 @@
+//go:build !darwin && !linux && !freebsd
 // +build !darwin,!linux,!freebsd
 
 package vfs

vfs/statvfs_linux.go

@@ -1,3 +1,4 @@
+//go:build linux
 // +build linux
 
 package vfs

vfs/statvfs_unix.go

@@ -1,3 +1,4 @@
+//go:build freebsd || darwin
 // +build freebsd darwin
 
 package vfs

vfs/sys_unix.go

@@ -1,3 +1,4 @@
+//go:build !windows
 // +build !windows
 
 package vfs

webdavd/webdavd_test.go

@@ -14,6 +14,7 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"strings"
 	"sync"
@@ -651,6 +652,110 @@ func TestBasicHandlingCryptFs(t *testing.T) {
 	assert.Len(t, common.Connections.GetStats(), 0)
 }
 
+func TestLockAfterDelete(t *testing.T) {
+	u := getTestUser()
+	user, _, err := httpdtest.AddUser(u, http.StatusCreated)
+	assert.NoError(t, err)
+
+	client := getWebDavClient(user, false, nil)
+	assert.NoError(t, checkBasicFunc(client))
+	testFilePath := filepath.Join(homeBasePath, testFileName)
+	testFileSize := int64(65535)
+	err = createTestFile(testFilePath, testFileSize)
+	assert.NoError(t, err)
+	err = uploadFile(testFilePath, testFileName, testFileSize, client)
+	assert.NoError(t, err)
+	lockBody := `<?xml version="1.0" encoding="utf-8" ?><d:lockinfo xmlns:d="DAV:"><d:lockscope><d:exclusive/></d:lockscope><d:locktype><d:write/></d:locktype></d:lockinfo>`
+	req, err := http.NewRequest("LOCK", fmt.Sprintf("http://%v/%v", webDavServerAddr, testFileName), bytes.NewReader([]byte(lockBody)))
+	assert.NoError(t, err)
+	req.SetBasicAuth(u.Username, u.Password)
+	httpClient := httpclient.GetHTTPClient()
+	resp, err := httpClient.Do(req)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	response, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	re := regexp.MustCompile(`\<D:locktoken><D:href>.*</D:href>`)
+	lockToken := string(re.Find(response))
+	lockToken = strings.Replace(lockToken, "<D:locktoken><D:href>", "", 1)
+	lockToken = strings.Replace(lockToken, "</D:href>", "", 1)
+	err = resp.Body.Close()
+	assert.NoError(t, err)
+
+	req, err = http.NewRequest(http.MethodDelete, fmt.Sprintf("http://%v/%v", webDavServerAddr, testFileName), nil)
+	assert.NoError(t, err)
+	req.Header.Set("If", fmt.Sprintf("(%v)", lockToken))
+	req.SetBasicAuth(u.Username, u.Password)
+	resp, err = httpClient.Do(req)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusNoContent, resp.StatusCode)
+	err = resp.Body.Close()
+	assert.NoError(t, err)
+	// if we try to lock again it must succeed, the lock must be deleted with the object
+	req, err = http.NewRequest("LOCK", fmt.Sprintf("http://%v/%v", webDavServerAddr, testFileName), bytes.NewReader([]byte(lockBody)))
+	assert.NoError(t, err)
+	req.SetBasicAuth(u.Username, u.Password)
+	resp, err = httpClient.Do(req)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusCreated, resp.StatusCode)
+	err = resp.Body.Close()
+	assert.NoError(t, err)
+
+	_, err = httpdtest.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+}
+
+func TestRenameWithLock(t *testing.T) {
+	u := getTestUser()
+	user, _, err := httpdtest.AddUser(u, http.StatusCreated)
+	assert.NoError(t, err)
+
+	client := getWebDavClient(user, false, nil)
+	assert.NoError(t, checkBasicFunc(client))
+	testFilePath := filepath.Join(homeBasePath, testFileName)
+	testFileSize := int64(65535)
+	err = createTestFile(testFilePath, testFileSize)
+	assert.NoError(t, err)
+	err = uploadFile(testFilePath, testFileName, testFileSize, client)
+	assert.NoError(t, err)
+
+	lockBody := `<?xml version="1.0" encoding="utf-8" ?><d:lockinfo xmlns:d="DAV:"><d:lockscope><d:exclusive/></d:lockscope><d:locktype><d:write/></d:locktype></d:lockinfo>`
+	req, err := http.NewRequest("LOCK", fmt.Sprintf("http://%v/%v", webDavServerAddr, testFileName), bytes.NewReader([]byte(lockBody)))
+	assert.NoError(t, err)
+	req.SetBasicAuth(u.Username, u.Password)
+	httpClient := httpclient.GetHTTPClient()
+	resp, err := httpClient.Do(req)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	response, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	re := regexp.MustCompile(`\<D:locktoken><D:href>.*</D:href>`)
+	lockToken := string(re.Find(response))
+	lockToken = strings.Replace(lockToken, "<D:locktoken><D:href>", "", 1)
+	lockToken = strings.Replace(lockToken, "</D:href>", "", 1)
+	err = resp.Body.Close()
+	assert.NoError(t, err)
+	// MOVE with a lock should succeeded
+	req, err = http.NewRequest("MOVE", fmt.Sprintf("http://%v/%v", webDavServerAddr, testFileName), nil)
+	assert.NoError(t, err)
+	req.Header.Set("If", fmt.Sprintf("(%v)", lockToken))
+	req.Header.Set("Overwrite", "T")
+	req.Header.Set("Destination", path.Join("/", testFileName+"1"))
+	req.SetBasicAuth(u.Username, u.Password)
+	resp, err = httpClient.Do(req)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusCreated, resp.StatusCode)
+	err = resp.Body.Close()
+	assert.NoError(t, err)
+
+	_, err = httpdtest.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+}
+
 func TestPropPatch(t *testing.T) {
 	u := getTestUser()
 	u.Username = u.Username + "1"