add support for the venerable FTP protocol

Fixes #46

.github/workflows/development.yml

@@ -61,6 +61,7 @@ jobs:
           go test -v -timeout 1m ./common -covermode=atomic
           go test -v -timeout 5m ./httpd -covermode=atomic
           go test -v -timeout 5m ./sftpd -covermode=atomic
+          go test -v -timeout 5m ./ftpd -covermode=atomic
         env:
           SFTPGO_DATA_PROVIDER__DRIVER: bolt
           SFTPGO_DATA_PROVIDER__NAME: 'sftpgo_bolt.db'
@@ -177,4 +178,4 @@ jobs:
       - name: Run golangci-lint
         uses: golangci/golangci-lint-action@v1
         with:
-          version: v1.27
+          version: v1.29

README.md

@@ -33,9 +33,10 @@ Fully featured and highly configurable SFTP server, written in Go
 - Atomic uploads are configurable.
 - Support for Git repositories over SSH.
 - SCP and rsync are supported.
-- Support for serving local filesystem, S3 Compatible Object Storage and Google Cloud Storage over SFTP/SCP.
+- FTP/S is supported.
+- Support for serving local filesystem, S3 Compatible Object Storage and Google Cloud Storage over SFTP/SCP/FTP.
 - [Prometheus metrics](./docs/metrics.md) are exposed.
-- Support for HAProxy PROXY protocol: you can proxy and/or load balance the SFTP/SCP service without losing the information about the client's address.
+- Support for HAProxy PROXY protocol: you can proxy and/or load balance the SFTP/SCP/FTP service without losing the information about the client's address.
 - [REST API](./docs/rest-api.md) for users and folders management, backup, restore and real time reports of the active connections with possibility of forcibly closing a connection.
 - [Web based administration interface](./docs/web-admin.md) to easily manage users, folders and connections.
 - Easy [migration](./examples/rest-api-cli#convert-users-from-other-stores) from Linux system user accounts.
@@ -51,7 +52,8 @@ SFTPGo is developed and tested on Linux. After each commit, the code is automati
 ## Requirements
 
 - Go 1.13 or higher as build only dependency.
-- A suitable SQL server or key/value store to use as data provider: PostgreSQL 9.4+ or MySQL 5.6+ or SQLite 3.x or bbolt 1.3.x
+- A suitable SQL server to use as data provider: PostgreSQL 9.4+ or MySQL 5.6+ or SQLite 3.x.
+- The SQL server is optional: you can choose to use an embedded bolt database as key/value store or an in memory data provider.
 
 ## Installation
 

cmd/portable.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"github.com/drakkan/sftpgo/common"
 	"github.com/drakkan/sftpgo/dataprovider"
 	"github.com/drakkan/sftpgo/service"
 	"github.com/drakkan/sftpgo/sftpd"
@@ -49,6 +50,9 @@ var (
 	portableGCSAutoCredentials   int
 	portableGCSStorageClass      string
 	portableGCSKeyPrefix         string
+	portableFTPDPort             int
+	portableFTPSCert             string
+	portableFTPSKey              string
 	portableCmd                  = &cobra.Command{
 		Use:   "portable",
 		Short: "Serve a single directory",
@@ -88,6 +92,14 @@ Please take a look at the usage below to customize the serving parameters`,
 				portableGCSCredentials = base64.StdEncoding.EncodeToString(creds)
 				portableGCSAutoCredentials = 0
 			}
+			if portableFTPDPort >= 0 && len(portableFTPSCert) > 0 && len(portableFTPSKey) > 0 {
+				_, err := common.NewCertManager(portableFTPSCert, portableFTPSKey, "FTP portable")
+				if err != nil {
+					fmt.Printf("Unable to load FTPS key pair, cert file %#v key file %#v error: %v\n",
+						portableFTPSCert, portableFTPSKey, err)
+					os.Exit(1)
+				}
+			}
 			service := service.Service{
 				ConfigDir:     filepath.Clean(defaultConfigDir),
 				ConfigFile:    defaultConfigName,
@@ -133,10 +145,12 @@ Please take a look at the usage below to customize the serving parameters`,
 					},
 				},
 			}
-			if err := service.StartPortableMode(portableSFTPDPort, portableSSHCommands, portableAdvertiseService,
-				portableAdvertiseCredentials); err == nil {
+			if err := service.StartPortableMode(portableSFTPDPort, portableFTPDPort, portableSSHCommands, portableAdvertiseService,
+				portableAdvertiseCredentials, portableFTPSCert, portableFTPSKey); err == nil {
 				service.Wait()
-				os.Exit(0)
+				if service.Error == nil {
+					os.Exit(0)
+				}
 			}
 			os.Exit(1)
 		},
@@ -150,7 +164,9 @@ func init() {
 This can be an absolute path or a path
 relative to the current directory
 `)
-	portableCmd.Flags().IntVarP(&portableSFTPDPort, "sftpd-port", "s", 0, "0 means a random  unprivileged port")
+	portableCmd.Flags().IntVarP(&portableSFTPDPort, "sftpd-port", "s", 0, "0 means a random unprivileged port")
+	portableCmd.Flags().IntVar(&portableFTPDPort, "ftpd-port", -1, `0 means a random unprivileged port,
+< 0 disabled`)
 	portableCmd.Flags().StringSliceVarP(&portableSSHCommands, "ssh-commands", "c", sftpd.GetDefaultSSHCommands(),
 		`SSH commands to enable.
 "*" means any supported SSH command
@@ -177,13 +193,13 @@ insensitive. The format is
 /dir::ext1,ext2.
 For example: "/somedir::.jpg,.png"`)
 	portableCmd.Flags().BoolVarP(&portableAdvertiseService, "advertise-service", "S", false,
-		`Advertise SFTP service using multicast
-DNS`)
+		`Advertise SFTP/FTP service using
+multicast DNS`)
 	portableCmd.Flags().BoolVarP(&portableAdvertiseCredentials, "advertise-credentials", "C", false,
-		`If the SFTP service is advertised via
-multicast DNS, this flag allows to put
-username/password inside the advertised
-TXT record`)
+		`If the SFTP/FTP service is
+advertised via multicast DNS, this
+flag allows to put username/password
+inside the advertised TXT record`)
 	portableCmd.Flags().IntVarP(&portableFsProvider, "fs-provider", "f", 0, `0 means local filesystem,
 1 Amazon S3 compatible,
 2 Google Cloud Storage`)
@@ -210,6 +226,8 @@ file`)
 	portableCmd.Flags().IntVar(&portableGCSAutoCredentials, "gcs-automatic-credentials", 1, `0 means explicit credentials using
 a JSON credentials file, 1 automatic
 `)
+	portableCmd.Flags().StringVar(&portableFTPSCert, "ftpd-cert", "", "Path to the certificate file for FTPS")
+	portableCmd.Flags().StringVar(&portableFTPSKey, "ftpd-key", "", "Path to the key file for FTPS")
 	rootCmd.AddCommand(portableCmd)
 }
 

common/actions_test.go

@@ -167,7 +167,7 @@ func TestPreDeleteAction(t *testing.T) {
 	c := NewBaseConnection("id", ProtocolSFTP, user, fs)
 
 	testfile := filepath.Join(user.HomeDir, "testfile")
-	err = ioutil.WriteFile(testfile, []byte("test"), 0666)
+	err = ioutil.WriteFile(testfile, []byte("test"), os.ModePerm)
 	assert.NoError(t, err)
 	info, err := os.Stat(testfile)
 	assert.NoError(t, err)

common/common.go

@@ -18,6 +18,7 @@ import (
 
 // constants
 const (
+	logSender                = "common"
 	uploadLogSender          = "Upload"
 	downloadLogSender        = "Download"
 	renameLogSender          = "Rename"
@@ -35,7 +36,7 @@ const (
 	operationRename          = "rename"
 	operationSSHCmd          = "ssh_cmd"
 	chtimesFormat            = "2006-01-02T15:04:05" // YYYY-MM-DDTHH:MM:SS
-	idleTimeoutCheckInterval = 5 * time.Minute
+	idleTimeoutCheckInterval = 3 * time.Minute
 )
 
 // Stat flags
@@ -56,6 +57,7 @@ const (
 	ProtocolSFTP = "SFTP"
 	ProtocolSCP  = "SCP"
 	ProtocolSSH  = "SSH"
+	ProtocolFTP  = "FTP"
 )
 
 // Upload modes
@@ -84,12 +86,13 @@ var (
 	QuotaScans            ActiveScans
 	idleTimeoutTicker     *time.Ticker
 	idleTimeoutTickerDone chan bool
-	supportedProcols      = []string{ProtocolSFTP, ProtocolSCP, ProtocolSSH}
+	supportedProcols      = []string{ProtocolSFTP, ProtocolSCP, ProtocolSSH, ProtocolFTP}
 )
 
 // Initialize sets the common configuration
 func Initialize(c Configuration) {
 	Config = c
+	Config.idleLoginTimeout = 2 * time.Minute
 	Config.idleTimeoutAsDuration = time.Duration(Config.IdleTimeout) * time.Minute
 	if Config.IdleTimeout > 0 {
 		startIdleTimeoutTicker(idleTimeoutCheckInterval)
@@ -220,6 +223,7 @@ type Configuration struct {
 	// connection will be rejected.
 	ProxyAllowed          []string `json:"proxy_allowed" mapstructure:"proxy_allowed"`
 	idleTimeoutAsDuration time.Duration
+	idleLoginTimeout      time.Duration
 }
 
 // IsAtomicUploadEnabled returns true if atomic upload is enabled
@@ -291,15 +295,35 @@ func (conns *ActiveConnections) Add(c ActiveConnection) {
 	logger.Debug(c.GetProtocol(), c.GetID(), "connection added, num open connections: %v", len(conns.connections))
 }
 
+// Swap replaces an existing connection with the given one.
+// This method is useful if you have to change some connection details
+// for example for FTP is used to update the connection once the user
+// authenticates
+func (conns *ActiveConnections) Swap(c ActiveConnection) error {
+	conns.Lock()
+	defer conns.Unlock()
+
+	for idx, conn := range conns.connections {
+		if conn.GetID() == c.GetID() {
+			conn = nil
+			conns.connections[idx] = c
+			return nil
+		}
+	}
+	return errors.New("connection to swap not found")
+}
+
 // Remove removes a connection from the active ones
-func (conns *ActiveConnections) Remove(c ActiveConnection) {
+func (conns *ActiveConnections) Remove(connectionID string) {
 	conns.Lock()
 	defer conns.Unlock()
 
+	var c ActiveConnection
 	indexToRemove := -1
-	for i, v := range conns.connections {
-		if v.GetID() == c.GetID() {
+	for i, conn := range conns.connections {
+		if conn.GetID() == connectionID {
 			indexToRemove = i
+			c = conn
 			break
 		}
 	}
@@ -307,19 +331,20 @@ func (conns *ActiveConnections) Remove(c ActiveConnection) {
 		conns.connections[indexToRemove] = conns.connections[len(conns.connections)-1]
 		conns.connections[len(conns.connections)-1] = nil
 		conns.connections = conns.connections[:len(conns.connections)-1]
+		metrics.UpdateActiveConnectionsSize(len(conns.connections))
 		logger.Debug(c.GetProtocol(), c.GetID(), "connection removed, num open connections: %v",
 			len(conns.connections))
+		// we have finished to send data here and most of the time the underlying network connection
+		// is already closed. Sometime a client can still be reading the last sended data, so we set
+		// a deadline instead of directly closing the network connection.
+		// Setting a deadline on an already closed connection has no effect.
+		// We only need to ensure that a connection will not remain indefinitely open and so the
+		// underlying file descriptor is not released.
+		// This should protect us against buggy clients and edge cases.
+		c.SetConnDeadline()
 	} else {
-		logger.Warn(c.GetProtocol(), c.GetID(), "connection to remove not found!")
+		logger.Warn(logSender, "", "connection to remove with id %#v not found!", connectionID)
 	}
-	// we have finished to send data here and most of the time the underlying network connection
-	// is already closed. Sometime a client can still be reading the last sended data, so we set
-	// a deadline instead of directly closing the network connection.
-	// Setting a deadline on an already closed connection has no effect.
-	// We only need to ensure that a connection will not remain indefinitely open and so the
-	// underlying file descriptor is not released.
-	// This should protect us against buggy clients and edge cases.
-	c.SetConnDeadline()
 }
 
 // Close closes an active connection.
@@ -330,10 +355,10 @@ func (conns *ActiveConnections) Close(connectionID string) bool {
 
 	for _, c := range conns.connections {
 		if c.GetID() == connectionID {
-			defer func() {
-				err := c.Disconnect()
-				logger.Debug(c.GetProtocol(), c.GetID(), "close connection requested, close err: %v", err)
-			}()
+			defer func(conn ActiveConnection) {
+				err := conn.Disconnect()
+				logger.Debug(conn.GetProtocol(), conn.GetID(), "close connection requested, close err: %v", err)
+			}(c)
 			result = true
 			break
 		}
@@ -348,11 +373,19 @@ func (conns *ActiveConnections) checkIdleConnections() {
 
 	for _, c := range conns.connections {
 		idleTime := time.Since(c.GetLastActivity())
-		if idleTime > Config.idleTimeoutAsDuration {
-			defer func() {
-				err := c.Disconnect()
-				logger.Debug(c.GetProtocol(), c.GetID(), "close idle connection, idle time: %v, close err: %v", idleTime, err)
-			}()
+		isUnauthenticatedFTPUser := (c.GetProtocol() == ProtocolFTP && len(c.GetUsername()) == 0)
+
+		if idleTime > Config.idleTimeoutAsDuration || (isUnauthenticatedFTPUser && idleTime > Config.idleLoginTimeout) {
+			defer func(conn ActiveConnection, isFTPNoAuth bool) {
+				err := conn.Disconnect()
+				logger.Debug(conn.GetProtocol(), conn.GetID(), "close idle connection, idle time: %v, username: %#v close err: %v",
+					idleTime, conn.GetUsername(), err)
+				if isFTPNoAuth {
+					logger.ConnectionFailedLog("", utils.GetIPFromRemoteAddress(c.GetRemoteAddress()),
+						"no_auth_tryed", "client idle")
+					metrics.AddNoAuthTryed()
+				}
+			}(c, isUnauthenticatedFTPUser)
 		}
 	}
 

common/common_test.go

@@ -19,7 +19,7 @@ import (
 )
 
 const (
-	logSender        = "common_test"
+	logSenderTest    = "common_test"
 	httpAddr         = "127.0.0.1:9999"
 	httpProxyAddr    = "127.0.0.1:7777"
 	configDir        = ".."
@@ -37,8 +37,18 @@ type fakeConnection struct {
 	sshCommand string
 }
 
+func (c *fakeConnection) AddUser(user dataprovider.User) error {
+	fs, err := user.GetFilesystem(c.GetID())
+	if err != nil {
+		return err
+	}
+	c.BaseConnection.User = user
+	c.BaseConnection.Fs = fs
+	return nil
+}
+
 func (c *fakeConnection) Disconnect() error {
-	Connections.Remove(c)
+	Connections.Remove(c.GetID())
 	return nil
 }
 
@@ -166,16 +176,30 @@ func TestIdleConnections(t *testing.T) {
 	user := dataprovider.User{
 		Username: username,
 	}
-	c := NewBaseConnection("id", ProtocolSFTP, user, nil)
+	c := NewBaseConnection("id1", ProtocolSFTP, user, nil)
 	c.lastActivity = time.Now().Add(-24 * time.Hour).UnixNano()
 	fakeConn := &fakeConnection{
 		BaseConnection: c,
 	}
 	Connections.Add(fakeConn)
 	assert.Equal(t, Connections.GetActiveSessions(username), 1)
+	c = NewBaseConnection("id2", ProtocolFTP, dataprovider.User{}, nil)
+	c.lastActivity = time.Now().UnixNano()
+	fakeConn = &fakeConnection{
+		BaseConnection: c,
+	}
+	Connections.Add(fakeConn)
+	assert.Equal(t, Connections.GetActiveSessions(username), 1)
+	assert.Len(t, Connections.GetStats(), 2)
+
 	startIdleTimeoutTicker(100 * time.Millisecond)
 	assert.Eventually(t, func() bool { return Connections.GetActiveSessions(username) == 0 }, 1*time.Second, 200*time.Millisecond)
 	stopIdleTimeoutTicker()
+	assert.Len(t, Connections.GetStats(), 1)
+	c.lastActivity = time.Now().Add(-24 * time.Hour).UnixNano()
+	startIdleTimeoutTicker(100 * time.Millisecond)
+	assert.Eventually(t, func() bool { return len(Connections.GetStats()) == 0 }, 1*time.Second, 200*time.Millisecond)
+	stopIdleTimeoutTicker()
 
 	Config = configCopy
 }
@@ -192,7 +216,34 @@ func TestCloseConnection(t *testing.T) {
 	assert.Eventually(t, func() bool { return len(Connections.GetStats()) == 0 }, 300*time.Millisecond, 50*time.Millisecond)
 	res = Connections.Close(fakeConn.GetID())
 	assert.False(t, res)
-	Connections.Remove(fakeConn)
+	Connections.Remove(fakeConn.GetID())
+}
+
+func TestSwapConnection(t *testing.T) {
+	c := NewBaseConnection("id", ProtocolFTP, dataprovider.User{}, nil)
+	fakeConn := &fakeConnection{
+		BaseConnection: c,
+	}
+	Connections.Add(fakeConn)
+	if assert.Len(t, Connections.GetStats(), 1) {
+		assert.Equal(t, "", Connections.GetStats()[0].Username)
+	}
+	c = NewBaseConnection("id", ProtocolFTP, dataprovider.User{
+		Username: userTestUsername,
+	}, nil)
+	fakeConn = &fakeConnection{
+		BaseConnection: c,
+	}
+	err := Connections.Swap(fakeConn)
+	assert.NoError(t, err)
+	if assert.Len(t, Connections.GetStats(), 1) {
+		assert.Equal(t, userTestUsername, Connections.GetStats()[0].Username)
+	}
+	res := Connections.Close(fakeConn.GetID())
+	assert.True(t, res)
+	assert.Eventually(t, func() bool { return len(Connections.GetStats()) == 0 }, 300*time.Millisecond, 50*time.Millisecond)
+	err = Connections.Swap(fakeConn)
+	assert.Error(t, err)
 }
 
 func TestAtomicUpload(t *testing.T) {
@@ -255,8 +306,8 @@ func TestConnectionStatus(t *testing.T) {
 	err = t2.Close()
 	assert.NoError(t, err)
 
-	Connections.Remove(fakeConn1)
-	Connections.Remove(fakeConn2)
+	Connections.Remove(fakeConn1.GetID())
+	Connections.Remove(fakeConn2.GetID())
 	stats = Connections.GetStats()
 	assert.Len(t, stats, 0)
 }

common/tlsutils.go

@@ -19,7 +19,7 @@ type CertManager struct {
 func (m *CertManager) LoadCertificate(logSender string) error {
 	newCert, err := tls.LoadX509KeyPair(m.certPath, m.keyPath)
 	if err != nil {
-		logger.Warn(logSender, "", "unable to load X509 ket pair, cert file %#v key file %#v error: %v",
+		logger.Warn(logSender, "", "unable to load X509 key pair, cert file %#v key file %#v error: %v",
 			m.certPath, m.keyPath, err)
 		return err
 	}

common/tlsutils_test.go

@@ -43,7 +43,7 @@ func TestLoadCertificate(t *testing.T) {
 	assert.NoError(t, err)
 	err = ioutil.WriteFile(keyPath, []byte(httpsKey), os.ModePerm)
 	assert.NoError(t, err)
-	certManager, err := NewCertManager(certPath, keyPath, logSender)
+	certManager, err := NewCertManager(certPath, keyPath, logSenderTest)
 	assert.NoError(t, err)
 	certFunc := certManager.GetCertificateFunc()
 	if assert.NotNil(t, certFunc) {
@@ -63,7 +63,7 @@ func TestLoadCertificate(t *testing.T) {
 }
 
 func TestLoadInvalidCert(t *testing.T) {
-	certManager, err := NewCertManager("test.crt", "test.key", logSender)
+	certManager, err := NewCertManager("test.crt", "test.key", logSenderTest)
 	assert.Error(t, err)
 	assert.Nil(t, certManager)
 }

config/config.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/drakkan/sftpgo/common"
 	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/ftpd"
 	"github.com/drakkan/sftpgo/httpclient"
 	"github.com/drakkan/sftpgo/httpd"
 	"github.com/drakkan/sftpgo/logger"
@@ -28,13 +29,15 @@ const (
 )
 
 var (
-	globalConf    globalConfig
-	defaultBanner = fmt.Sprintf("SFTPGo_%v", version.Get().Version)
+	globalConf         globalConfig
+	defaultSFTPDBanner = fmt.Sprintf("SFTPGo_%v", version.Get().Version)
+	defaultFTPDBanner  = fmt.Sprintf("SFTPGo %v ready", version.Get().Version)
 )
 
 type globalConfig struct {
 	Common       common.Configuration `json:"common" mapstructure:"common"`
 	SFTPD        sftpd.Configuration  `json:"sftpd" mapstructure:"sftpd"`
+	FTPD         ftpd.Configuration   `json:"ftpd" mapstructure:"ftpd"`
 	ProviderConf dataprovider.Config  `json:"data_provider" mapstructure:"data_provider"`
 	HTTPDConfig  httpd.Conf           `json:"httpd" mapstructure:"httpd"`
 	HTTPConfig   httpclient.Config    `json:"http" mapstructure:"http"`
@@ -55,7 +58,7 @@ func init() {
 			ProxyAllowed:  []string{},
 		},
 		SFTPD: sftpd.Configuration{
-			Banner:                  defaultBanner,
+			Banner:                  defaultSFTPDBanner,
 			BindPort:                2022,
 			BindAddress:             "",
 			MaxAuthTries:            0,
@@ -68,6 +71,20 @@ func init() {
 			EnabledSSHCommands:      sftpd.GetDefaultSSHCommands(),
 			KeyboardInteractiveHook: "",
 		},
+		FTPD: ftpd.Configuration{
+			BindPort:                 0,
+			BindAddress:              "",
+			Banner:                   defaultFTPDBanner,
+			BannerFile:               "",
+			ActiveTransfersPortNon20: false,
+			ForcePassiveIP:           "",
+			PassivePortRange: ftpd.PortRange{
+				Start: 50000,
+				End:   50100,
+			},
+			CertificateFile:    "",
+			CertificateKeyFile: "",
+		},
 		ProviderConf: dataprovider.Config{
 			Driver:           "sqlite",
 			Name:             "sftpgo.db",
@@ -136,6 +153,16 @@ func SetSFTPDConfig(config sftpd.Configuration) {
 	globalConf.SFTPD = config
 }
 
+// GetFTPDConfig returns the configuration for the FTP server
+func GetFTPDConfig() ftpd.Configuration {
+	return globalConf.FTPD
+}
+
+// SetFTPDConfig sets the configuration for the FTP server
+func SetFTPDConfig(config ftpd.Configuration) {
+	globalConf.FTPD = config
+}
+
 // GetHTTPDConfig returns the configuration for the HTTP server
 func GetHTTPDConfig() httpd.Conf {
 	return globalConf.HTTPDConfig
@@ -193,7 +220,10 @@ func LoadConfig(configDir, configName string) error {
 	}
 	checkCommonParamsCompatibility()
 	if strings.TrimSpace(globalConf.SFTPD.Banner) == "" {
-		globalConf.SFTPD.Banner = defaultBanner
+		globalConf.SFTPD.Banner = defaultSFTPDBanner
+	}
+	if strings.TrimSpace(globalConf.FTPD.Banner) == "" {
+		globalConf.FTPD.Banner = defaultFTPDBanner
 	}
 	if len(globalConf.ProviderConf.UsersBaseDir) > 0 && !utils.IsFileInputValid(globalConf.ProviderConf.UsersBaseDir) {
 		err = fmt.Errorf("invalid users base dir %#v will be ignored", globalConf.ProviderConf.UsersBaseDir)

config/config_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/drakkan/sftpgo/common"
 	"github.com/drakkan/sftpgo/config"
 	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/ftpd"
 	"github.com/drakkan/sftpgo/httpclient"
 	"github.com/drakkan/sftpgo/httpd"
 	"github.com/drakkan/sftpgo/sftpd"
@@ -36,11 +37,11 @@ func TestLoadConfigTest(t *testing.T) {
 	configFilePath := filepath.Join(configDir, confName)
 	err = config.LoadConfig(configDir, tempConfigName)
 	assert.NotNil(t, err)
-	err = ioutil.WriteFile(configFilePath, []byte("{invalid json}"), 0666)
+	err = ioutil.WriteFile(configFilePath, []byte("{invalid json}"), os.ModePerm)
 	assert.NoError(t, err)
 	err = config.LoadConfig(configDir, tempConfigName)
 	assert.NotNil(t, err)
-	err = ioutil.WriteFile(configFilePath, []byte("{\"sftpd\": {\"bind_port\": \"a\"}}"), 0666)
+	err = ioutil.WriteFile(configFilePath, []byte("{\"sftpd\": {\"bind_port\": \"a\"}}"), os.ModePerm)
 	assert.NoError(t, err)
 	err = config.LoadConfig(configDir, tempConfigName)
 	assert.NotNil(t, err)
@@ -59,7 +60,7 @@ func TestEmptyBanner(t *testing.T) {
 	c := make(map[string]sftpd.Configuration)
 	c["sftpd"] = sftpdConf
 	jsonConf, _ := json.Marshal(c)
-	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm)
 	assert.NoError(t, err)
 	err = config.LoadConfig(configDir, tempConfigName)
 	assert.NoError(t, err)
@@ -67,6 +68,20 @@ func TestEmptyBanner(t *testing.T) {
 	assert.NotEmpty(t, strings.TrimSpace(sftpdConf.Banner))
 	err = os.Remove(configFilePath)
 	assert.NoError(t, err)
+
+	ftpdConf := config.GetFTPDConfig()
+	ftpdConf.Banner = " "
+	c1 := make(map[string]ftpd.Configuration)
+	c1["ftpd"] = ftpdConf
+	jsonConf, _ = json.Marshal(c1)
+	err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm)
+	assert.NoError(t, err)
+	err = config.LoadConfig(configDir, tempConfigName)
+	assert.NoError(t, err)
+	ftpdConf = config.GetFTPDConfig()
+	assert.NotEmpty(t, strings.TrimSpace(ftpdConf.Banner))
+	err = os.Remove(configFilePath)
+	assert.NoError(t, err)
 }
 
 func TestInvalidUploadMode(t *testing.T) {
@@ -81,7 +96,7 @@ func TestInvalidUploadMode(t *testing.T) {
 	c["common"] = commonConf
 	jsonConf, err := json.Marshal(c)
 	assert.NoError(t, err)
-	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm)
 	assert.NoError(t, err)
 	err = config.LoadConfig(configDir, tempConfigName)
 	assert.NotNil(t, err)
@@ -101,7 +116,7 @@ func TestInvalidExternalAuthScope(t *testing.T) {
 	c["data_provider"] = providerConf
 	jsonConf, err := json.Marshal(c)
 	assert.NoError(t, err)
-	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm)
 	assert.NoError(t, err)
 	err = config.LoadConfig(configDir, tempConfigName)
 	assert.NotNil(t, err)
@@ -121,7 +136,7 @@ func TestInvalidCredentialsPath(t *testing.T) {
 	c["data_provider"] = providerConf
 	jsonConf, err := json.Marshal(c)
 	assert.NoError(t, err)
-	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm)
 	assert.NoError(t, err)
 	err = config.LoadConfig(configDir, tempConfigName)
 	assert.NotNil(t, err)
@@ -141,7 +156,7 @@ func TestInvalidProxyProtocol(t *testing.T) {
 	c["common"] = commonConf
 	jsonConf, err := json.Marshal(c)
 	assert.NoError(t, err)
-	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm)
 	assert.NoError(t, err)
 	err = config.LoadConfig(configDir, tempConfigName)
 	assert.NotNil(t, err)
@@ -161,7 +176,7 @@ func TestInvalidUsersBaseDir(t *testing.T) {
 	c["data_provider"] = providerConf
 	jsonConf, err := json.Marshal(c)
 	assert.NoError(t, err)
-	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm)
 	assert.NoError(t, err)
 	err = config.LoadConfig(configDir, tempConfigName)
 	assert.NotNil(t, err)
@@ -187,7 +202,7 @@ func TestCommonParamsCompatibility(t *testing.T) {
 	c["sftpd"] = sftpdConf
 	jsonConf, err := json.Marshal(c)
 	assert.NoError(t, err)
-	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm)
 	assert.NoError(t, err)
 	err = config.LoadConfig(configDir, tempConfigName)
 	assert.NoError(t, err)
@@ -223,7 +238,7 @@ func TestHostKeyCompatibility(t *testing.T) {
 	c["sftpd"] = sftpdConf
 	jsonConf, err := json.Marshal(c)
 	assert.NoError(t, err)
-	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm)
 	assert.NoError(t, err)
 	err = config.LoadConfig(configDir, tempConfigName)
 	assert.NoError(t, err)
@@ -252,4 +267,10 @@ func TestSetGetConfig(t *testing.T) {
 	commonConf.IdleTimeout = 10
 	config.SetCommonConfig(commonConf)
 	assert.Equal(t, commonConf.IdleTimeout, config.GetCommonConfig().IdleTimeout)
+	ftpdConf := config.GetFTPDConfig()
+	ftpdConf.CertificateFile = "cert"
+	ftpdConf.CertificateKeyFile = "key"
+	config.SetFTPDConfig(ftpdConf)
+	assert.Equal(t, ftpdConf.CertificateFile, config.GetFTPDConfig().CertificateFile)
+	assert.Equal(t, ftpdConf.CertificateKeyFile, config.GetFTPDConfig().CertificateKeyFile)
 }

dataprovider/user.go

@@ -46,13 +46,14 @@ const (
 	PermChtimes = "chtimes"
 )
 
-// Available SSH login methods
+// Available login methods
 const (
 	SSHLoginMethodPublicKey           = "publickey"
 	SSHLoginMethodPassword            = "password"
 	SSHLoginMethodKeyboardInteractive = "keyboard-interactive"
 	SSHLoginMethodKeyAndPassword      = "publickey+password"
 	SSHLoginMethodKeyAndKeyboardInt   = "publickey+keyboard-interactive"
+	FTPLoginMethodPassword            = "ftp-password"
 )
 
 var (

docs/full-configuration.md

@@ -78,6 +78,16 @@ The configuration file contains the following sections:
   - `keyboard_interactive_auth_hook`, string. Absolute path to an external program or an HTTP URL to invoke for keyboard interactive authentication. See the "Keyboard Interactive Authentication" paragraph for more details.
   - `proxy_protocol`, integer.  Deprecated, please use the same key in `common` section.
   - `proxy_allowed`, list of strings. Deprecated, please use the same key in `common` section.
+- **"ftpd"**, the configuration for the FTP server
+  - `bind_port`, integer. The port used for serving FTP requests. 0 means disabled. Default: 0.
+  - `bind_address`, string. Leave blank to listen on all available network interfaces. Default: "".
+  - `banner`, string. Greeting banner displayed when a connection first comes in. Leave empty to use the default banner. Default `SFTPGo <version> ready`, for example `SFTPGo 1.0.0-dev ready`.
+  - `banner_file`, path to the banner file. The contents of the specified file, if any, are diplayed when someone connects to the server. It can be a path relative to the config dir or an absolute one. If set, it overrides the banner string provided by the `banner` option. Leave empty to disable.
+  - `active_transfers_port_non_20`, boolean. Do not impose the port 20 for active data transfers. Enabling this option allows to run SFTPGo with less privilege. Default: false.
+  - `force_passive_ip`, ip address. External IP address to expose for passive connections. Leavy empty to autodetect. Defaut: "".
+  - `passive_port_range`, struct containing the key `start` and `end`. Port Range for data connections. Random if not specified. Default range is 50000-50100.
+  - `certificate_file`, string. Certificate for FTPS. This can be an absolute path or a path relative to the config dir.
+  - `certificate_key_file`, string. Private key matching the above certificate. This can be an absolute path or a path relative to the config dir. If both the certificate and the private key are provided, the server will accept both plain FTP an explicit FTP over TLS. Certificate and key files can be reloaded on demand sending a `SIGHUP` signal on Unix based systems and a `paramchange` request to the running service on Windows.
 - **"data_provider"**, the configuration for the data provider
   - `driver`, string. Supported drivers are `sqlite`, `mysql`, `postgresql`, `bolt`, `memory`
   - `name`, string. Database name. For driver `sqlite` this can be the database name relative to the config dir or the absolute path to the SQLite database. For driver `memory` this is the (optional) path relative to the config dir or the absolute path to the users dump, obtained using the `dumpdata` REST API, to load. This dump will be loaded at startup and can be reloaded on demand sending a `SIGHUP` signal on Unix based systems and a `paramchange` request to the running service on Windows. The `memory` provider will not modify the provided file so quota usage and last login will not be persisted

docs/portable-mode.md

@@ -15,12 +15,12 @@ Usage:
   sftpgo portable [flags]
 
 Flags:
-  -C, --advertise-credentials            If the SFTP service is advertised via
-                                         multicast DNS, this flag allows to put
-                                         username/password inside the advertised
-                                         TXT record
-  -S, --advertise-service                Advertise SFTP service using multicast
-                                         DNS
+  -C, --advertise-credentials            If the SFTP/FTP service is
+                                         advertised via multicast DNS, this
+                                         flag allows to put username/password
+                                         inside the advertised TXT record
+  -S, --advertise-service                Advertise SFTP/FTP service using
+                                         multicast DNS
       --allowed-extensions stringArray   Allowed file extensions case
                                          insensitive. The format is
                                          /dir::ext1,ext2.
@@ -36,6 +36,10 @@ Flags:
   -f, --fs-provider int                  0 means local filesystem,
                                          1 Amazon S3 compatible,
                                          2 Google Cloud Storage
+      --ftpd-cert string                 Path to the certificate file for FTPS
+      --ftpd-key string                  Path to the key file for FTPS
+      --ftpd-port int                    0 means a random unprivileged port,
+                                         < 0 disabled (default -1)
       --gcs-automatic-credentials int    0 means explicit credentials using
                                          a JSON credentials file, 1 automatic
                                           (default 1)
@@ -67,7 +71,7 @@ Flags:
                                          parallel (default 2)
       --s3-upload-part-size int          The buffer size for multipart uploads
                                          (MB) (default 5)
-  -s, --sftpd-port int                   0 means a random  unprivileged port
+  -s, --sftpd-port int                   0 means a random unprivileged port
   -c, --ssh-commands strings             SSH commands to enable.
                                          "*" means any supported SSH command
                                          including scp
@@ -76,9 +80,9 @@ Flags:
                                          value
 ```
 
-In portable mode, SFTPGo can advertise the SFTP service and, optionally, the credentials via multicast DNS, so there is a standard way to discover the service and to automatically connect to it.
+In portable mode, SFTPGo can advertise the SFTP/FTP services and, optionally, the credentials via multicast DNS, so there is a standard way to discover the service and to automatically connect to it.
 
-Here is an example of the advertised service including credentials as seen using `avahi-browse`:
+Here is an example of the advertised SFTP service including credentials as seen using `avahi-browse`:
 
 ```console
 = enp0s31f6 IPv4 SFTPGo portable 53705                         SFTP File Transfer   local

ftpd/ftpd.go

@@ -0,0 +1,82 @@
+// Package ftpd implements the FTP protocol
+package ftpd
+
+import (
+	"path/filepath"
+
+	ftpserver "github.com/fclairamb/ftpserverlib"
+
+	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/utils"
+)
+
+const (
+	logSender = "ftpd"
+)
+
+var (
+	server *Server
+)
+
+// PortRange defines a port range
+type PortRange struct {
+	// Range start
+	Start int `json:"start" mapstructure:"start"`
+	// Range end
+	End int `json:"end" mapstructure:"end"`
+}
+
+// Configuration defines the configuration for the ftp server
+type Configuration struct {
+	// The port used for serving FTP requests
+	BindPort int `json:"bind_port" mapstructure:"bind_port"`
+	// The address to listen on. A blank value means listen on all available network interfaces.
+	BindAddress string `json:"bind_address" mapstructure:"bind_address"`
+	// External IP address to expose for passive connections.
+	ForcePassiveIP string `json:"force_passive_ip" mapstructure:"force_passive_ip"`
+	// Greeting banner displayed when a connection first comes in
+	Banner string `json:"banner" mapstructure:"banner"`
+	// the contents of the specified file, if any, are diplayed when someone connects to the server.
+	// If set, it overrides the banner string provided by the banner option
+	BannerFile string `json:"banner_file" mapstructure:"banner_file"`
+	// If files containing a certificate and matching private key for the server are provided the server will accept
+	// both plain FTP an explicit FTP over TLS.
+	// Certificate and key files can be reloaded on demand sending a "SIGHUP" signal on Unix based systems and a
+	// "paramchange" request to the running service on Windows.
+	CertificateFile    string `json:"certificate_file" mapstructure:"certificate_file"`
+	CertificateKeyFile string `json:"certificate_key_file" mapstructure:"certificate_key_file"`
+	// Do not impose the port 20 for active data transfer. Enabling this option allows to run SFTPGo with less privilege
+	ActiveTransfersPortNon20 bool `json:"active_transfers_port_non_20" mapstructure:"active_transfers_port_non_20"`
+	// Port Range for data connections. Random if not specified
+	PassivePortRange PortRange `json:"passive_port_range" mapstructure:"passive_port_range"`
+}
+
+// Initialize configures and starts the FTP server
+func (c *Configuration) Initialize(configDir string) error {
+	var err error
+	logger.Debug(logSender, "", "initializing FTP server with config %+v", c)
+	server, err = NewServer(c, configDir)
+	if err != nil {
+		return err
+	}
+	ftpServer := ftpserver.NewFtpServer(server)
+	return ftpServer.ListenAndServe()
+}
+
+// ReloadTLSCertificate reloads the TLS certificate and key from the configured paths
+func ReloadTLSCertificate() error {
+	if server != nil && server.certMgr != nil {
+		return server.certMgr.LoadCertificate(logSender)
+	}
+	return nil
+}
+
+func getConfigPath(name, configDir string) string {
+	if !utils.IsFileInputValid(name) {
+		return ""
+	}
+	if len(name) > 0 && !filepath.IsAbs(name) {
+		return filepath.Join(configDir, name)
+	}
+	return name
+}

ftpd/ftpd_test.go

@@ -0,0 +1,1263 @@
+package ftpd_test
+
+import (
+	"crypto/rand"
+	"crypto/tls"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/jlaffaye/ftp"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/drakkan/sftpgo/common"
+	"github.com/drakkan/sftpgo/config"
+	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/ftpd"
+	"github.com/drakkan/sftpgo/httpd"
+	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/vfs"
+)
+
+const (
+	logSender       = "ftpdTesting"
+	ftpServerAddr   = "127.0.0.1:2121"
+	defaultUsername = "test_user_ftp"
+	defaultPassword = "test_password"
+	configDir       = ".."
+	osWindows       = "windows"
+	ftpsCert        = `-----BEGIN CERTIFICATE-----
+MIICHTCCAaKgAwIBAgIUHnqw7QnB1Bj9oUsNpdb+ZkFPOxMwCgYIKoZIzj0EAwIw
+RTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGElu
+dGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMDAyMDQwOTUzMDRaFw0zMDAyMDEw
+OTUzMDRaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYD
+VQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwdjAQBgcqhkjOPQIBBgUrgQQA
+IgNiAARCjRMqJ85rzMC998X5z761nJ+xL3bkmGVqWvrJ51t5OxV0v25NsOgR82CA
+NXUgvhVYs7vNFN+jxtb2aj6Xg+/2G/BNxkaFspIVCzgWkxiz7XE4lgUwX44FCXZM
+3+JeUbKjUzBRMB0GA1UdDgQWBBRhLw+/o3+Z02MI/d4tmaMui9W16jAfBgNVHSME
+GDAWgBRhLw+/o3+Z02MI/d4tmaMui9W16jAPBgNVHRMBAf8EBTADAQH/MAoGCCqG
+SM49BAMCA2kAMGYCMQDqLt2lm8mE+tGgtjDmtFgdOcI72HSbRQ74D5rYTzgST1rY
+/8wTi5xl8TiFUyLMUsICMQC5ViVxdXbhuG7gX6yEqSkMKZICHpO8hqFwOD/uaFVI
+dV4vKmHUzwK/eIx+8Ay3neE=
+-----END CERTIFICATE-----`
+	ftpsKey = `-----BEGIN EC PARAMETERS-----
+BgUrgQQAIg==
+-----END EC PARAMETERS-----
+-----BEGIN EC PRIVATE KEY-----
+MIGkAgEBBDCfMNsN6miEE3rVyUPwElfiJSWaR5huPCzUenZOfJT04GAcQdWvEju3
+UM2lmBLIXpGgBwYFK4EEACKhZANiAARCjRMqJ85rzMC998X5z761nJ+xL3bkmGVq
+WvrJ51t5OxV0v25NsOgR82CANXUgvhVYs7vNFN+jxtb2aj6Xg+/2G/BNxkaFspIV
+CzgWkxiz7XE4lgUwX44FCXZM3+JeUbI=
+-----END EC PRIVATE KEY-----`
+)
+
+var (
+	allPerms     = []string{dataprovider.PermAny}
+	homeBasePath string
+	hookCmdPath  string
+	extAuthPath  string
+	preLoginPath string
+	logFilePath  string
+)
+
+func TestMain(m *testing.M) {
+	logFilePath = filepath.Join(configDir, "sftpgo_ftpd_test.log")
+	bannerFileName := "banner_file"
+	bannerFile := filepath.Join(configDir, bannerFileName)
+	logger.InitLogger(logFilePath, 5, 1, 28, false, zerolog.DebugLevel)
+	logger.DebugToConsole("aaa %v", bannerFile)
+	err := ioutil.WriteFile(bannerFile, []byte("SFTPGo test ready\nsimple banner line\n"), os.ModePerm)
+	if err != nil {
+		logger.ErrorToConsole("error creating banner file: %v", err)
+	}
+	err = config.LoadConfig(configDir, "")
+	if err != nil {
+		logger.ErrorToConsole("error loading configuration: %v", err)
+		os.Exit(1)
+	}
+	providerConf := config.GetProviderConf()
+	logger.InfoToConsole("Starting FTPD tests, provider: %v", providerConf.Driver)
+
+	commonConf := config.GetCommonConfig()
+	// we run the test cases with UploadMode atomic and resume support. The non atomic code path
+	// simply does not execute some code so if it works in atomic mode will
+	// work in non atomic mode too
+	commonConf.UploadMode = 2
+	homeBasePath = os.TempDir()
+	if runtime.GOOS != osWindows {
+		commonConf.Actions.ExecuteOn = []string{"download", "upload", "rename", "delete"}
+		commonConf.Actions.Hook = hookCmdPath
+		hookCmdPath, err = exec.LookPath("true")
+		if err != nil {
+			logger.Warn(logSender, "", "unable to get hook command: %v", err)
+			logger.WarnToConsole("unable to get hook command: %v", err)
+		}
+	}
+
+	certPath := filepath.Join(os.TempDir(), "test.crt")
+	keyPath := filepath.Join(os.TempDir(), "test.key")
+	err = ioutil.WriteFile(certPath, []byte(ftpsCert), os.ModePerm)
+	if err != nil {
+		logger.ErrorToConsole("error writing FTPS certificate: %v", err)
+		os.Exit(1)
+	}
+	err = ioutil.WriteFile(keyPath, []byte(ftpsKey), os.ModePerm)
+	if err != nil {
+		logger.ErrorToConsole("error writing FTPS private key: %v", err)
+		os.Exit(1)
+	}
+
+	common.Initialize(commonConf)
+
+	err = dataprovider.Initialize(providerConf, configDir)
+	if err != nil {
+		logger.ErrorToConsole("error initializing data provider: %v", err)
+		os.Exit(1)
+	}
+
+	httpConfig := config.GetHTTPConfig()
+	httpConfig.Initialize(configDir)
+
+	httpdConf := config.GetHTTPDConfig()
+	httpdConf.BindPort = 8079
+	httpd.SetBaseURLAndCredentials("http://127.0.0.1:8079", "", "")
+
+	ftpdConf := config.GetFTPDConfig()
+	ftpdConf.BindPort = 2121
+	ftpdConf.PassivePortRange.Start = 0
+	ftpdConf.PassivePortRange.End = 0
+	ftpdConf.BannerFile = bannerFileName
+	ftpdConf.CertificateFile = certPath
+	ftpdConf.CertificateKeyFile = keyPath
+
+	extAuthPath = filepath.Join(homeBasePath, "extauth.sh")
+	preLoginPath = filepath.Join(homeBasePath, "prelogin.sh")
+
+	go func() {
+		logger.Debug(logSender, "", "initializing FTP server with config %+v", ftpdConf)
+		if err := ftpdConf.Initialize(configDir); err != nil {
+			logger.ErrorToConsole("could not start FTP server: %v", err)
+			os.Exit(1)
+		}
+	}()
+
+	go func() {
+		if err := httpdConf.Initialize(configDir, false); err != nil {
+			logger.ErrorToConsole("could not start HTTP server: %v", err)
+			os.Exit(1)
+		}
+	}()
+
+	waitTCPListening(fmt.Sprintf("%s:%d", ftpdConf.BindAddress, ftpdConf.BindPort))
+	waitTCPListening(fmt.Sprintf("%s:%d", httpdConf.BindAddress, httpdConf.BindPort))
+	ftpd.ReloadTLSCertificate() //nolint:errcheck
+
+	exitCode := m.Run()
+	os.Remove(logFilePath)
+	os.Remove(bannerFile)
+	os.Remove(extAuthPath)
+	os.Remove(preLoginPath)
+	os.Remove(certPath)
+	os.Remove(keyPath)
+	os.Exit(exitCode)
+}
+
+func TestBasicFTPHandling(t *testing.T) {
+	u := getTestUser()
+	u.QuotaSize = 6553600
+	user, _, err := httpd.AddUser(u, http.StatusOK)
+	assert.NoError(t, err)
+	client, err := getFTPClient(user, true)
+	if assert.NoError(t, err) {
+		assert.Len(t, common.Connections.GetStats(), 1)
+		testFileName := "test_file.dat" //nolint:goconst
+		testFilePath := filepath.Join(homeBasePath, testFileName)
+		testFileSize := int64(65535)
+		expectedQuotaSize := user.UsedQuotaSize + testFileSize
+		expectedQuotaFiles := user.UsedQuotaFiles + 1
+		err = createTestFile(testFilePath, testFileSize)
+		assert.NoError(t, err)
+
+		err = checkBasicFTP(client)
+		assert.NoError(t, err)
+		err = ftpUploadFile(testFilePath, path.Join("/missing_dir", testFileName), testFileSize, client, 0)
+		assert.Error(t, err)
+		err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0)
+		assert.NoError(t, err)
+		// overwrite an existing file
+		err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0)
+		assert.NoError(t, err)
+		localDownloadPath := filepath.Join(homeBasePath, "test_download.dat")
+		err = ftpDownloadFile(testFileName, localDownloadPath, testFileSize, client, 0)
+		assert.NoError(t, err)
+		user, _, err = httpd.GetUserByID(user.ID, http.StatusOK)
+		assert.NoError(t, err)
+		assert.Equal(t, expectedQuotaFiles, user.UsedQuotaFiles)
+		assert.Equal(t, expectedQuotaSize, user.UsedQuotaSize)
+		err = client.Rename(testFileName, testFileName+"1")
+		assert.NoError(t, err)
+		err = client.Delete(testFileName)
+		assert.Error(t, err)
+		err = client.Delete(testFileName + "1")
+		assert.NoError(t, err)
+		user, _, err = httpd.GetUserByID(user.ID, http.StatusOK)
+		assert.NoError(t, err)
+		assert.Equal(t, expectedQuotaFiles-1, user.UsedQuotaFiles)
+		assert.Equal(t, expectedQuotaSize-testFileSize, user.UsedQuotaSize)
+		curDir, err := client.CurrentDir()
+		if assert.NoError(t, err) {
+			assert.Equal(t, "/", curDir)
+		}
+		testDir := "testDir"
+		err = client.MakeDir(testDir)
+		assert.NoError(t, err)
+		err = client.ChangeDir(testDir)
+		assert.NoError(t, err)
+		curDir, err = client.CurrentDir()
+		if assert.NoError(t, err) {
+			assert.Equal(t, path.Join("/", testDir), curDir)
+		}
+		err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0)
+		assert.NoError(t, err)
+		size, err := client.FileSize(path.Join("/", testDir, testFileName))
+		assert.NoError(t, err)
+		assert.Equal(t, testFileSize, size)
+		err = client.ChangeDirToParent()
+		assert.NoError(t, err)
+		curDir, err = client.CurrentDir()
+		if assert.NoError(t, err) {
+			assert.Equal(t, "/", curDir)
+		}
+		err = client.Delete(path.Join("/", testDir, testFileName))
+		assert.NoError(t, err)
+		err = client.RemoveDir(testDir)
+		assert.NoError(t, err)
+
+		err = os.Remove(testFilePath)
+		assert.NoError(t, err)
+		err = os.Remove(localDownloadPath)
+		assert.NoError(t, err)
+		err = client.Quit()
+		assert.NoError(t, err)
+	}
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+	assert.Eventually(t, func() bool { return len(common.Connections.GetStats()) == 0 }, 1*time.Second, 50*time.Millisecond)
+}
+
+func TestLoginInvalidPwd(t *testing.T) {
+	u := getTestUser()
+	user, _, err := httpd.AddUser(u, http.StatusOK)
+	assert.NoError(t, err)
+	user.Password = "wrong"
+	_, err = getFTPClient(user, false)
+	assert.Error(t, err)
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+}
+
+func TestLoginExternalAuth(t *testing.T) {
+	if runtime.GOOS == osWindows {
+		t.Skip("this test is not available on Windows")
+	}
+	u := getTestUser()
+	err := dataprovider.Close()
+	assert.NoError(t, err)
+	err = config.LoadConfig(configDir, "")
+	assert.NoError(t, err)
+	providerConf := config.GetProviderConf()
+	err = ioutil.WriteFile(extAuthPath, getExtAuthScriptContent(u, false, ""), 0755)
+	assert.NoError(t, err)
+	providerConf.ExternalAuthHook = extAuthPath
+	providerConf.ExternalAuthScope = 0
+	err = dataprovider.Initialize(providerConf, configDir)
+	assert.NoError(t, err)
+	client, err := getFTPClient(u, true)
+	if assert.NoError(t, err) {
+		err = checkBasicFTP(client)
+		assert.NoError(t, err)
+		err := client.Quit()
+		assert.NoError(t, err)
+	}
+	u.Username = defaultUsername + "1"
+	client, err = getFTPClient(u, true)
+	if !assert.Error(t, err) {
+		err := client.Quit()
+		assert.NoError(t, err)
+	}
+
+	users, _, err := httpd.GetUsers(0, 0, defaultUsername, http.StatusOK)
+	assert.NoError(t, err)
+	if assert.Len(t, users, 1) {
+		user := users[0]
+		_, err = httpd.RemoveUser(user, http.StatusOK)
+		assert.NoError(t, err)
+		err = os.RemoveAll(user.GetHomeDir())
+		assert.NoError(t, err)
+	}
+	err = dataprovider.Close()
+	assert.NoError(t, err)
+	err = config.LoadConfig(configDir, "")
+	assert.NoError(t, err)
+	providerConf = config.GetProviderConf()
+	err = dataprovider.Initialize(providerConf, configDir)
+	assert.NoError(t, err)
+	err = os.Remove(extAuthPath)
+	assert.NoError(t, err)
+}
+
+func TestPreLoginHook(t *testing.T) {
+	if runtime.GOOS == osWindows {
+		t.Skip("this test is not available on Windows")
+	}
+	u := getTestUser()
+	err := dataprovider.Close()
+	assert.NoError(t, err)
+	err = config.LoadConfig(configDir, "")
+	assert.NoError(t, err)
+	providerConf := config.GetProviderConf()
+	err = ioutil.WriteFile(preLoginPath, getPreLoginScriptContent(u, false), 0755)
+	assert.NoError(t, err)
+	providerConf.PreLoginHook = preLoginPath
+	err = dataprovider.Initialize(providerConf, configDir)
+	assert.NoError(t, err)
+	users, _, err := httpd.GetUsers(0, 0, defaultUsername, http.StatusOK)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(users))
+	client, err := getFTPClient(u, false)
+	if assert.NoError(t, err) {
+		err = checkBasicFTP(client)
+		assert.NoError(t, err)
+		err := client.Quit()
+		assert.NoError(t, err)
+	}
+
+	users, _, err = httpd.GetUsers(0, 0, defaultUsername, http.StatusOK)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(users))
+	user := users[0]
+
+	// test login with an existing user
+	client, err = getFTPClient(user, true)
+	if assert.NoError(t, err) {
+		err = checkBasicFTP(client)
+		assert.NoError(t, err)
+		err := client.Quit()
+		assert.NoError(t, err)
+	}
+
+	err = ioutil.WriteFile(preLoginPath, getPreLoginScriptContent(user, true), 0755)
+	assert.NoError(t, err)
+	client, err = getFTPClient(u, false)
+	if !assert.Error(t, err) {
+		err := client.Quit()
+		assert.NoError(t, err)
+	}
+	user.Status = 0
+	err = ioutil.WriteFile(preLoginPath, getPreLoginScriptContent(user, false), 0755)
+	assert.NoError(t, err)
+	client, err = getFTPClient(u, false)
+	if !assert.Error(t, err, "pre-login script returned a disabled user, login must fail") {
+		err := client.Quit()
+		assert.NoError(t, err)
+	}
+
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+	err = dataprovider.Close()
+	assert.NoError(t, err)
+	err = config.LoadConfig(configDir, "")
+	assert.NoError(t, err)
+	providerConf = config.GetProviderConf()
+	err = dataprovider.Initialize(providerConf, configDir)
+	assert.NoError(t, err)
+	err = os.Remove(preLoginPath)
+	assert.NoError(t, err)
+}
+
+func TestMaxSessions(t *testing.T) {
+	u := getTestUser()
+	u.MaxSessions = 1
+	user, _, err := httpd.AddUser(u, http.StatusOK)
+	assert.NoError(t, err)
+	client, err := getFTPClient(user, true)
+	if assert.NoError(t, err) {
+		defer func() {
+			err := client.Quit()
+			assert.NoError(t, err)
+		}()
+		err = checkBasicFTP(client)
+		assert.NoError(t, err)
+		_, err = getFTPClient(user, false)
+		assert.Error(t, err)
+	}
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+}
+
+func TestZeroBytesTransfers(t *testing.T) {
+	u := getTestUser()
+	user, _, err := httpd.AddUser(u, http.StatusOK)
+	assert.NoError(t, err)
+	for _, useTLS := range []bool{true, false} {
+		client, err := getFTPClient(user, useTLS)
+		if assert.NoError(t, err) {
+			testFileName := "testfilename"
+			err = checkBasicFTP(client)
+			assert.NoError(t, err)
+			localDownloadPath := filepath.Join(homeBasePath, "empty_download")
+			err = ioutil.WriteFile(localDownloadPath, []byte(""), os.ModePerm)
+			assert.NoError(t, err)
+			err = ftpUploadFile(localDownloadPath, testFileName, 0, client, 0)
+			assert.NoError(t, err)
+			size, err := client.FileSize(testFileName)
+			assert.NoError(t, err)
+			assert.Equal(t, int64(0), size)
+			err = os.Remove(localDownloadPath)
+			assert.NoError(t, err)
+			assert.NoFileExists(t, localDownloadPath)
+			err = ftpDownloadFile(testFileName, localDownloadPath, 0, client, 0)
+			assert.NoError(t, err)
+			assert.FileExists(t, localDownloadPath)
+			err = client.Quit()
+			assert.NoError(t, err)
+			err = os.Remove(localDownloadPath)
+			assert.NoError(t, err)
+		}
+	}
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+}
+
+func TestDownloadsError(t *testing.T) {
+	u := getTestUser()
+	u.QuotaFiles = 1
+	subDir1 := "sub1"
+	subDir2 := "sub2"
+	u.Permissions[path.Join("/", subDir1)] = []string{dataprovider.PermListItems}
+	u.Permissions[path.Join("/", subDir2)] = []string{dataprovider.PermListItems, dataprovider.PermUpload,
+		dataprovider.PermDelete, dataprovider.PermDownload}
+	u.Filters.FileExtensions = []dataprovider.ExtensionsFilter{
+		{
+			Path:              "/sub2",
+			AllowedExtensions: []string{},
+			DeniedExtensions:  []string{".zip"},
+		},
+	}
+	user, _, err := httpd.AddUser(u, http.StatusOK)
+	assert.NoError(t, err)
+	client, err := getFTPClient(user, true)
+	if assert.NoError(t, err) {
+		testFilePath1 := filepath.Join(user.HomeDir, subDir1, "file.zip")
+		testFilePath2 := filepath.Join(user.HomeDir, subDir2, "file.zip")
+		err = os.MkdirAll(filepath.Dir(testFilePath1), os.ModePerm)
+		assert.NoError(t, err)
+		err = os.MkdirAll(filepath.Dir(testFilePath2), os.ModePerm)
+		assert.NoError(t, err)
+		err = ioutil.WriteFile(testFilePath1, []byte("file1"), os.ModePerm)
+		assert.NoError(t, err)
+		err = ioutil.WriteFile(testFilePath2, []byte("file2"), os.ModePerm)
+		assert.NoError(t, err)
+		localDownloadPath := filepath.Join(homeBasePath, "test_download.dat")
+		err = ftpDownloadFile(path.Join("/", subDir1, "file.zip"), localDownloadPath, 5, client, 0)
+		assert.Error(t, err)
+		err = ftpDownloadFile(path.Join("/", subDir2, "file.zip"), localDownloadPath, 5, client, 0)
+		assert.Error(t, err)
+		err = ftpDownloadFile("/missing.zip", localDownloadPath, 5, client, 0)
+		assert.Error(t, err)
+		err = client.Quit()
+		assert.NoError(t, err)
+		err = os.Remove(localDownloadPath)
+		assert.NoError(t, err)
+	}
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+}
+
+func TestUploadErrors(t *testing.T) {
+	u := getTestUser()
+	u.QuotaSize = 65535
+	subDir1 := "sub1"
+	subDir2 := "sub2"
+	u.Permissions[path.Join("/", subDir1)] = []string{dataprovider.PermListItems}
+	u.Permissions[path.Join("/", subDir2)] = []string{dataprovider.PermListItems, dataprovider.PermUpload,
+		dataprovider.PermDelete}
+	u.Filters.FileExtensions = []dataprovider.ExtensionsFilter{
+		{
+			Path:              "/sub2",
+			AllowedExtensions: []string{},
+			DeniedExtensions:  []string{".zip"},
+		},
+	}
+	user, _, err := httpd.AddUser(u, http.StatusOK)
+	assert.NoError(t, err)
+	client, err := getFTPClient(user, true)
+	if assert.NoError(t, err) {
+		testFileName := "test_file.dat"
+		testFilePath := filepath.Join(homeBasePath, testFileName)
+		testFileSize := user.QuotaSize
+		err = createTestFile(testFilePath, testFileSize)
+		assert.NoError(t, err)
+		err = client.MakeDir(subDir1)
+		assert.NoError(t, err)
+		err = client.MakeDir(subDir2)
+		assert.NoError(t, err)
+		err = client.ChangeDir(subDir1)
+		assert.NoError(t, err)
+		err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0)
+		assert.Error(t, err)
+		err = client.ChangeDirToParent()
+		assert.NoError(t, err)
+		err = client.ChangeDir(subDir2)
+		assert.NoError(t, err)
+		err = ftpUploadFile(testFilePath, testFileName+".zip", testFileSize, client, 0)
+		assert.Error(t, err)
+		err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0)
+		assert.NoError(t, err)
+		err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0)
+		assert.Error(t, err)
+		err = client.ChangeDir("/")
+		assert.NoError(t, err)
+		err = ftpUploadFile(testFilePath, subDir1, testFileSize, client, 0)
+		assert.Error(t, err)
+		// overquota
+		err = ftpUploadFile(testFilePath, testFileName+"1", testFileSize, client, 0)
+		assert.Error(t, err)
+		err = client.Delete(path.Join("/", subDir2, testFileName))
+		assert.NoError(t, err)
+		err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0)
+		assert.NoError(t, err)
+		err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0)
+		assert.Error(t, err)
+		err = client.Quit()
+		assert.NoError(t, err)
+	}
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+}
+
+func TestResume(t *testing.T) {
+	u := getTestUser()
+	user, _, err := httpd.AddUser(u, http.StatusOK)
+	assert.NoError(t, err)
+	client, err := getFTPClient(user, true)
+	if assert.NoError(t, err) {
+		testFileName := "test_file.dat"
+		testFilePath := filepath.Join(homeBasePath, testFileName)
+		data := []byte("test data")
+		err = ioutil.WriteFile(testFilePath, data, os.ModePerm)
+		assert.NoError(t, err)
+		err = ftpUploadFile(testFilePath, testFileName, int64(len(data)), client, 0)
+		assert.NoError(t, err)
+		err = ftpUploadFile(testFilePath, testFileName, int64(len(data)+5), client, 5)
+		assert.NoError(t, err)
+		readed, err := ioutil.ReadFile(filepath.Join(user.GetHomeDir(), testFileName))
+		assert.NoError(t, err)
+		assert.Equal(t, "test test data", string(readed))
+		localDownloadPath := filepath.Join(homeBasePath, "test_download.dat")
+		err = ftpDownloadFile(testFileName, localDownloadPath, int64(len(data)), client, 5)
+		assert.NoError(t, err)
+		readed, err = ioutil.ReadFile(localDownloadPath)
+		assert.NoError(t, err)
+		assert.Equal(t, data, readed)
+		err = client.Delete(testFileName)
+		assert.NoError(t, err)
+		err = ftpUploadFile(testFilePath, testFileName, int64(len(data)), client, 0)
+		assert.NoError(t, err)
+		// now append to a file
+		srcFile, err := os.Open(testFilePath)
+		if assert.NoError(t, err) {
+			err = client.Append(testFileName, srcFile)
+			assert.NoError(t, err)
+			err = srcFile.Close()
+			assert.NoError(t, err)
+			size, err := client.FileSize(testFileName)
+			assert.NoError(t, err)
+			assert.Equal(t, int64(2*len(data)), size)
+			err = ftpDownloadFile(testFileName, localDownloadPath, int64(2*len(data)), client, 0)
+			assert.NoError(t, err)
+			readed, err = ioutil.ReadFile(localDownloadPath)
+			assert.NoError(t, err)
+			expected := append(data, data...)
+			assert.Equal(t, expected, readed)
+		}
+		err = client.Quit()
+		assert.NoError(t, err)
+		err = os.Remove(testFilePath)
+		assert.NoError(t, err)
+		err = os.Remove(localDownloadPath)
+		assert.NoError(t, err)
+	}
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+}
+
+func TestQuotaLimits(t *testing.T) {
+	u := getTestUser()
+	u.QuotaFiles = 1
+	user, _, err := httpd.AddUser(u, http.StatusOK)
+	assert.NoError(t, err)
+	testFileSize := int64(65535)
+	testFileName := "test_file.dat"
+	testFilePath := filepath.Join(homeBasePath, testFileName)
+	err = createTestFile(testFilePath, testFileSize)
+	assert.NoError(t, err)
+	testFileSize1 := int64(131072)
+	testFileName1 := "test_file1.dat"
+	testFilePath1 := filepath.Join(homeBasePath, testFileName1)
+	err = createTestFile(testFilePath1, testFileSize1)
+	assert.NoError(t, err)
+	testFileSize2 := int64(32768)
+	testFileName2 := "test_file2.dat"
+	testFilePath2 := filepath.Join(homeBasePath, testFileName2)
+	err = createTestFile(testFilePath2, testFileSize2)
+	assert.NoError(t, err)
+	// test quota files
+	client, err := getFTPClient(user, false)
+	if assert.NoError(t, err) {
+		err = ftpUploadFile(testFilePath, testFileName+".quota", testFileSize, client, 0)
+		assert.NoError(t, err)
+		err = ftpUploadFile(testFilePath, testFileName+".quota1", testFileSize, client, 0)
+		assert.Error(t, err)
+		err = client.Rename(testFileName+".quota", testFileName)
+		assert.NoError(t, err)
+		err = client.Quit()
+		assert.NoError(t, err)
+	}
+	// test quota size
+	user.QuotaSize = testFileSize - 1
+	user.QuotaFiles = 0
+	user, _, err = httpd.UpdateUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	client, err = getFTPClient(user, true)
+	if assert.NoError(t, err) {
+		err = ftpUploadFile(testFilePath, testFileName+".quota", testFileSize, client, 0)
+		assert.Error(t, err)
+		err = client.Rename(testFileName, testFileName+".quota")
+		assert.NoError(t, err)
+		err = client.Quit()
+		assert.NoError(t, err)
+	}
+	// now test quota limits while uploading the current file, we have 1 bytes remaining
+	user.QuotaSize = testFileSize + 1
+	user.QuotaFiles = 0
+	user, _, err = httpd.UpdateUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	client, err = getFTPClient(user, false)
+	if assert.NoError(t, err) {
+		err = ftpUploadFile(testFilePath1, testFileName1, testFileSize1, client, 0)
+		assert.Error(t, err)
+		_, err = client.FileSize(testFileName1)
+		assert.Error(t, err)
+		err = client.Rename(testFileName+".quota", testFileName)
+		assert.NoError(t, err)
+		// overwriting an existing file will work if the resulting size is lesser or equal than the current one
+		err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0)
+		assert.NoError(t, err)
+		err = ftpUploadFile(testFilePath2, testFileName, testFileSize2, client, 0)
+		assert.NoError(t, err)
+		err = ftpUploadFile(testFilePath1, testFileName, testFileSize1, client, 0)
+		assert.Error(t, err)
+		err = ftpUploadFile(testFilePath1, testFileName, testFileSize1, client, 10)
+		assert.Error(t, err)
+		err = ftpUploadFile(testFilePath2, testFileName, testFileSize2, client, 0)
+		assert.NoError(t, err)
+		err = client.Quit()
+		assert.NoError(t, err)
+	}
+
+	err = os.Remove(testFilePath)
+	assert.NoError(t, err)
+	err = os.Remove(testFilePath1)
+	assert.NoError(t, err)
+	err = os.Remove(testFilePath2)
+	assert.NoError(t, err)
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+}
+
+func TestLoginWithIPilters(t *testing.T) {
+	u := getTestUser()
+	u.Filters.DeniedIP = []string{"192.167.0.0/24", "172.18.0.0/16"}
+	u.Filters.AllowedIP = []string{"172.19.0.0/16"}
+	user, _, err := httpd.AddUser(u, http.StatusOK)
+	assert.NoError(t, err)
+	client, err := getFTPClient(user, true)
+	if !assert.Error(t, err) {
+		err = client.Quit()
+		assert.NoError(t, err)
+	}
+
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+}
+
+func TestLoginInvalidFs(t *testing.T) {
+	u := getTestUser()
+	u.FsConfig.Provider = 2
+	u.FsConfig.GCSConfig.Bucket = "test"
+	u.FsConfig.GCSConfig.Credentials = base64.StdEncoding.EncodeToString([]byte("invalid JSON for credentials"))
+	user, _, err := httpd.AddUser(u, http.StatusOK)
+	assert.NoError(t, err)
+	// now remove the credentials file so the filesystem creation will fail
+	providerConf := config.GetProviderConf()
+	credentialsFile := filepath.Join(providerConf.CredentialsPath, fmt.Sprintf("%v_gcs_credentials.json", u.Username))
+	if !filepath.IsAbs(credentialsFile) {
+		credentialsFile = filepath.Join(configDir, credentialsFile)
+	}
+	err = os.Remove(credentialsFile)
+	assert.NoError(t, err)
+	client, err := getFTPClient(user, false)
+	if !assert.Error(t, err) {
+		err = client.Quit()
+		assert.NoError(t, err)
+	}
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+}
+
+func TestClientClose(t *testing.T) {
+	u := getTestUser()
+	user, _, err := httpd.AddUser(u, http.StatusOK)
+	assert.NoError(t, err)
+	client, err := getFTPClient(user, true)
+	if assert.NoError(t, err) {
+		err = checkBasicFTP(client)
+		assert.NoError(t, err)
+		stats := common.Connections.GetStats()
+		if assert.Len(t, stats, 1) {
+			common.Connections.Close(stats[0].ConnectionID)
+			assert.Eventually(t, func() bool { return len(common.Connections.GetStats()) == 0 },
+				1*time.Second, 50*time.Millisecond)
+		}
+	}
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+}
+
+func TestRename(t *testing.T) {
+	u := getTestUser()
+	user, _, err := httpd.AddUser(u, http.StatusOK)
+	assert.NoError(t, err)
+	testDir := "adir"
+	testFileName := "test_file.dat"
+	testFilePath := filepath.Join(homeBasePath, testFileName)
+	testFileSize := int64(65535)
+	err = createTestFile(testFilePath, testFileSize)
+	assert.NoError(t, err)
+	client, err := getFTPClient(user, false)
+	if assert.NoError(t, err) {
+		assert.NoError(t, err)
+		err = checkBasicFTP(client)
+		assert.NoError(t, err)
+		err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0)
+		assert.NoError(t, err)
+		err = client.MakeDir(testDir)
+		assert.NoError(t, err)
+		err = client.Rename(testFileName, path.Join("missing", testFileName))
+		assert.Error(t, err)
+		err = client.Rename(testFileName, path.Join(testDir, testFileName))
+		assert.NoError(t, err)
+		size, err := client.FileSize(path.Join(testDir, testFileName))
+		assert.NoError(t, err)
+		assert.Equal(t, testFileSize, size)
+		if runtime.GOOS != osWindows {
+			otherDir := "dir"
+			err = client.MakeDir(otherDir)
+			assert.NoError(t, err)
+			err = client.MakeDir(path.Join(otherDir, testDir))
+			assert.NoError(t, err)
+			code, response, err := client.SendCustomCommand(fmt.Sprintf("SITE CHMOD 0001 %v", otherDir))
+			assert.NoError(t, err)
+			assert.Equal(t, ftp.StatusCommandOK, code)
+			assert.Equal(t, "SITE CHMOD command successful", response)
+			err = client.Rename(testDir, path.Join(otherDir, testDir))
+			assert.Error(t, err)
+
+			code, response, err = client.SendCustomCommand(fmt.Sprintf("SITE CHMOD 755 %v", otherDir))
+			assert.NoError(t, err)
+			assert.Equal(t, ftp.StatusCommandOK, code)
+			assert.Equal(t, "SITE CHMOD command successful", response)
+		}
+		err = client.Quit()
+		assert.NoError(t, err)
+	}
+	user.Permissions[path.Join("/", testDir)] = []string{dataprovider.PermListItems}
+	user, _, err = httpd.UpdateUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	client, err = getFTPClient(user, false)
+	if assert.NoError(t, err) {
+		defer func() {
+			err := client.Quit()
+			assert.NoError(t, err)
+		}()
+
+		err = client.Rename(path.Join(testDir, testFileName), testFileName)
+		assert.Error(t, err)
+	}
+
+	err = os.Remove(testFilePath)
+	assert.NoError(t, err)
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+}
+
+func TestStat(t *testing.T) {
+	u := getTestUser()
+	u.Permissions["/subdir"] = []string{dataprovider.PermUpload}
+	user, _, err := httpd.AddUser(u, http.StatusOK)
+	assert.NoError(t, err)
+	client, err := getFTPClient(user, false)
+	if assert.NoError(t, err) {
+		defer func() {
+			err := client.Quit()
+			assert.NoError(t, err)
+		}()
+
+		subDir := "subdir"
+		testFileName := "test_file.dat"
+		testFilePath := filepath.Join(homeBasePath, testFileName)
+		testFileSize := int64(65535)
+		err = createTestFile(testFilePath, testFileSize)
+		assert.NoError(t, err)
+		err = client.MakeDir(subDir)
+		assert.NoError(t, err)
+		err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0)
+		assert.NoError(t, err)
+		err = ftpUploadFile(testFilePath, path.Join("/", subDir, testFileName), testFileSize, client, 0)
+		assert.Error(t, err)
+		size, err := client.FileSize(testFileName)
+		assert.NoError(t, err)
+		assert.Equal(t, testFileSize, size)
+		_, err = client.FileSize(path.Join("/", subDir, testFileName))
+		assert.Error(t, err)
+		_, err = client.FileSize("missing file")
+		assert.Error(t, err)
+
+		err = os.Remove(testFilePath)
+		assert.NoError(t, err)
+	}
+
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+}
+
+func TestUploadOverwriteVfolder(t *testing.T) {
+	u := getTestUser()
+	vdir := "/vdir"
+	mappedPath := filepath.Join(os.TempDir(), "vdir")
+	u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{
+		BaseVirtualFolder: vfs.BaseVirtualFolder{
+			MappedPath: mappedPath,
+		},
+		VirtualPath: vdir,
+		QuotaSize:   -1,
+		QuotaFiles:  -1,
+	})
+	err := os.MkdirAll(mappedPath, os.ModePerm)
+	assert.NoError(t, err)
+	user, _, err := httpd.AddUser(u, http.StatusOK)
+	assert.NoError(t, err)
+	client, err := getFTPClient(user, false)
+	if assert.NoError(t, err) {
+		testFileName := "test_file.dat"
+		testFilePath := filepath.Join(homeBasePath, testFileName)
+		testFileSize := int64(65535)
+		err = createTestFile(testFilePath, testFileSize)
+		assert.NoError(t, err)
+		err = ftpUploadFile(testFilePath, path.Join(vdir, testFileName), testFileSize, client, 0)
+		assert.NoError(t, err)
+		folder, _, err := httpd.GetFolders(0, 0, mappedPath, http.StatusOK)
+		assert.NoError(t, err)
+		if assert.Len(t, folder, 1) {
+			f := folder[0]
+			assert.Equal(t, testFileSize, f.UsedQuotaSize)
+			assert.Equal(t, 1, f.UsedQuotaFiles)
+		}
+		err = ftpUploadFile(testFilePath, path.Join(vdir, testFileName), testFileSize, client, 0)
+		assert.NoError(t, err)
+		folder, _, err = httpd.GetFolders(0, 0, mappedPath, http.StatusOK)
+		assert.NoError(t, err)
+		if assert.Len(t, folder, 1) {
+			f := folder[0]
+			assert.Equal(t, testFileSize, f.UsedQuotaSize)
+			assert.Equal(t, 1, f.UsedQuotaFiles)
+		}
+		err = client.Quit()
+		assert.NoError(t, err)
+		err = os.Remove(testFilePath)
+		assert.NoError(t, err)
+	}
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	_, err = httpd.RemoveFolder(vfs.BaseVirtualFolder{MappedPath: mappedPath}, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+	err = os.RemoveAll(mappedPath)
+	assert.NoError(t, err)
+}
+
+func TestAllocate(t *testing.T) {
+	u := getTestUser()
+	mappedPath := filepath.Join(os.TempDir(), "vdir")
+	u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{
+		BaseVirtualFolder: vfs.BaseVirtualFolder{
+			MappedPath: mappedPath,
+		},
+		VirtualPath: "/vdir",
+		QuotaSize:   110,
+	})
+	err := os.MkdirAll(mappedPath, os.ModePerm)
+	assert.NoError(t, err)
+	user, _, err := httpd.AddUser(u, http.StatusOK)
+	assert.NoError(t, err)
+	client, err := getFTPClient(user, false)
+	if assert.NoError(t, err) {
+		code, response, err := client.SendCustomCommand("allo 2000000")
+		assert.NoError(t, err)
+		assert.Equal(t, ftp.StatusCommandOK, code)
+		assert.Equal(t, "Done !", response)
+		err = client.Quit()
+		assert.NoError(t, err)
+	}
+	user.QuotaSize = 100
+	user, _, err = httpd.UpdateUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	client, err = getFTPClient(user, false)
+	if assert.NoError(t, err) {
+		testFileName := "test_file.dat"
+		testFilePath := filepath.Join(homeBasePath, testFileName)
+		testFileSize := user.QuotaSize - 1
+		err = createTestFile(testFilePath, testFileSize)
+		assert.NoError(t, err)
+		code, response, err := client.SendCustomCommand("allo 99")
+		assert.NoError(t, err)
+		assert.Equal(t, ftp.StatusCommandOK, code)
+		assert.Equal(t, "Done !", response)
+		code, response, err = client.SendCustomCommand("allo 100")
+		assert.NoError(t, err)
+		assert.Equal(t, ftp.StatusCommandOK, code)
+		assert.Equal(t, "Done !", response)
+		code, response, err = client.SendCustomCommand("allo 150")
+		assert.NoError(t, err)
+		assert.Equal(t, ftp.StatusFileUnavailable, code)
+		assert.Contains(t, response, common.ErrQuotaExceeded.Error())
+
+		err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0)
+		assert.NoError(t, err)
+		// we still have space in vdir
+		code, response, err = client.SendCustomCommand("allo 50")
+		assert.NoError(t, err)
+		assert.Equal(t, ftp.StatusCommandOK, code)
+		assert.Equal(t, "Done !", response)
+		err = ftpUploadFile(testFilePath, path.Join("/vdir", testFileName), testFileSize, client, 0)
+		assert.NoError(t, err)
+		code, response, err = client.SendCustomCommand("allo 50")
+		assert.NoError(t, err)
+		assert.Equal(t, ftp.StatusFileUnavailable, code)
+		assert.Contains(t, response, common.ErrQuotaExceeded.Error())
+
+		err = client.Quit()
+		assert.NoError(t, err)
+		err = os.Remove(testFilePath)
+		assert.NoError(t, err)
+	}
+
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	_, err = httpd.RemoveFolder(vfs.BaseVirtualFolder{MappedPath: mappedPath}, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+	err = os.RemoveAll(mappedPath)
+	assert.NoError(t, err)
+}
+
+func TestChtimes(t *testing.T) {
+	u := getTestUser()
+	user, _, err := httpd.AddUser(u, http.StatusOK)
+	assert.NoError(t, err)
+	client, err := getFTPClient(user, false)
+	if assert.NoError(t, err) {
+		defer func() {
+			err := client.Quit()
+			assert.NoError(t, err)
+		}()
+
+		testFileName := "test_file.dat"
+		testFilePath := filepath.Join(homeBasePath, testFileName)
+		testFileSize := int64(65535)
+		err = createTestFile(testFilePath, testFileSize)
+		assert.NoError(t, err)
+		err = checkBasicFTP(client)
+		assert.NoError(t, err)
+		err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0)
+		assert.NoError(t, err)
+
+		mtime := time.Now().Format("20060102150405")
+		code, response, err := client.SendCustomCommand(fmt.Sprintf("MFMT %v %v", mtime, testFileName))
+		assert.NoError(t, err)
+		assert.Equal(t, ftp.StatusFile, code)
+		assert.Equal(t, fmt.Sprintf("Modify=%v; %v", mtime, testFileName), response)
+
+		err = os.Remove(testFilePath)
+		assert.NoError(t, err)
+	}
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+}
+
+func TestChmod(t *testing.T) {
+	if runtime.GOOS == osWindows {
+		t.Skip("chmod is partially supported on Windows")
+	}
+	u := getTestUser()
+	user, _, err := httpd.AddUser(u, http.StatusOK)
+	assert.NoError(t, err)
+	client, err := getFTPClient(user, true)
+	if assert.NoError(t, err) {
+		defer func() {
+			err := client.Quit()
+			assert.NoError(t, err)
+		}()
+
+		testFileName := "test_file.dat"
+		testFilePath := filepath.Join(homeBasePath, testFileName)
+		testFileSize := int64(131072)
+		err = createTestFile(testFilePath, testFileSize)
+		assert.NoError(t, err)
+		err = checkBasicFTP(client)
+		assert.NoError(t, err)
+		err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0)
+		assert.NoError(t, err)
+
+		code, response, err := client.SendCustomCommand(fmt.Sprintf("SITE CHMOD 600 %v", testFileName))
+		assert.NoError(t, err)
+		assert.Equal(t, ftp.StatusCommandOK, code)
+		assert.Equal(t, "SITE CHMOD command successful", response)
+
+		fi, err := os.Stat(filepath.Join(user.HomeDir, testFileName))
+		if assert.NoError(t, err) {
+			assert.Equal(t, os.FileMode(0600), fi.Mode().Perm())
+		}
+
+		err = os.Remove(testFilePath)
+		assert.NoError(t, err)
+	}
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+}
+
+func checkBasicFTP(client *ftp.ServerConn) error {
+	_, err := client.CurrentDir()
+	if err != nil {
+		return err
+	}
+	err = client.NoOp()
+	if err != nil {
+		return err
+	}
+	_, err = client.List(".")
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func ftpUploadFile(localSourcePath string, remoteDestPath string, expectedSize int64, client *ftp.ServerConn, offset uint64) error {
+	srcFile, err := os.Open(localSourcePath)
+	if err != nil {
+		return err
+	}
+	defer srcFile.Close()
+	if offset > 0 {
+		err = client.StorFrom(remoteDestPath, srcFile, offset)
+	} else {
+		err = client.Stor(remoteDestPath, srcFile)
+	}
+	if err != nil {
+		return err
+	}
+	if expectedSize > 0 {
+		size, err := client.FileSize(remoteDestPath)
+		if err != nil {
+			return err
+		}
+		if size != expectedSize {
+			return fmt.Errorf("uploaded file size does not match, actual: %v, expected: %v", size, expectedSize)
+		}
+	}
+	return nil
+}
+
+func ftpDownloadFile(remoteSourcePath string, localDestPath string, expectedSize int64, client *ftp.ServerConn, offset uint64) error {
+	downloadDest, err := os.Create(localDestPath)
+	if err != nil {
+		return err
+	}
+	defer downloadDest.Close()
+	var r *ftp.Response
+	if offset > 0 {
+		r, err = client.RetrFrom(remoteSourcePath, offset)
+	} else {
+		r, err = client.Retr(remoteSourcePath)
+	}
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+
+	written, err := io.Copy(downloadDest, r)
+	if err != nil {
+		return err
+	}
+	if written != expectedSize {
+		return fmt.Errorf("downloaded file size does not match, actual: %v, expected: %v", written, expectedSize)
+	}
+	return nil
+}
+
+func getFTPClient(user dataprovider.User, useTLS bool) (*ftp.ServerConn, error) {
+	ftpOptions := []ftp.DialOption{ftp.DialWithTimeout(5 * time.Second)}
+	if useTLS {
+		tlsConfig := &tls.Config{
+			ServerName:         "localhost",
+			InsecureSkipVerify: true, // use this for tests only
+		}
+		ftpOptions = append(ftpOptions, ftp.DialWithExplicitTLS(tlsConfig))
+	}
+	client, err := ftp.Dial(ftpServerAddr, ftpOptions...)
+	if err != nil {
+		return nil, err
+	}
+	pwd := defaultPassword
+	if len(user.Password) > 0 {
+		pwd = user.Password
+	}
+	err = client.Login(user.Username, pwd)
+	if err != nil {
+		return nil, err
+	}
+	return client, err
+}
+
+func waitTCPListening(address string) {
+	for {
+		conn, err := net.Dial("tcp", address)
+		if err != nil {
+			logger.WarnToConsole("tcp server %v not listening: %v\n", address, err)
+			time.Sleep(100 * time.Millisecond)
+			continue
+		}
+		logger.InfoToConsole("tcp server %v now listening\n", address)
+		conn.Close()
+		break
+	}
+}
+
+func getTestUser() dataprovider.User {
+	user := dataprovider.User{
+		Username:       defaultUsername,
+		Password:       defaultPassword,
+		HomeDir:        filepath.Join(homeBasePath, defaultUsername),
+		Status:         1,
+		ExpirationDate: 0,
+	}
+	user.Permissions = make(map[string][]string)
+	user.Permissions["/"] = allPerms
+	return user
+}
+
+func getExtAuthScriptContent(user dataprovider.User, nonJSONResponse bool, username string) []byte {
+	extAuthContent := []byte("#!/bin/sh\n\n")
+	extAuthContent = append(extAuthContent, []byte(fmt.Sprintf("if test \"$SFTPGO_AUTHD_USERNAME\" = \"%v\"; then\n", user.Username))...)
+	if len(username) > 0 {
+		user.Username = username
+	}
+	u, _ := json.Marshal(user)
+	if nonJSONResponse {
+		extAuthContent = append(extAuthContent, []byte("echo 'text response'\n")...)
+	} else {
+		extAuthContent = append(extAuthContent, []byte(fmt.Sprintf("echo '%v'\n", string(u)))...)
+	}
+	extAuthContent = append(extAuthContent, []byte("else\n")...)
+	if nonJSONResponse {
+		extAuthContent = append(extAuthContent, []byte("echo 'text response'\n")...)
+	} else {
+		extAuthContent = append(extAuthContent, []byte("echo '{\"username\":\"\"}'\n")...)
+	}
+	extAuthContent = append(extAuthContent, []byte("fi\n")...)
+	return extAuthContent
+}
+
+func getPreLoginScriptContent(user dataprovider.User, nonJSONResponse bool) []byte {
+	content := []byte("#!/bin/sh\n\n")
+	if nonJSONResponse {
+		content = append(content, []byte("echo 'text response'\n")...)
+		return content
+	}
+	if len(user.Username) > 0 {
+		u, _ := json.Marshal(user)
+		content = append(content, []byte(fmt.Sprintf("echo '%v'\n", string(u)))...)
+	}
+	return content
+}
+
+func createTestFile(path string, size int64) error {
+	baseDir := filepath.Dir(path)
+	if _, err := os.Stat(baseDir); os.IsNotExist(err) {
+		err = os.MkdirAll(baseDir, os.ModePerm)
+		if err != nil {
+			return err
+		}
+	}
+	content := make([]byte, size)
+	_, err := rand.Read(content)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(path, content, os.ModePerm)
+}

ftpd/handler.go

@@ -0,0 +1,387 @@
+package ftpd
+
+import (
+	"errors"
+	"os"
+	"path"
+	"time"
+
+	ftpserver "github.com/fclairamb/ftpserverlib"
+	"github.com/spf13/afero"
+
+	"github.com/drakkan/sftpgo/common"
+	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/vfs"
+)
+
+var (
+	errNotImplemented = errors.New("Not implemented")
+)
+
+// Connection details for an FTP connection.
+// It implements common.ActiveConnection and ftpserver.ClientDriver interfaces
+type Connection struct {
+	*common.BaseConnection
+	clientContext ftpserver.ClientContext
+}
+
+// GetClientVersion returns the connected client's version.
+// It returns "Unknown" if the client does not advertise its
+// version
+func (c *Connection) GetClientVersion() string {
+	version := c.clientContext.GetClientVersion()
+	if len(version) > 0 {
+		return version
+	}
+	return "Unknown"
+}
+
+// GetRemoteAddress return the connected client's address
+func (c *Connection) GetRemoteAddress() string {
+	return c.clientContext.RemoteAddr().String()
+}
+
+// SetConnDeadline does nothing
+func (c *Connection) SetConnDeadline() {}
+
+// Disconnect disconnects the client
+func (c *Connection) Disconnect() error {
+	return c.clientContext.Close(ftpserver.StatusServiceNotAvailable, "connection closed")
+}
+
+// GetCommand returns an empty string
+func (c *Connection) GetCommand() string {
+	return ""
+}
+
+// Create is not implemented we use ClientDriverExtentionFileTransfer
+func (c *Connection) Create(name string) (afero.File, error) {
+	return nil, errNotImplemented
+}
+
+// Mkdir creates a directory using the connection filesystem
+func (c *Connection) Mkdir(name string, perm os.FileMode) error {
+	c.UpdateLastActivity()
+
+	p, err := c.Fs.ResolvePath(name)
+	if err != nil {
+		return c.GetFsError(err)
+	}
+	return c.CreateDir(p, name)
+}
+
+// MkdirAll is not implemented, we don't need it
+func (c *Connection) MkdirAll(path string, perm os.FileMode) error {
+	return errNotImplemented
+}
+
+// Open is not implemented we use ClientDriverExtentionFileTransfer and ClientDriverExtensionFileList
+func (c *Connection) Open(name string) (afero.File, error) {
+	return nil, errNotImplemented
+}
+
+// OpenFile is not implemented we use ClientDriverExtentionFileTransfer
+func (c *Connection) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
+	return nil, errNotImplemented
+}
+
+// Remove removes a file or an empty directory
+func (c *Connection) Remove(name string) error {
+	c.UpdateLastActivity()
+
+	p, err := c.Fs.ResolvePath(name)
+	if err != nil {
+		return c.GetFsError(err)
+	}
+
+	var fi os.FileInfo
+	if fi, err = c.Fs.Lstat(p); err != nil {
+		c.Log(logger.LevelWarn, "failed to remove a file %#v: stat error: %+v", p, err)
+		return c.GetFsError(err)
+	}
+
+	if fi.IsDir() && fi.Mode()&os.ModeSymlink != os.ModeSymlink {
+		return c.RemoveDir(p, name)
+	}
+	return c.RemoveFile(p, name, fi)
+}
+
+// RemoveAll is not implemented, we don't need it
+func (c *Connection) RemoveAll(path string) error {
+	return errNotImplemented
+}
+
+// Rename renames a file or a directory
+func (c *Connection) Rename(oldname, newname string) error {
+	c.UpdateLastActivity()
+
+	p, err := c.Fs.ResolvePath(oldname)
+	if err != nil {
+		return c.GetFsError(err)
+	}
+	t, err := c.Fs.ResolvePath(newname)
+	if err != nil {
+		return c.GetFsError(err)
+	}
+
+	if err = c.BaseConnection.Rename(p, t, oldname, newname); err != nil {
+		return err
+	}
+
+	vfs.SetPathPermissions(c.Fs, t, c.User.GetUID(), c.User.GetGID())
+	return nil
+}
+
+// Stat returns a FileInfo describing the named file/directory, or an error,
+// if any happens
+func (c *Connection) Stat(name string) (os.FileInfo, error) {
+	c.UpdateLastActivity()
+
+	if !c.User.HasPerm(dataprovider.PermListItems, path.Dir(name)) {
+		return nil, c.GetPermissionDeniedError()
+	}
+
+	p, err := c.Fs.ResolvePath(name)
+	if err != nil {
+		return nil, c.GetFsError(err)
+	}
+	fi, err := c.Fs.Stat(p)
+	if err != nil {
+		c.Log(logger.LevelWarn, "error running stat on path: %+v", err)
+		return nil, c.GetFsError(err)
+	}
+	return fi, nil
+}
+
+// Name returns the name of this connection
+func (c *Connection) Name() string {
+	return c.GetID()
+}
+
+// Chmod changes the mode of the named file/directory
+func (c *Connection) Chmod(name string, mode os.FileMode) error {
+	c.UpdateLastActivity()
+
+	p, err := c.Fs.ResolvePath(name)
+	if err != nil {
+		return c.GetFsError(err)
+	}
+	attrs := common.StatAttributes{
+		Flags: common.StatAttrPerms,
+		Mode:  mode,
+	}
+	return c.SetStat(p, name, &attrs)
+}
+
+// Chtimes changes the access and modification times of the named file
+func (c *Connection) Chtimes(name string, atime time.Time, mtime time.Time) error {
+	c.UpdateLastActivity()
+
+	p, err := c.Fs.ResolvePath(name)
+	if err != nil {
+		return c.GetFsError(err)
+	}
+	attrs := common.StatAttributes{
+		Flags: common.StatAttrTimes,
+		Atime: atime,
+		Mtime: mtime,
+	}
+	return c.SetStat(p, name, &attrs)
+}
+
+// AllocateSpace implements ClientDriverExtensionAllocate
+func (c *Connection) AllocateSpace(size int) error {
+	c.UpdateLastActivity()
+	// we don't have a path here so we check home dir and any virtual folders
+	// we return no error if there is space in any folder
+	folders := []string{"/"}
+	for _, v := range c.User.VirtualFolders {
+		// the space is checked for the parent folder
+		folders = append(folders, path.Join(v.VirtualPath, "fakefile.txt"))
+	}
+	for _, f := range folders {
+		quotaResult := c.HasSpace(false, f)
+		if quotaResult.HasSpace {
+			if quotaResult.QuotaSize == 0 {
+				// unlimited size is allowed
+				return nil
+			}
+			if quotaResult.GetRemainingSize() > int64(size) {
+				return nil
+			}
+		}
+	}
+	return common.ErrQuotaExceeded
+}
+
+// ReadDir implements ClientDriverExtensionFilelist
+func (c *Connection) ReadDir(name string) ([]os.FileInfo, error) {
+	c.UpdateLastActivity()
+
+	p, err := c.Fs.ResolvePath(name)
+	if err != nil {
+		return nil, c.GetFsError(err)
+	}
+	return c.ListDir(p, name)
+}
+
+// GetHandle implements ClientDriverExtentionFileTransfer
+func (c *Connection) GetHandle(name string, flags int) (ftpserver.FileTransfer, error) {
+	c.UpdateLastActivity()
+
+	p, err := c.Fs.ResolvePath(name)
+	if err != nil {
+		return nil, c.GetFsError(err)
+	}
+	if flags&os.O_WRONLY != 0 {
+		return c.uploadFile(p, name, flags)
+	}
+	return c.downloadFile(p, name)
+}
+
+func (c *Connection) downloadFile(fsPath, ftpPath string) (ftpserver.FileTransfer, error) {
+	if !c.User.HasPerm(dataprovider.PermDownload, path.Dir(ftpPath)) {
+		return nil, c.GetPermissionDeniedError()
+	}
+
+	if !c.User.IsFileAllowed(ftpPath) {
+		c.Log(logger.LevelWarn, "reading file %#v is not allowed", ftpPath)
+		return nil, c.GetPermissionDeniedError()
+	}
+
+	file, r, cancelFn, err := c.Fs.Open(fsPath)
+	if err != nil {
+		c.Log(logger.LevelWarn, "could not open file %#v for reading: %+v", fsPath, err)
+		return nil, c.GetFsError(err)
+	}
+
+	baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, fsPath, ftpPath, common.TransferDownload,
+		0, 0, false)
+	t := newTransfer(baseTransfer, nil, r, 0)
+
+	return t, nil
+}
+
+func (c *Connection) uploadFile(fsPath, ftpPath string, flags int) (ftpserver.FileTransfer, error) {
+	if !c.User.IsFileAllowed(ftpPath) {
+		c.Log(logger.LevelWarn, "writing file %#v is not allowed", ftpPath)
+		return nil, c.GetPermissionDeniedError()
+	}
+
+	filePath := fsPath
+	if common.Config.IsAtomicUploadEnabled() && c.Fs.IsAtomicUploadSupported() {
+		filePath = c.Fs.GetAtomicUploadPath(fsPath)
+	}
+
+	stat, statErr := c.Fs.Lstat(fsPath)
+	if (statErr == nil && stat.Mode()&os.ModeSymlink == os.ModeSymlink) || c.Fs.IsNotExist(statErr) {
+		if !c.User.HasPerm(dataprovider.PermUpload, path.Dir(ftpPath)) {
+			return nil, c.GetPermissionDeniedError()
+		}
+		return c.handleFTPUploadToNewFile(fsPath, filePath, ftpPath)
+	}
+
+	if statErr != nil {
+		c.Log(logger.LevelError, "error performing file stat %#v: %+v", fsPath, statErr)
+		return nil, c.GetFsError(statErr)
+	}
+
+	// This happen if we upload a file that has the same name of an existing directory
+	if stat.IsDir() {
+		c.Log(logger.LevelWarn, "attempted to open a directory for writing to: %#v", fsPath)
+		return nil, c.GetOpUnsupportedError()
+	}
+
+	if !c.User.HasPerm(dataprovider.PermOverwrite, path.Dir(ftpPath)) {
+		return nil, c.GetPermissionDeniedError()
+	}
+
+	return c.handleFTPUploadToExistingFile(flags, fsPath, filePath, stat.Size(), ftpPath)
+}
+
+func (c *Connection) handleFTPUploadToNewFile(resolvedPath, filePath, requestPath string) (ftpserver.FileTransfer, error) {
+	quotaResult := c.HasSpace(true, requestPath)
+	if !quotaResult.HasSpace {
+		c.Log(logger.LevelInfo, "denying file write due to quota limits")
+		return nil, common.ErrQuotaExceeded
+	}
+	file, w, cancelFn, err := c.Fs.Create(filePath, 0)
+	if err != nil {
+		c.Log(logger.LevelWarn, "error creating file %#v: %+v", resolvedPath, err)
+		return nil, c.GetFsError(err)
+	}
+
+	vfs.SetPathPermissions(c.Fs, filePath, c.User.GetUID(), c.User.GetGID())
+
+	baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, resolvedPath, requestPath,
+		common.TransferUpload, 0, 0, true)
+	t := newTransfer(baseTransfer, w, nil, quotaResult.GetRemainingSize())
+
+	return t, nil
+}
+
+func (c *Connection) handleFTPUploadToExistingFile(flags int, resolvedPath, filePath string, fileSize int64,
+	requestPath string) (ftpserver.FileTransfer, error) {
+	var err error
+	quotaResult := c.HasSpace(false, requestPath)
+	if !quotaResult.HasSpace {
+		c.Log(logger.LevelInfo, "denying file write due to quota limits")
+		return nil, common.ErrQuotaExceeded
+	}
+	minWriteOffset := int64(0)
+
+	if flags&os.O_APPEND != 0 && flags&os.O_TRUNC == 0 && !c.Fs.IsUploadResumeSupported() {
+		c.Log(logger.LevelInfo, "upload resume requested for path: %#v but not supported in fs implementation", resolvedPath)
+		return nil, c.GetOpUnsupportedError()
+	}
+
+	if common.Config.IsAtomicUploadEnabled() && c.Fs.IsAtomicUploadSupported() {
+		err = c.Fs.Rename(resolvedPath, filePath)
+		if err != nil {
+			c.Log(logger.LevelWarn, "error renaming existing file for atomic upload, source: %#v, dest: %#v, err: %+v",
+				resolvedPath, filePath, err)
+			return nil, c.GetFsError(err)
+		}
+	}
+
+	file, w, cancelFn, err := c.Fs.Create(filePath, flags)
+	if err != nil {
+		c.Log(logger.LevelWarn, "error opening existing file, flags: %v, source: %#v, err: %+v", flags, filePath, err)
+		return nil, c.GetFsError(err)
+	}
+
+	initialSize := int64(0)
+	// if there is a size limit remaining size cannot be 0 here, since quotaResult.HasSpace
+	// will return false in this case and we deny the upload before
+	maxWriteSize := quotaResult.GetRemainingSize()
+	if flags&os.O_APPEND != 0 && flags&os.O_TRUNC == 0 {
+		c.Log(logger.LevelDebug, "upload resume requested, file path: %#v initial size: %v", filePath, fileSize)
+		minWriteOffset = fileSize
+	} else {
+		if vfs.IsLocalOsFs(c.Fs) {
+			vfolder, err := c.User.GetVirtualFolderForPath(path.Dir(requestPath))
+			if err == nil {
+				dataprovider.UpdateVirtualFolderQuota(vfolder.BaseVirtualFolder, 0, -fileSize, false) //nolint:errcheck
+				if vfolder.IsIncludedInUserQuota() {
+					dataprovider.UpdateUserQuota(c.User, 0, -fileSize, false) //nolint:errcheck
+				}
+			} else {
+				dataprovider.UpdateUserQuota(c.User, 0, -fileSize, false) //nolint:errcheck
+			}
+		} else {
+			initialSize = fileSize
+		}
+		if maxWriteSize > 0 {
+			maxWriteSize += fileSize
+		}
+	}
+
+	vfs.SetPathPermissions(c.Fs, filePath, c.User.GetUID(), c.User.GetGID())
+
+	baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, resolvedPath, requestPath,
+		common.TransferUpload, minWriteOffset, initialSize, false)
+	t := newTransfer(baseTransfer, w, nil, maxWriteSize)
+
+	return t, nil
+}

ftpd/internal_test.go

@@ -0,0 +1,441 @@
+package ftpd
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/eikenb/pipeat"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/drakkan/sftpgo/common"
+	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/vfs"
+)
+
+const (
+	configDir = ".."
+)
+
+type mockFTPClientContext struct {
+}
+
+func (cc mockFTPClientContext) Path() string {
+	return ""
+}
+
+func (cc mockFTPClientContext) SetDebug(debug bool) {}
+
+func (cc mockFTPClientContext) Debug() bool {
+	return false
+}
+
+func (cc mockFTPClientContext) ID() uint32 {
+	return 1
+}
+
+func (cc mockFTPClientContext) RemoteAddr() net.Addr {
+	return &net.IPAddr{IP: []byte("127.0.0.1")}
+}
+
+func (cc mockFTPClientContext) LocalAddr() net.Addr {
+	return &net.IPAddr{IP: []byte("127.0.0.1")}
+}
+
+func (cc mockFTPClientContext) GetClientVersion() string {
+	return "mock version"
+}
+
+func (cc mockFTPClientContext) Close(code int, message string) error {
+	return nil
+}
+
+// MockOsFs mockable OsFs
+type MockOsFs struct {
+	vfs.Fs
+	err                     error
+	statErr                 error
+	isAtomicUploadSupported bool
+}
+
+// Name returns the name for the Fs implementation
+func (fs MockOsFs) Name() string {
+	return "mockOsFs"
+}
+
+// IsUploadResumeSupported returns true if upload resume is supported
+func (MockOsFs) IsUploadResumeSupported() bool {
+	return false
+}
+
+// IsAtomicUploadSupported returns true if atomic upload is supported
+func (fs MockOsFs) IsAtomicUploadSupported() bool {
+	return fs.isAtomicUploadSupported
+}
+
+// Stat returns a FileInfo describing the named file
+func (fs MockOsFs) Stat(name string) (os.FileInfo, error) {
+	if fs.statErr != nil {
+		return nil, fs.statErr
+	}
+	return os.Stat(name)
+}
+
+// Lstat returns a FileInfo describing the named file
+func (fs MockOsFs) Lstat(name string) (os.FileInfo, error) {
+	if fs.statErr != nil {
+		return nil, fs.statErr
+	}
+	return os.Lstat(name)
+}
+
+// Remove removes the named file or (empty) directory.
+func (fs MockOsFs) Remove(name string, isDir bool) error {
+	if fs.err != nil {
+		return fs.err
+	}
+	return os.Remove(name)
+}
+
+// Rename renames (moves) source to target
+func (fs MockOsFs) Rename(source, target string) error {
+	if fs.err != nil {
+		return fs.err
+	}
+	return os.Rename(source, target)
+}
+
+func newMockOsFs(err, statErr error, atomicUpload bool, connectionID, rootDir string) vfs.Fs {
+	return &MockOsFs{
+		Fs:                      vfs.NewOsFs(connectionID, rootDir, nil),
+		err:                     err,
+		statErr:                 statErr,
+		isAtomicUploadSupported: atomicUpload,
+	}
+}
+
+func TestInitialization(t *testing.T) {
+	c := &Configuration{
+		BindPort:           2121,
+		CertificateFile:    "acert",
+		CertificateKeyFile: "akey",
+	}
+	err := c.Initialize(configDir)
+	assert.Error(t, err)
+	c.CertificateFile = ""
+	c.CertificateKeyFile = ""
+	c.BannerFile = "afile"
+	server, err := NewServer(c, configDir)
+	if assert.NoError(t, err) {
+		assert.Equal(t, "", server.initialMsg)
+		_, err = server.GetTLSConfig()
+		assert.Error(t, err)
+	}
+	err = ReloadTLSCertificate()
+	assert.NoError(t, err)
+}
+
+func TestServerGetSettings(t *testing.T) {
+	oldConfig := common.Config
+	c := &Configuration{
+		BindPort: 2121,
+		PassivePortRange: PortRange{
+			Start: 10000,
+			End:   11000,
+		},
+	}
+	server, err := NewServer(c, configDir)
+	assert.NoError(t, err)
+	settings, err := server.GetSettings()
+	assert.NoError(t, err)
+	assert.Equal(t, 10000, settings.PassiveTransferPortRange.Start)
+	assert.Equal(t, 11000, settings.PassiveTransferPortRange.End)
+
+	common.Config.ProxyProtocol = 1
+	common.Config.ProxyAllowed = []string{"invalid"}
+	_, err = server.GetSettings()
+	assert.Error(t, err)
+	server.config.BindPort = 8021
+	_, err = server.GetSettings()
+	assert.Error(t, err)
+
+	common.Config = oldConfig
+}
+
+func TestUserInvalidParams(t *testing.T) {
+	u := dataprovider.User{
+		HomeDir: "invalid",
+	}
+	c := &Configuration{
+		BindPort: 2121,
+		PassivePortRange: PortRange{
+			Start: 10000,
+			End:   11000,
+		},
+	}
+	server, err := NewServer(c, configDir)
+	assert.NoError(t, err)
+	_, err = server.validateUser(u, mockFTPClientContext{})
+	assert.Error(t, err)
+
+	u.Username = "a"
+	u.HomeDir = filepath.Clean(os.TempDir())
+	subDir := "subdir"
+	mappedPath1 := filepath.Join(os.TempDir(), "vdir1")
+	vdirPath1 := "/vdir1"
+	mappedPath2 := filepath.Join(os.TempDir(), "vdir1", subDir)
+	vdirPath2 := "/vdir2"
+	u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{
+		BaseVirtualFolder: vfs.BaseVirtualFolder{
+			MappedPath: mappedPath1,
+		},
+		VirtualPath: vdirPath1,
+	})
+	u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{
+		BaseVirtualFolder: vfs.BaseVirtualFolder{
+			MappedPath: mappedPath2,
+		},
+		VirtualPath: vdirPath2,
+	})
+	_, err = server.validateUser(u, mockFTPClientContext{})
+	assert.Error(t, err)
+	u.VirtualFolders = nil
+	_, err = server.validateUser(u, mockFTPClientContext{})
+	assert.Error(t, err)
+}
+
+func TestClientVersion(t *testing.T) {
+	mockCC := mockFTPClientContext{}
+	connID := fmt.Sprintf("%v", mockCC.ID())
+	user := dataprovider.User{}
+	connection := &Connection{
+		BaseConnection: common.NewBaseConnection(connID, common.ProtocolFTP, user, nil),
+		clientContext:  mockCC,
+	}
+	common.Connections.Add(connection)
+	stats := common.Connections.GetStats()
+	if assert.Len(t, stats, 1) {
+		assert.Equal(t, "mock version", stats[0].ClientVersion)
+		common.Connections.Remove(connection.GetID())
+	}
+	assert.Len(t, common.Connections.GetStats(), 0)
+}
+
+func TestDriverMethodsNotImplemented(t *testing.T) {
+	mockCC := mockFTPClientContext{}
+	connID := fmt.Sprintf("%v", mockCC.ID())
+	user := dataprovider.User{}
+	connection := &Connection{
+		BaseConnection: common.NewBaseConnection(connID, common.ProtocolFTP, user, nil),
+		clientContext:  mockCC,
+	}
+	_, err := connection.Create("")
+	assert.EqualError(t, err, errNotImplemented.Error())
+	err = connection.MkdirAll("", os.ModePerm)
+	assert.EqualError(t, err, errNotImplemented.Error())
+	_, err = connection.Open("")
+	assert.EqualError(t, err, errNotImplemented.Error())
+	_, err = connection.OpenFile("", 0, os.ModePerm)
+	assert.EqualError(t, err, errNotImplemented.Error())
+	err = connection.RemoveAll("")
+	assert.EqualError(t, err, errNotImplemented.Error())
+	assert.Equal(t, connection.GetID(), connection.Name())
+}
+
+func TestResolvePathErrors(t *testing.T) {
+	user := dataprovider.User{
+		HomeDir: "invalid",
+	}
+	user.Permissions = make(map[string][]string)
+	user.Permissions["/"] = []string{dataprovider.PermAny}
+	mockCC := mockFTPClientContext{}
+	connID := fmt.Sprintf("%v", mockCC.ID())
+	fs := vfs.NewOsFs(connID, user.HomeDir, nil)
+	connection := &Connection{
+		BaseConnection: common.NewBaseConnection(connID, common.ProtocolFTP, user, fs),
+		clientContext:  mockCC,
+	}
+	err := connection.Mkdir("", os.ModePerm)
+	if assert.Error(t, err) {
+		assert.EqualError(t, err, common.ErrGenericFailure.Error())
+	}
+	err = connection.Remove("")
+	if assert.Error(t, err) {
+		assert.EqualError(t, err, common.ErrGenericFailure.Error())
+	}
+	err = connection.Rename("", "")
+	if assert.Error(t, err) {
+		assert.EqualError(t, err, common.ErrGenericFailure.Error())
+	}
+	_, err = connection.Stat("")
+	if assert.Error(t, err) {
+		assert.EqualError(t, err, common.ErrGenericFailure.Error())
+	}
+	err = connection.Chmod("", os.ModePerm)
+	if assert.Error(t, err) {
+		assert.EqualError(t, err, common.ErrGenericFailure.Error())
+	}
+	err = connection.Chtimes("", time.Now(), time.Now())
+	if assert.Error(t, err) {
+		assert.EqualError(t, err, common.ErrGenericFailure.Error())
+	}
+	_, err = connection.ReadDir("")
+	if assert.Error(t, err) {
+		assert.EqualError(t, err, common.ErrGenericFailure.Error())
+	}
+	_, err = connection.GetHandle("", 0)
+	if assert.Error(t, err) {
+		assert.EqualError(t, err, common.ErrGenericFailure.Error())
+	}
+}
+
+func TestUploadFileStatError(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("this test is not available on Windows")
+	}
+	user := dataprovider.User{
+		Username: "user",
+		HomeDir:  filepath.Clean(os.TempDir()),
+	}
+	user.Permissions = make(map[string][]string)
+	user.Permissions["/"] = []string{dataprovider.PermAny}
+	mockCC := mockFTPClientContext{}
+	connID := fmt.Sprintf("%v", mockCC.ID())
+	fs := vfs.NewOsFs(connID, user.HomeDir, nil)
+	connection := &Connection{
+		BaseConnection: common.NewBaseConnection(connID, common.ProtocolFTP, user, fs),
+		clientContext:  mockCC,
+	}
+	testFile := filepath.Join(user.HomeDir, "test", "testfile")
+	err := os.MkdirAll(filepath.Dir(testFile), os.ModePerm)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(testFile, []byte("data"), os.ModePerm)
+	assert.NoError(t, err)
+	err = os.Chmod(filepath.Dir(testFile), 0001)
+	assert.NoError(t, err)
+	_, err = connection.uploadFile(testFile, "test", 0)
+	assert.Error(t, err)
+	err = os.Chmod(filepath.Dir(testFile), os.ModePerm)
+	assert.NoError(t, err)
+	err = os.RemoveAll(filepath.Dir(testFile))
+	assert.NoError(t, err)
+}
+
+func TestUploadOverwriteErrors(t *testing.T) {
+	user := dataprovider.User{
+		Username: "user",
+		HomeDir:  filepath.Clean(os.TempDir()),
+	}
+	user.Permissions = make(map[string][]string)
+	user.Permissions["/"] = []string{dataprovider.PermAny}
+	mockCC := mockFTPClientContext{}
+	connID := fmt.Sprintf("%v", mockCC.ID())
+	fs := newMockOsFs(nil, nil, false, connID, user.GetHomeDir())
+	connection := &Connection{
+		BaseConnection: common.NewBaseConnection(connID, common.ProtocolFTP, user, fs),
+		clientContext:  mockCC,
+	}
+	flags := 0
+	flags |= os.O_APPEND
+	_, err := connection.handleFTPUploadToExistingFile(flags, "", "", 0, "")
+	if assert.Error(t, err) {
+		assert.EqualError(t, err, common.ErrOpUnsupported.Error())
+	}
+
+	f, err := ioutil.TempFile("", "temp")
+	assert.NoError(t, err)
+	err = f.Close()
+	assert.NoError(t, err)
+	flags = 0
+	flags |= os.O_CREATE
+	flags |= os.O_TRUNC
+	tr, err := connection.handleFTPUploadToExistingFile(flags, f.Name(), f.Name(), 123, f.Name())
+	if assert.NoError(t, err) {
+		transfer := tr.(*transfer)
+		transfers := connection.GetTransfers()
+		if assert.Equal(t, 1, len(transfers)) {
+			assert.Equal(t, transfers[0].ID, transfer.GetID())
+			assert.Equal(t, int64(123), transfer.InitialSize)
+			err = transfer.Close()
+			assert.NoError(t, err)
+			assert.Equal(t, 0, len(connection.GetTransfers()))
+		}
+	}
+	err = os.Remove(f.Name())
+	assert.NoError(t, err)
+
+	_, err = connection.handleFTPUploadToExistingFile(0, filepath.Join(os.TempDir(), "sub", "file"),
+		filepath.Join(os.TempDir(), "sub", "file1"), 0, "/sub/file1")
+	assert.Error(t, err)
+	connection.Fs = vfs.NewOsFs(connID, user.GetHomeDir(), nil)
+	_, err = connection.handleFTPUploadToExistingFile(0, "missing1", "missing2", 0, "missing")
+	assert.Error(t, err)
+}
+
+func TestTransferErrors(t *testing.T) {
+	testfile := "testfile"
+	file, err := os.Create(testfile)
+	assert.NoError(t, err)
+	user := dataprovider.User{
+		Username: "user",
+		HomeDir:  filepath.Clean(os.TempDir()),
+	}
+	user.Permissions = make(map[string][]string)
+	user.Permissions["/"] = []string{dataprovider.PermAny}
+	mockCC := mockFTPClientContext{}
+	connID := fmt.Sprintf("%v", mockCC.ID())
+	fs := newMockOsFs(nil, nil, false, connID, user.GetHomeDir())
+	connection := &Connection{
+		BaseConnection: common.NewBaseConnection(connID, common.ProtocolFTP, user, fs),
+		clientContext:  mockCC,
+	}
+	baseTransfer := common.NewBaseTransfer(file, connection.BaseConnection, nil, file.Name(), testfile, common.TransferDownload,
+		0, 0, false)
+	tr := newTransfer(baseTransfer, nil, nil, 0)
+	err = tr.Close()
+	assert.NoError(t, err)
+	buf := make([]byte, 64)
+	_, err = tr.Read(buf)
+	assert.Error(t, err)
+	err = tr.Close()
+	if assert.Error(t, err) {
+		assert.EqualError(t, err, common.ErrTransferClosed.Error())
+	}
+	assert.Len(t, connection.GetTransfers(), 0)
+
+	r, _, err := pipeat.Pipe()
+	assert.NoError(t, err)
+	baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testfile, testfile,
+		common.TransferUpload, 0, 0, false)
+	tr = newTransfer(baseTransfer, nil, r, 0)
+	err = tr.closeIO()
+	assert.NoError(t, err)
+
+	r, w, err := pipeat.Pipe()
+	assert.NoError(t, err)
+	pipeWriter := vfs.NewPipeWriter(w)
+	baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testfile, testfile,
+		common.TransferUpload, 0, 0, false)
+	tr = newTransfer(baseTransfer, pipeWriter, nil, 0)
+	_, err = tr.Seek(1, 0)
+	if assert.Error(t, err) {
+		assert.EqualError(t, err, common.ErrOpUnsupported.Error())
+	}
+
+	err = r.Close()
+	assert.NoError(t, err)
+	errFake := fmt.Errorf("fake upload error")
+	go func() {
+		time.Sleep(100 * time.Millisecond)
+		pipeWriter.Done(errFake)
+	}()
+	err = tr.closeIO()
+	assert.EqualError(t, err, errFake.Error())
+	err = os.Remove(testfile)
+	assert.NoError(t, err)
+}

ftpd/server.go

@@ -0,0 +1,190 @@
+package ftpd
+
+import (
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"path/filepath"
+
+	ftpserver "github.com/fclairamb/ftpserverlib"
+
+	"github.com/drakkan/sftpgo/common"
+	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/metrics"
+	"github.com/drakkan/sftpgo/utils"
+)
+
+// Server implements the ftpserverlib MainDriver interface
+type Server struct {
+	config     *Configuration
+	certMgr    *common.CertManager
+	initialMsg string
+}
+
+// NewServer returns a new FTP server driver
+func NewServer(config *Configuration, configDir string) (*Server, error) {
+	var err error
+	server := &Server{
+		config:     config,
+		certMgr:    nil,
+		initialMsg: config.Banner,
+	}
+	certificateFile := getConfigPath(config.CertificateFile, configDir)
+	certificateKeyFile := getConfigPath(config.CertificateKeyFile, configDir)
+	if len(certificateFile) > 0 && len(certificateKeyFile) > 0 {
+		server.certMgr, err = common.NewCertManager(certificateFile, certificateKeyFile, logSender)
+		if err != nil {
+			return server, err
+		}
+	}
+	if len(config.BannerFile) > 0 {
+		bannerFilePath := config.BannerFile
+		if !filepath.IsAbs(bannerFilePath) {
+			bannerFilePath = filepath.Join(configDir, bannerFilePath)
+		}
+		bannerContent, err := ioutil.ReadFile(bannerFilePath)
+		if err == nil {
+			server.initialMsg = string(bannerContent)
+		} else {
+			logger.WarnToConsole("unable to read FTPD banner file: %v", err)
+			logger.Warn(logSender, "", "unable to read banner file: %v", err)
+		}
+	}
+	return server, err
+}
+
+// GetSettings returns FTP server settings
+func (s *Server) GetSettings() (*ftpserver.Settings, error) {
+	var portRange *ftpserver.PortRange = nil
+	if s.config.PassivePortRange.Start > 0 && s.config.PassivePortRange.End > s.config.PassivePortRange.Start {
+		portRange = &ftpserver.PortRange{
+			Start: s.config.PassivePortRange.Start,
+			End:   s.config.PassivePortRange.End,
+		}
+	}
+	var ftpListener net.Listener
+	if common.Config.ProxyProtocol > 0 {
+		listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", s.config.BindAddress, s.config.BindPort))
+		if err != nil {
+			logger.Warn(logSender, "", "error starting listener on address %s:%d: %v", s.config.BindAddress, s.config.BindPort, err)
+			return nil, err
+		}
+		ftpListener, err = common.Config.GetProxyListener(listener)
+		if err != nil {
+			logger.Warn(logSender, "", "error enabling proxy listener: %v", err)
+			return nil, err
+		}
+	}
+
+	return &ftpserver.Settings{
+		Listener:                 ftpListener,
+		ListenAddr:               fmt.Sprintf("%s:%d", s.config.BindAddress, s.config.BindPort),
+		PublicHost:               s.config.ForcePassiveIP,
+		PassiveTransferPortRange: portRange,
+		ActiveTransferPortNon20:  s.config.ActiveTransfersPortNon20,
+		IdleTimeout:              -1,
+		ConnectionTimeout:        30,
+	}, nil
+}
+
+// ClientConnected is called to send the very first welcome message
+func (s *Server) ClientConnected(cc ftpserver.ClientContext) (string, error) {
+	connID := fmt.Sprintf("%v", cc.ID())
+	user := dataprovider.User{}
+	connection := &Connection{
+		BaseConnection: common.NewBaseConnection(connID, common.ProtocolFTP, user, nil),
+		clientContext:  cc,
+	}
+	common.Connections.Add(connection)
+	return s.initialMsg, nil
+}
+
+// ClientDisconnected is called when the user disconnects, even if he never authenticated
+func (s *Server) ClientDisconnected(cc ftpserver.ClientContext) {
+	connID := fmt.Sprintf("%v_%v", common.ProtocolFTP, cc.ID())
+	common.Connections.Remove(connID)
+}
+
+// AuthUser authenticates the user and selects an handling driver
+func (s *Server) AuthUser(cc ftpserver.ClientContext, username, password string) (ftpserver.ClientDriver, error) {
+	remoteAddr := cc.RemoteAddr().String()
+	user, err := dataprovider.CheckUserAndPass(username, password)
+	if err != nil {
+		updateLoginMetrics(username, remoteAddr, dataprovider.FTPLoginMethodPassword, err)
+		return nil, err
+	}
+
+	connection, err := s.validateUser(user, cc)
+
+	defer updateLoginMetrics(username, remoteAddr, dataprovider.FTPLoginMethodPassword, err)
+
+	if err != nil {
+		return nil, err
+	}
+	connection.Fs.CheckRootPath(connection.GetUsername(), user.GetUID(), user.GetGID())
+	connection.Log(logger.LevelInfo, "User id: %d, logged in with FTP, username: %#v, home_dir: %#v remote addr: %#v",
+		user.ID, user.Username, user.HomeDir, remoteAddr)
+	dataprovider.UpdateLastLogin(user) //nolint:errcheck
+	return connection, nil
+}
+
+// GetTLSConfig returns a TLS Certificate to use
+func (s *Server) GetTLSConfig() (*tls.Config, error) {
+	if s.certMgr != nil {
+		return &tls.Config{
+			GetCertificate: s.certMgr.GetCertificateFunc(),
+		}, nil
+	}
+	return nil, errors.New("no TLS certificate configured")
+}
+
+func (s *Server) validateUser(user dataprovider.User, cc ftpserver.ClientContext) (*Connection, error) {
+	connectionID := fmt.Sprintf("%v_%v", common.ProtocolFTP, cc.ID())
+	if !filepath.IsAbs(user.HomeDir) {
+		logger.Warn(logSender, connectionID, "user %#v has an invalid home dir: %#v. Home dir must be an absolute path, login not allowed",
+			user.Username, user.HomeDir)
+		return nil, fmt.Errorf("cannot login user with invalid home dir: %#v", user.HomeDir)
+	}
+	if user.MaxSessions > 0 {
+		activeSessions := common.Connections.GetActiveSessions(user.Username)
+		if activeSessions >= user.MaxSessions {
+			logger.Debug(logSender, connectionID, "authentication refused for user: %#v, too many open sessions: %v/%v", user.Username,
+				activeSessions, user.MaxSessions)
+			return nil, fmt.Errorf("too many open sessions: %v", activeSessions)
+		}
+	}
+	if dataprovider.GetQuotaTracking() > 0 && user.HasOverlappedMappedPaths() {
+		logger.Debug(logSender, connectionID, "cannot login user %#v, overlapping mapped folders are allowed only with quota tracking disabled",
+			user.Username)
+		return nil, errors.New("overlapping mapped folders are allowed only with quota tracking disabled")
+	}
+	remoteAddr := cc.RemoteAddr().String()
+	if !user.IsLoginFromAddrAllowed(remoteAddr) {
+		logger.Debug(logSender, connectionID, "cannot login user %#v, remote address is not allowed: %v", user.Username, remoteAddr)
+		return nil, fmt.Errorf("Login for user %#v is not allowed from this address: %v", user.Username, remoteAddr)
+	}
+	fs, err := user.GetFilesystem(connectionID)
+	if err != nil {
+		return nil, err
+	}
+	connection := &Connection{
+		BaseConnection: common.NewBaseConnection(fmt.Sprintf("%v", cc.ID()), common.ProtocolFTP, user, fs),
+		clientContext:  cc,
+	}
+	err = common.Connections.Swap(connection)
+	if err != nil {
+		return nil, errors.New("Internal authentication error")
+	}
+	return connection, nil
+}
+
+func updateLoginMetrics(username, remoteAddress, method string, err error) {
+	metrics.AddLoginAttempt(method)
+	if err != nil {
+		logger.ConnectionFailedLog(username, utils.GetIPFromRemoteAddress(remoteAddress), method, err.Error())
+	}
+	metrics.AddLoginResult(method, err)
+}

ftpd/transfer.go

@@ -0,0 +1,128 @@
+package ftpd
+
+import (
+	"io"
+	"sync/atomic"
+
+	"github.com/eikenb/pipeat"
+
+	"github.com/drakkan/sftpgo/common"
+	"github.com/drakkan/sftpgo/vfs"
+)
+
+// transfer contains the transfer details for an upload or a download.
+// It implements the ftpserver.FileTransfer interface to handle files downloads and uploads
+type transfer struct {
+	*common.BaseTransfer
+	writer       io.WriteCloser
+	reader       io.ReadCloser
+	isFinished   bool
+	maxWriteSize int64
+}
+
+func newTransfer(baseTransfer *common.BaseTransfer, pipeWriter *vfs.PipeWriter, pipeReader *pipeat.PipeReaderAt,
+	maxWriteSize int64) *transfer {
+	var writer io.WriteCloser
+	var reader io.ReadCloser
+	if baseTransfer.File != nil {
+		writer = baseTransfer.File
+		reader = baseTransfer.File
+	} else if pipeWriter != nil {
+		writer = pipeWriter
+	} else if pipeReader != nil {
+		reader = pipeReader
+	}
+	return &transfer{
+		BaseTransfer: baseTransfer,
+		writer:       writer,
+		reader:       reader,
+		isFinished:   false,
+		maxWriteSize: maxWriteSize,
+	}
+}
+
+// Read reads the contents to downloads.
+func (t *transfer) Read(p []byte) (n int, err error) {
+	t.Connection.UpdateLastActivity()
+	var readed int
+	var e error
+
+	readed, e = t.reader.Read(p)
+	atomic.AddInt64(&t.BytesSent, int64(readed))
+
+	if e != nil && e != io.EOF {
+		t.TransferError(e)
+		return readed, e
+	}
+	t.HandleThrottle()
+	return readed, e
+}
+
+// Write writes the uploaded contents.
+func (t *transfer) Write(p []byte) (n int, err error) {
+	t.Connection.UpdateLastActivity()
+	var written int
+	var e error
+
+	written, e = t.writer.Write(p)
+	atomic.AddInt64(&t.BytesReceived, int64(written))
+
+	if t.maxWriteSize > 0 && e == nil && atomic.LoadInt64(&t.BytesReceived) > t.maxWriteSize {
+		e = common.ErrQuotaExceeded
+	}
+	if e != nil {
+		t.TransferError(e)
+		return written, e
+	}
+	t.HandleThrottle()
+	return written, e
+}
+
+// Seek sets the offset to resume an upload or a download
+func (t *transfer) Seek(offset int64, whence int) (int64, error) {
+	if t.File != nil {
+		return t.File.Seek(offset, whence)
+	}
+	return 0, common.ErrOpUnsupported
+}
+
+// Close it is called when the transfer is completed.
+func (t *transfer) Close() error {
+	if err := t.setFinished(); err != nil {
+		return err
+	}
+	err := t.closeIO()
+	errBaseClose := t.BaseTransfer.Close()
+	if errBaseClose != nil {
+		err = errBaseClose
+	}
+	return t.Connection.GetFsError(err)
+}
+
+func (t *transfer) closeIO() error {
+	var err error
+	if t.File != nil {
+		err = t.File.Close()
+	} else if t.writer != nil {
+		err = t.writer.Close()
+		t.Lock()
+		// we set ErrTransfer here so quota is not updated, in this case the uploads are atomic
+		if err != nil && t.ErrTransfer == nil {
+			t.ErrTransfer = err
+		}
+		t.Unlock()
+	} else if t.reader != nil {
+		err = t.reader.Close()
+	}
+	return err
+}
+
+func (t *transfer) setFinished() error {
+	t.Lock()
+	defer t.Unlock()
+	if t.isFinished {
+		return common.ErrTransferClosed
+	}
+	t.isFinished = true
+	return nil
+}

go.mod

@@ -8,12 +8,14 @@ require (
 	github.com/alexedwards/argon2id v0.0.0-20200522061839-9369edc04b05
 	github.com/aws/aws-sdk-go v1.33.1
 	github.com/eikenb/pipeat v0.0.0-20200430215831-470df5986b6d
+	github.com/fclairamb/ftpserverlib v0.8.0
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/go-chi/chi v4.1.2+incompatible
 	github.com/go-chi/render v1.0.1
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/grandcat/zeroconf v1.0.0
+	github.com/jlaffaye/ftp v0.0.0-20200720194710-13949d38913e
 	github.com/lib/pq v1.7.0
 	github.com/mattn/go-sqlite3 v1.14.0
 	github.com/miekg/dns v1.1.29 // indirect
@@ -26,7 +28,7 @@ require (
 	github.com/prometheus/client_golang v1.7.1
 	github.com/rs/xid v1.2.1
 	github.com/rs/zerolog v1.19.0
-	github.com/spf13/afero v1.3.1 // indirect
+	github.com/spf13/afero v1.3.2
 	github.com/spf13/cast v1.3.1 // indirect
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
@@ -47,4 +49,8 @@ require (
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )
 
-replace golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20200705203859-05ad140ecdbd
+replace (
+	github.com/fclairamb/ftpserverlib => github.com/drakkan/ftpserverlib v0.0.0-20200729185904-a61d63fc1db1
+	github.com/jlaffaye/ftp => github.com/drakkan/ftp v0.0.0-20200727182237-9cca2b71337f
+	golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20200705203859-05ad140ecdbd
+)

go.sum

@@ -35,8 +35,13 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
+github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
+github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
+github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -44,18 +49,25 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
 github.com/alexedwards/argon2id v0.0.0-20200522061839-9369edc04b05 h1:votg1faEmwABhCeJ4tiBrvwk4BWftQGkEtFy5iuI7rU=
 github.com/alexedwards/argon2id v0.0.0-20200522061839-9369edc04b05/go.mod h1:GFtu6vaWaRJV5EvSFaVqgq/3Iq95xyYElBV/aupGzUo=
 github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
+github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
+github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
+github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.33.1 h1:yz9XmNzPshz/lhfAZvLfMnIS9HPo8+boGRcWqDVX+T0=
 github.com/aws/aws-sdk-go v1.33.1/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
+github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
+github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
 github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -66,17 +78,24 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
+github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -84,13 +103,29 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/drakkan/crypto v0.0.0-20200705203859-05ad140ecdbd h1:7DQ8ayx6QylOxQrQ6oHdO+gk1cqtaINUekLrwD9gGNc=
 github.com/drakkan/crypto v0.0.0-20200705203859-05ad140ecdbd/go.mod h1:v3bhWOXGYda7H5d2s5t9XA6th3fxW3s0MQxU1R96G/w=
+github.com/drakkan/ftp v0.0.0-20200727182237-9cca2b71337f h1:O58KACT8jWFlYJ3VSNFk+T+s7eX67LQwkC+FTRmCPa8=
+github.com/drakkan/ftp v0.0.0-20200727182237-9cca2b71337f/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU=
+github.com/drakkan/ftpserverlib v0.0.0-20200728193209-1f77405f11b5 h1:pAThLlS4VfkAOPk/tex9/qVwsbb1CRT9YkBjsBVBFNI=
+github.com/drakkan/ftpserverlib v0.0.0-20200728193209-1f77405f11b5/go.mod h1:Jwd+zOP3T0kwiCQcgjpu3VWtc7AI6Nu4UPN2HYqaniM=
+github.com/drakkan/ftpserverlib v0.0.0-20200729180240-d4d48e974eed h1:mIaBQdWAOkh6Rt+O9sEo/gGJaC7DnPuxpqVpPZYz0z8=
+github.com/drakkan/ftpserverlib v0.0.0-20200729180240-d4d48e974eed/go.mod h1:Jwd+zOP3T0kwiCQcgjpu3VWtc7AI6Nu4UPN2HYqaniM=
+github.com/drakkan/ftpserverlib v0.0.0-20200729185904-a61d63fc1db1 h1:PkZAqIdsFJpfyRLmCOHZHsu3VUDgqClze0+hai6HBL8=
+github.com/drakkan/ftpserverlib v0.0.0-20200729185904-a61d63fc1db1/go.mod h1:Jwd+zOP3T0kwiCQcgjpu3VWtc7AI6Nu4UPN2HYqaniM=
+github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
+github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
+github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
+github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
 github.com/eikenb/pipeat v0.0.0-20200430215831-470df5986b6d h1:8RvCRWer7TB2n+DKhW4uW15hRiqPmabSnSyYhju/Nuw=
 github.com/eikenb/pipeat v0.0.0-20200430215831-470df5986b6d/go.mod h1:+JPhBw5JdJrSF80r6xsSg1TYHjyAGxYs4X24VyUdMZU=
+github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
+github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
@@ -104,14 +139,20 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -137,6 +178,7 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -157,19 +199,28 @@ github.com/google/pprof v0.0.0-20200507031123-427632fa3b1c/go.mod h1:ZgVRPoUq/hf
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
+github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grandcat/zeroconf v1.0.0 h1:uHhahLBKqwWBV6WZUDAT71044vwOTL+McW0mBJvo6kE=
 github.com/grandcat/zeroconf v1.0.0/go.mod h1:lTKmG1zh86XyCoUeIHSA4FJMBwCJiQmGfcP2PdzytEs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
+github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
@@ -180,6 +231,7 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -189,12 +241,18 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
@@ -211,11 +269,16 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY=
 github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
+github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
+github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA=
 github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
@@ -243,23 +306,51 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nathanaelle/password/v2 v2.0.1 h1:ItoCTdsuIWzilYmllQPa3DR3YoCXcpfxScWLqr8Ii2s=
 github.com/nathanaelle/password/v2 v2.0.1/go.mod h1:eaoT+ICQEPNtikBRIAatN8ThWwMhVG+r1jTw60BvPJk=
+github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
+github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU=
+github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k=
+github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w=
+github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
+github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
+github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
+github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
+github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
+github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
+github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA=
+github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
+github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
+github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
 github.com/otiai10/copy v1.2.0 h1:HvG945u96iNadPoG2/Ja2+AUJeW5YuFQMixq9yirC+k=
 github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw=
 github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
 github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
 github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
+github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.8.0 h1:Keo9qb7iRJs2voHvunFtuuYFsbWeOBh8/P9v/kVMFtw=
 github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs=
+github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
+github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
+github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pires/go-proxyproto v0.1.3 h1:2XEuhsQluSNA5QIQkiUv8PfgZ51sNYIQkq/yFquiSQM=
 github.com/pires/go-proxyproto v0.1.3/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9Gw5rnCjcwzAY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
 github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pkg/sftp v1.11.1-0.20200716191756-97b9df616e69 h1:qbrvNcVkxFdNuawO0LsSRRVMubCHOHdDTbb5EO/hiPE=
 github.com/pkg/sftp v1.11.1-0.20200716191756-97b9df616e69/go.mod h1:PIrgHN0+qgDmYTNiwryjoEqmXo9tv8aMwQ//Yg1xwIs=
@@ -267,26 +358,35 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og=
 github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
 github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
@@ -296,7 +396,9 @@ github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJ
 github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/secsy/goftp v0.0.0-20190720192957-f31499d7c79a/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY=
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
@@ -304,24 +406,30 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.3.1 h1:GPTpEAuNr98px18yNQ66JllNil98wfRZ/5Ukny8FeQA=
-github.com/spf13/afero v1.3.1/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=
+github.com/spf13/afero v1.3.2 h1:GDarE4TJQI52kYSbSAmLiId1Elfj+xgSDqrUZxFhxlU=
+github.com/spf13/afero v1.3.2/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM=
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
+github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
+github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
+github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
@@ -332,24 +440,36 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
+github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
 go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
+go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
+go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -383,11 +503,13 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -398,6 +520,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -428,9 +551,11 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -440,10 +565,12 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -467,10 +594,12 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -488,6 +617,8 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -497,6 +628,7 @@ golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
@@ -517,6 +649,7 @@ golang.org/x/tools v0.0.0-20200702044944-0cc1aa72b347/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@@ -533,6 +666,7 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
 google.golang.org/api v0.28.0 h1:jMF5hhVfMkTZwHW1SDpKq5CkgWLXOb31Foaca9Zr3oM=
 google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
@@ -543,6 +677,7 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
 google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
@@ -567,11 +702,15 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
 google.golang.org/genproto v0.0.0-20200626011028-ee7919e894b5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200702021140-07506425bd67 h1:4BC1C1i30F3MZeiIO6y6IIo4DxrtOwITK87bQl3lhFA=
 google.golang.org/genproto v0.0.0-20200702021140-07506425bd67/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
@@ -595,13 +734,19 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
+gopkg.in/dutchcoders/goftp.v1 v1.0.0-20170301105846-ed59a591ce14/go.mod h1:nzmlZQ+UqB5+55CRTV/dOaiK8OrPl6Co96Ob8lH4Wxw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
 gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -611,6 +756,7 @@ gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -621,3 +767,5 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=

httpd/httpd_test.go

@@ -1274,14 +1274,14 @@ func TestProviderErrors(t *testing.T) {
 	backupContent, err := json.Marshal(backupData)
 	assert.NoError(t, err)
 	backupFilePath := filepath.Join(backupsPath, "backup.json")
-	err = ioutil.WriteFile(backupFilePath, backupContent, 0666)
+	err = ioutil.WriteFile(backupFilePath, backupContent, os.ModePerm)
 	assert.NoError(t, err)
 	_, _, err = httpd.Loaddata(backupFilePath, "", "", http.StatusInternalServerError)
 	assert.NoError(t, err)
 	backupData.Folders = append(backupData.Folders, vfs.BaseVirtualFolder{MappedPath: os.TempDir()})
 	backupContent, err = json.Marshal(backupData)
 	assert.NoError(t, err)
-	err = ioutil.WriteFile(backupFilePath, backupContent, 0666)
+	err = ioutil.WriteFile(backupFilePath, backupContent, os.ModePerm)
 	assert.NoError(t, err)
 	_, _, err = httpd.Loaddata(backupFilePath, "", "", http.StatusInternalServerError)
 	assert.NoError(t, err)
@@ -1422,7 +1422,7 @@ func TestLoaddata(t *testing.T) {
 	backupContent, err := json.Marshal(backupData)
 	assert.NoError(t, err)
 	backupFilePath := filepath.Join(backupsPath, "backup.json")
-	err = ioutil.WriteFile(backupFilePath, backupContent, 0666)
+	err = ioutil.WriteFile(backupFilePath, backupContent, os.ModePerm)
 	assert.NoError(t, err)
 	_, _, err = httpd.Loaddata(backupFilePath, "a", "", http.StatusBadRequest)
 	assert.NoError(t, err)
@@ -1489,7 +1489,7 @@ func TestLoaddataMode(t *testing.T) {
 	backupData.Users = append(backupData.Users, user)
 	backupContent, _ := json.Marshal(backupData)
 	backupFilePath := filepath.Join(backupsPath, "backup.json")
-	err := ioutil.WriteFile(backupFilePath, backupContent, 0666)
+	err := ioutil.WriteFile(backupFilePath, backupContent, os.ModePerm)
 	assert.NoError(t, err)
 	_, _, err = httpd.Loaddata(backupFilePath, "0", "0", http.StatusOK)
 	assert.NoError(t, err)
@@ -2821,7 +2821,7 @@ func createTestFile(path string, size int64) error {
 			return err
 		}
 	}
-	return ioutil.WriteFile(path, content, 0666)
+	return ioutil.WriteFile(path, content, os.ModePerm)
 }
 
 func getMultipartFormData(values url.Values, fileFieldName, filePath string) (bytes.Buffer, string, error) {

httpd/internal_test.go

@@ -441,7 +441,7 @@ func TestBasicAuth(t *testing.T) {
 	oldAuthPassword := authPassword
 	authUserFile := filepath.Join(os.TempDir(), "http_users.txt")
 	authUserData := []byte("test1:$2y$05$bcHSED7aO1cfLto6ZdDBOOKzlwftslVhtpIkRhAtSa4GuLmk5mola\n")
-	err := ioutil.WriteFile(authUserFile, authUserData, 0666)
+	err := ioutil.WriteFile(authUserFile, authUserData, os.ModePerm)
 	assert.NoError(t, err)
 	httpAuth, _ = newBasicAuthProvider(authUserFile)
 	_, _, err = GetVersion(http.StatusUnauthorized)
@@ -454,7 +454,7 @@ func TestBasicAuth(t *testing.T) {
 	defer resp.Body.Close()
 	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
 	authUserData = append(authUserData, []byte("test2:$apr1$gLnIkRIf$Xr/6aJfmIrihP4b2N2tcs/\n")...)
-	err = ioutil.WriteFile(authUserFile, authUserData, 0666)
+	err = ioutil.WriteFile(authUserFile, authUserData, os.ModePerm)
 	assert.NoError(t, err)
 	SetBaseURLAndCredentials(httpBaseURL, "test2", "password2")
 	_, _, err = GetVersion(http.StatusOK)
@@ -463,31 +463,31 @@ func TestBasicAuth(t *testing.T) {
 	_, _, err = GetVersion(http.StatusOK)
 	assert.Error(t, err)
 	authUserData = append(authUserData, []byte("test3:$apr1$gLnIkRIf$Xr/6$aJfmIr$ihP4b2N2tcs/\n")...)
-	err = ioutil.WriteFile(authUserFile, authUserData, 0666)
+	err = ioutil.WriteFile(authUserFile, authUserData, os.ModePerm)
 	assert.NoError(t, err)
 	SetBaseURLAndCredentials(httpBaseURL, "test3", "wrong_password")
 	_, _, err = GetVersion(http.StatusUnauthorized)
 	assert.NoError(t, err)
 	authUserData = append(authUserData, []byte("test4:$invalid$gLnIkRIf$Xr/6$aJfmIr$ihP4b2N2tcs/\n")...)
-	err = ioutil.WriteFile(authUserFile, authUserData, 0666)
+	err = ioutil.WriteFile(authUserFile, authUserData, os.ModePerm)
 	assert.NoError(t, err)
 	SetBaseURLAndCredentials(httpBaseURL, "test3", "password2")
 	_, _, err = GetVersion(http.StatusUnauthorized)
 	assert.NoError(t, err)
 	if runtime.GOOS != "windows" {
 		authUserData = append(authUserData, []byte("test5:$apr1$gLnIkRIf$Xr/6aJfmIrihP4b2N2tcs/\n")...)
-		err = ioutil.WriteFile(authUserFile, authUserData, 0666)
+		err = ioutil.WriteFile(authUserFile, authUserData, os.ModePerm)
 		assert.NoError(t, err)
 		err = os.Chmod(authUserFile, 0001)
 		assert.NoError(t, err)
 		SetBaseURLAndCredentials(httpBaseURL, "test5", "password2")
 		_, _, err = GetVersion(http.StatusUnauthorized)
 		assert.NoError(t, err)
-		err = os.Chmod(authUserFile, 0666)
+		err = os.Chmod(authUserFile, os.ModePerm)
 		assert.NoError(t, err)
 	}
 	authUserData = append(authUserData, []byte("\"foo\"bar\"\r\n")...)
-	err = ioutil.WriteFile(authUserFile, authUserData, 0666)
+	err = ioutil.WriteFile(authUserFile, authUserData, os.ModePerm)
 	assert.NoError(t, err)
 	SetBaseURLAndCredentials(httpBaseURL, "test2", "password2")
 	_, _, err = GetVersion(http.StatusUnauthorized)

metrics/metrics.go

@@ -36,40 +36,40 @@ var (
 		Help: "Total number of logged in users",
 	})
 
-	// totalUploads is the metric that reports the total number of successful SFTP/SCP uploads
+	// totalUploads is the metric that reports the total number of successful uploads
 	totalUploads = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "sftpgo_uploads_total",
-		Help: "The total number of successful SFTP/SCP uploads",
+		Help: "The total number of successful uploads",
 	})
 
-	// totalDownloads is the metric that reports the total number of successful SFTP/SCP downloads
+	// totalDownloads is the metric that reports the total number of successful downloads
 	totalDownloads = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "sftpgo_downloads_total",
-		Help: "The total number of successful SFTP/SCP downloads",
+		Help: "The total number of successful downloads",
 	})
 
-	// totalUploadErrors is the metric that reports the total number of SFTP/SCP upload errors
+	// totalUploadErrors is the metric that reports the total number of upload errors
 	totalUploadErrors = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "sftpgo_upload_errors_total",
-		Help: "The total number of SFTP/SCP upload errors",
+		Help: "The total number of upload errors",
 	})
 
-	// totalDownloadErrors is the metric that reports the total number of SFTP/SCP download errors
+	// totalDownloadErrors is the metric that reports the total number of download errors
 	totalDownloadErrors = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "sftpgo_download_errors_total",
-		Help: "The total number of SFTP/SCP download errors",
+		Help: "The total number of download errors",
 	})
 
-	// totalUploadSize is the metric that reports the total SFTP/SCP uploads size as bytes
+	// totalUploadSize is the metric that reports the total uploads size as bytes
 	totalUploadSize = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "sftpgo_upload_size",
-		Help: "The total SFTP/SCP upload size as bytes, partial uploads are included",
+		Help: "The total upload size as bytes, partial uploads are included",
 	})
 
-	// totalDownloadSize is the metric that reports the total SFTP/SCP downloads size as bytes
+	// totalDownloadSize is the metric that reports the total downloads size as bytes
 	totalDownloadSize = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "sftpgo_download_size",
-		Help: "The total SFTP/SCP download size as bytes, partial downloads are included",
+		Help: "The total download size as bytes, partial downloads are included",
 	})
 
 	// totalSSHCommands is the metric that reports the total number of executed SSH commands
@@ -90,6 +90,13 @@ var (
 		Help: "The total number of login attempts",
 	})
 
+	// totalNoAuthTryed is te metric that reports the total number of clients disconnected
+	// for inactivity before trying to login
+	totalNoAuthTryed = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "sftpgo_no_auth_total",
+		Help: "The total number of clients disconnected for inactivity before trying to login",
+	})
+
 	// totalLoginOK is the metric that reports the total number of successful logins
 	totalLoginOK = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "sftpgo_login_ok_total",
@@ -604,6 +611,12 @@ func AddLoginResult(authMethod string, err error) {
 	}
 }
 
+// AddNoAuthTryed increments the metric for clients disconnected
+// for inactivity before trying to login
+func AddNoAuthTryed() {
+	totalNoAuthTryed.Inc()
+}
+
 // HTTPRequestServed increments the metrics for HTTP requests
 func HTTPRequestServed(status int) {
 	totalHTTPRequests.Inc()

metrics/metrics_disabled.go

@@ -60,6 +60,10 @@ func AddLoginAttempt(authMethod string) {}
 // AddLoginResult increments the metrics for login results
 func AddLoginResult(authMethod string, err error) {}
 
+// AddNoAuthTryed increments the metric for clients disconnected
+// for inactivity before trying to login
+func AddNoAuthTryed() {}
+
 // HTTPRequestServed increments the metrics for HTTP requests
 func HTTPRequestServed(status int) {}
 

service/service.go

@@ -77,12 +77,6 @@ func (s *Service) Start() error {
 		return err
 	}
 
-	httpConfig := config.GetHTTPConfig()
-	httpConfig.Initialize(s.ConfigDir)
-
-	sftpdConf := config.GetSFTPDConfig()
-	httpdConf := config.GetHTTPDConfig()
-
 	if s.PortableMode == 1 {
 		// create the user for portable mode
 		err = dataprovider.AddUser(s.PortableUser)
@@ -92,6 +86,19 @@ func (s *Service) Start() error {
 		}
 	}
 
+	httpConfig := config.GetHTTPConfig()
+	httpConfig.Initialize(s.ConfigDir)
+
+	s.startServices()
+
+	return nil
+}
+
+func (s *Service) startServices() {
+	sftpdConf := config.GetSFTPDConfig()
+	ftpdConf := config.GetFTPDConfig()
+	httpdConf := config.GetHTTPDConfig()
+
 	go func() {
 		logger.Debug(logSender, "", "initializing SFTP server with config %+v", sftpdConf)
 		if err := sftpdConf.Initialize(s.ConfigDir); err != nil {
@@ -117,7 +124,18 @@ func (s *Service) Start() error {
 			logger.DebugToConsole("HTTP server not started, disabled in config file")
 		}
 	}
-	return nil
+	if ftpdConf.BindPort > 0 {
+		go func() {
+			if err := ftpdConf.Initialize(s.ConfigDir); err != nil {
+				logger.Error(logSender, "", "could not start FTP server: %v", err)
+				logger.ErrorToConsole("could not start FTP server: %v", err)
+				s.Error = err
+			}
+			s.Shutdown <- true
+		}()
+	} else {
+		logger.Debug(logSender, "", "FTP server not started, disabled in config file")
+	}
 }
 
 // Wait blocks until the service exits

service/service_portable.go

@@ -23,7 +23,8 @@ import (
 )
 
 // StartPortableMode starts the service in portable mode
-func (s *Service) StartPortableMode(sftpdPort int, enabledSSHCommands []string, advertiseService, advertiseCredentials bool) error {
+func (s *Service) StartPortableMode(sftpdPort, ftpPort int, enabledSSHCommands []string, advertiseService, advertiseCredentials bool,
+	ftpsCert, ftpsKey string) error {
 	if s.PortableMode != 1 {
 		return fmt.Errorf("service is not configured for portable mode")
 	}
@@ -67,14 +68,44 @@ func (s *Service) StartPortableMode(sftpdPort int, enabledSSHCommands []string,
 	}
 	config.SetSFTPDConfig(sftpdConf)
 
+	if ftpPort >= 0 {
+		ftpConf := config.GetFTPDConfig()
+		if ftpPort > 0 {
+			ftpConf.BindPort = ftpPort
+		} else {
+			ftpConf.BindPort = 49152 + rand.Intn(15000)
+		}
+		ftpConf.Banner = fmt.Sprintf("SFTPGo portable %v ready", version.Get().Version)
+		ftpConf.CertificateFile = ftpsCert
+		ftpConf.CertificateKeyFile = ftpsKey
+		config.SetFTPDConfig(ftpConf)
+	}
+
 	err = s.Start()
 	if err != nil {
 		return err
 	}
-	var mDNSService *zeroconf.Server
+
+	s.advertiseServices(advertiseService, advertiseCredentials)
+
+	var ftpInfo string
+	if config.GetFTPDConfig().BindPort > 0 {
+		ftpInfo = fmt.Sprintf("FTP port: %v", config.GetFTPDConfig().BindPort)
+	}
+	logger.InfoToConsole("Portable mode ready, SFTP port: %v, user: %#v, password: %#v, public keys: %v, directory: %#v, "+
+		"permissions: %+v, enabled ssh commands: %v file extensions filters: %+v %v", sftpdConf.BindPort, s.PortableUser.Username,
+		printablePassword, s.PortableUser.PublicKeys, s.getPortableDirToServe(), s.PortableUser.Permissions,
+		sftpdConf.EnabledSSHCommands, s.PortableUser.Filters.FileExtensions, ftpInfo)
+	return nil
+}
+
+func (s *Service) advertiseServices(advertiseService, advertiseCredentials bool) {
+	var mDNSServiceSFTP *zeroconf.Server
+	var mDNSServiceFTP *zeroconf.Server
+	var err error
 	if advertiseService {
 		meta := []string{
-			fmt.Sprintf("version=%v", version.GetAsString()),
+			fmt.Sprintf("version=%v", version.Get().Version),
 		}
 		if advertiseCredentials {
 			logger.InfoToConsole("Advertising credentials via multicast DNS")
@@ -85,7 +116,8 @@ func (s *Service) StartPortableMode(sftpdPort int, enabledSSHCommands []string,
 				logger.InfoToConsole("Unable to advertise key based credentials via multicast DNS, we don't have the private key")
 			}
 		}
-		mDNSService, err = zeroconf.Register(
+		sftpdConf := config.GetSFTPDConfig()
+		mDNSServiceSFTP, err = zeroconf.Register(
 			fmt.Sprintf("SFTPGo portable %v", sftpdConf.BindPort), // service instance name
 			"_sftp-ssh._tcp",   // service type and protocol
 			"local.",           // service domain
@@ -94,28 +126,41 @@ func (s *Service) StartPortableMode(sftpdPort int, enabledSSHCommands []string,
 			nil,                // register on all network interfaces
 		)
 		if err != nil {
-			mDNSService = nil
+			mDNSServiceSFTP = nil
 			logger.WarnToConsole("Unable to advertise SFTP service via multicast DNS: %v", err)
 		} else {
 			logger.InfoToConsole("SFTP service advertised via multicast DNS")
 		}
+		ftpdConf := config.GetFTPDConfig()
+		mDNSServiceFTP, err = zeroconf.Register(
+			fmt.Sprintf("SFTPGo portable %v", ftpdConf.BindPort),
+			"_ftp._tcp",
+			"local.",
+			ftpdConf.BindPort,
+			meta,
+			nil,
+		)
+		if err != nil {
+			mDNSServiceFTP = nil
+			logger.WarnToConsole("Unable to advertise FTP service via multicast DNS: %v", err)
+		} else {
+			logger.InfoToConsole("FTP service advertised via multicast DNS")
+		}
 	}
 	sig := make(chan os.Signal, 1)
 	signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
 	go func() {
 		<-sig
-		if mDNSService != nil {
-			logger.InfoToConsole("unregistering multicast DNS service")
-			mDNSService.Shutdown()
+		if mDNSServiceSFTP != nil {
+			logger.InfoToConsole("unregistering multicast DNS SFTP service")
+			mDNSServiceSFTP.Shutdown()
+		}
+		if mDNSServiceFTP != nil {
+			logger.InfoToConsole("unregistering multicast DNS FTP service")
+			mDNSServiceFTP.Shutdown()
 		}
 		s.Stop()
 	}()
-
-	logger.InfoToConsole("Portable mode ready, SFTP port: %v, user: %#v, password: %#v, public keys: %v, directory: %#v, "+
-		"permissions: %+v, enabled ssh commands: %v file extensions filters: %+v", sftpdConf.BindPort, s.PortableUser.Username,
-		printablePassword, s.PortableUser.PublicKeys, s.getPortableDirToServe(), s.PortableUser.Permissions,
-		sftpdConf.EnabledSSHCommands, s.PortableUser.Filters.FileExtensions)
-	return nil
 }
 
 func (s *Service) getPortableDirToServe() string {

service/service_windows.go

@@ -12,6 +12,7 @@ import (
 	"golang.org/x/sys/windows/svc/mgr"
 
 	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/ftpd"
 	"github.com/drakkan/sftpgo/httpd"
 	"github.com/drakkan/sftpgo/logger"
 )
@@ -93,6 +94,10 @@ loop:
 			if err != nil {
 				logger.Warn(logSender, "", "error reloading TLS certificate: %v", err)
 			}
+			err = ftpd.ReloadTLSCertificate()
+			if err != nil {
+				logger.Warn(logSender, "", "error reloading FTPD TLS certificate: %v", err)
+			}
 		case rotateLogCmd:
 			logger.Debug(logSender, "", "Received log file rotation request")
 			err := logger.RotateLogFile()

service/sighup_unix.go

@@ -8,6 +8,7 @@ import (
 	"syscall"
 
 	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/ftpd"
 	"github.com/drakkan/sftpgo/httpd"
 	"github.com/drakkan/sftpgo/logger"
 )
@@ -26,6 +27,10 @@ func registerSigHup() {
 			if err != nil {
 				logger.Warn(logSender, "", "error reloading TLS certificate: %v", err)
 			}
+			err = ftpd.ReloadTLSCertificate()
+			if err != nil {
+				logger.Warn(logSender, "", "error reloading FTPD TLS certificate: %v", err)
+			}
 		}
 	}()
 }

sftpd/handler.go

@@ -74,7 +74,7 @@ func (c *Connection) Fileread(request *sftp.Request) (io.ReaderAt, error) {
 
 	baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, p, request.Filepath, common.TransferDownload,
 		0, 0, false)
-	t := newTranfer(baseTransfer, nil, r, 0)
+	t := newTransfer(baseTransfer, nil, r, 0)
 
 	return t, nil
 }
@@ -164,14 +164,6 @@ func (c *Connection) Filecmd(request *sftp.Request) error {
 		return sftp.ErrSSHFxOpUnsupported
 	}
 
-	var fileLocation = p
-	if target != "" {
-		fileLocation = target
-	}
-
-	// we return if we remove a file or a dir so source path or target path always exists here
-	vfs.SetPathPermissions(c.Fs, fileLocation, c.User.GetUID(), c.User.GetGID())
-
 	return sftp.ErrSSHFxOk
 }
 
@@ -276,7 +268,7 @@ func (c *Connection) handleSFTPUploadToNewFile(resolvedPath, filePath, requestPa
 
 	baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, resolvedPath, requestPath,
 		common.TransferUpload, 0, 0, true)
-	t := newTranfer(baseTransfer, w, nil, quotaResult.GetRemainingSize())
+	t := newTransfer(baseTransfer, w, nil, quotaResult.GetRemainingSize())
 
 	return t, nil
 }
@@ -343,7 +335,7 @@ func (c *Connection) handleSFTPUploadToExistingFile(pflags sftp.FileOpenFlags, r
 
 	baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, resolvedPath, requestPath,
 		common.TransferUpload, minWriteOffset, initialSize, false)
-	t := newTranfer(baseTransfer, w, nil, maxWriteSize)
+	t := newTransfer(baseTransfer, w, nil, maxWriteSize)
 
 	return t, nil
 }

sftpd/internal_test.go

@@ -159,7 +159,7 @@ func TestUploadResumeInvalidOffset(t *testing.T) {
 	fs := vfs.NewOsFs("", os.TempDir(), nil)
 	conn := common.NewBaseConnection("", common.ProtocolSFTP, user, fs)
 	baseTransfer := common.NewBaseTransfer(file, conn, nil, file.Name(), testfile, common.TransferUpload, 10, 0, false)
-	transfer := newTranfer(baseTransfer, nil, nil, 0)
+	transfer := newTransfer(baseTransfer, nil, nil, 0)
 	_, err = transfer.WriteAt([]byte("test"), 0)
 	assert.Error(t, err, "upload with invalid offset must fail")
 	if assert.Error(t, transfer.ErrTransfer) {
@@ -187,7 +187,7 @@ func TestReadWriteErrors(t *testing.T) {
 	fs := vfs.NewOsFs("", os.TempDir(), nil)
 	conn := common.NewBaseConnection("", common.ProtocolSFTP, user, fs)
 	baseTransfer := common.NewBaseTransfer(file, conn, nil, file.Name(), testfile, common.TransferDownload, 0, 0, false)
-	transfer := newTranfer(baseTransfer, nil, nil, 0)
+	transfer := newTransfer(baseTransfer, nil, nil, 0)
 	err = file.Close()
 	assert.NoError(t, err)
 	_, err = transfer.WriteAt([]byte("test"), 0)
@@ -201,7 +201,7 @@ func TestReadWriteErrors(t *testing.T) {
 	r, _, err := pipeat.Pipe()
 	assert.NoError(t, err)
 	baseTransfer = common.NewBaseTransfer(nil, conn, nil, file.Name(), testfile, common.TransferDownload, 0, 0, false)
-	transfer = newTranfer(baseTransfer, nil, r, 0)
+	transfer = newTransfer(baseTransfer, nil, r, 0)
 	err = transfer.closeIO()
 	assert.NoError(t, err)
 	_, err = transfer.ReadAt(buf, 0)
@@ -211,7 +211,7 @@ func TestReadWriteErrors(t *testing.T) {
 	assert.NoError(t, err)
 	pipeWriter := vfs.NewPipeWriter(w)
 	baseTransfer = common.NewBaseTransfer(nil, conn, nil, file.Name(), testfile, common.TransferDownload, 0, 0, false)
-	transfer = newTranfer(baseTransfer, pipeWriter, nil, 0)
+	transfer = newTransfer(baseTransfer, pipeWriter, nil, 0)
 
 	err = r.Close()
 	assert.NoError(t, err)
@@ -243,7 +243,7 @@ func TestTransferCancelFn(t *testing.T) {
 	fs := vfs.NewOsFs("", os.TempDir(), nil)
 	conn := common.NewBaseConnection("", common.ProtocolSFTP, user, fs)
 	baseTransfer := common.NewBaseTransfer(file, conn, cancelFn, file.Name(), testfile, common.TransferDownload, 0, 0, false)
-	transfer := newTranfer(baseTransfer, nil, nil, 0)
+	transfer := newTransfer(baseTransfer, nil, nil, 0)
 
 	errFake := errors.New("fake error, this will trigger cancelFn")
 	transfer.TransferError(errFake)
@@ -273,7 +273,7 @@ func TestMockFsErrors(t *testing.T) {
 	}
 	testfile := filepath.Join(u.HomeDir, "testfile")
 	request := sftp.NewRequest("Remove", testfile)
-	err := ioutil.WriteFile(testfile, []byte("test"), 0666)
+	err := ioutil.WriteFile(testfile, []byte("test"), os.ModePerm)
 	assert.NoError(t, err)
 	_, err = c.Filewrite(request)
 	assert.EqualError(t, err, sftp.ErrSSHFxFailure.Error())
@@ -979,7 +979,7 @@ func TestSystemCommandErrors(t *testing.T) {
 	sshCmd.connection.channel = &mockSSHChannel
 	baseTransfer := common.NewBaseTransfer(nil, sshCmd.connection.BaseConnection, nil, "", "", common.TransferDownload,
 		0, 0, false)
-	transfer := newTranfer(baseTransfer, nil, nil, 0)
+	transfer := newTransfer(baseTransfer, nil, nil, 0)
 	destBuff := make([]byte, 65535)
 	dst := bytes.NewBuffer(destBuff)
 	_, err = transfer.copyFromReaderToWriter(dst, sshCmd.connection.channel)
@@ -1308,7 +1308,7 @@ func TestSCPErrorsMockFs(t *testing.T) {
 	assert.EqualError(t, err, errFake.Error())
 
 	testfile := filepath.Join(u.HomeDir, "testfile")
-	err = ioutil.WriteFile(testfile, []byte("test"), 0666)
+	err = ioutil.WriteFile(testfile, []byte("test"), os.ModePerm)
 	assert.NoError(t, err)
 	stat, err := os.Stat(u.HomeDir)
 	assert.NoError(t, err)
@@ -1476,7 +1476,7 @@ func TestSCPDownloadFileData(t *testing.T) {
 			args:       []string{"-r", "-f", "/tmp"},
 		},
 	}
-	err := ioutil.WriteFile(testfile, []byte("test"), 0666)
+	err := ioutil.WriteFile(testfile, []byte("test"), os.ModePerm)
 	assert.NoError(t, err)
 	stat, err := os.Stat(testfile)
 	assert.NoError(t, err)
@@ -1531,7 +1531,7 @@ func TestSCPUploadFiledata(t *testing.T) {
 
 	baseTransfer := common.NewBaseTransfer(file, scpCommand.connection.BaseConnection, nil, file.Name(),
 		"/"+testfile, common.TransferDownload, 0, 0, true)
-	transfer := newTranfer(baseTransfer, nil, nil, 0)
+	transfer := newTransfer(baseTransfer, nil, nil, 0)
 
 	err = scpCommand.getUploadFileData(2, transfer)
 	assert.Error(t, err, "upload must fail, we send a fake write error message")
@@ -1563,7 +1563,7 @@ func TestSCPUploadFiledata(t *testing.T) {
 	file, err = os.Create(testfile)
 	assert.NoError(t, err)
 	baseTransfer.File = file
-	transfer = newTranfer(baseTransfer, nil, nil, 0)
+	transfer = newTransfer(baseTransfer, nil, nil, 0)
 	transfer.Connection.AddTransfer(transfer)
 	err = scpCommand.getUploadFileData(2, transfer)
 	assert.Error(t, err, "upload must fail, we have not enough data to read")
@@ -1615,7 +1615,7 @@ func TestUploadError(t *testing.T) {
 	assert.NoError(t, err)
 	baseTransfer := common.NewBaseTransfer(file, connection.BaseConnection, nil, testfile,
 		testfile, common.TransferUpload, 0, 0, true)
-	transfer := newTranfer(baseTransfer, nil, nil, 0)
+	transfer := newTransfer(baseTransfer, nil, nil, 0)
 
 	errFake := errors.New("fake error")
 	transfer.TransferError(errFake)
@@ -1678,7 +1678,7 @@ func TestLoadHostKeys(t *testing.T) {
 	err := c.checkAndLoadHostKeys(configDir, serverConfig)
 	assert.Error(t, err)
 	testfile := filepath.Join(os.TempDir(), "invalidkey")
-	err = ioutil.WriteFile(testfile, []byte("some bytes"), 0666)
+	err = ioutil.WriteFile(testfile, []byte("some bytes"), os.ModePerm)
 	assert.NoError(t, err)
 	c.HostKeys = []string{testfile}
 	err = c.checkAndLoadHostKeys(configDir, serverConfig)
@@ -1726,7 +1726,7 @@ func TestCertCheckerInitErrors(t *testing.T) {
 	err := c.initializeCertChecker("")
 	assert.Error(t, err)
 	testfile := filepath.Join(os.TempDir(), "invalidkey")
-	err = ioutil.WriteFile(testfile, []byte("some bytes"), 0666)
+	err = ioutil.WriteFile(testfile, []byte("some bytes"), os.ModePerm)
 	assert.NoError(t, err)
 	c.TrustedUserCAKeys = []string{testfile}
 	err = c.initializeCertChecker("")

sftpd/scp.go

@@ -30,7 +30,7 @@ type scpCommand struct {
 
 func (c *scpCommand) handle() error {
 	common.Connections.Add(c.connection)
-	defer common.Connections.Remove(c.connection)
+	defer common.Connections.Remove(c.connection.GetID())
 
 	var err error
 	destPath := c.getDestPath()
@@ -226,7 +226,7 @@ func (c *scpCommand) handleUploadFile(resolvedPath, filePath string, sizeToRead
 
 	baseTransfer := common.NewBaseTransfer(file, c.connection.BaseConnection, cancelFn, resolvedPath, requestPath,
 		common.TransferUpload, 0, initialSize, isNewFile)
-	t := newTranfer(baseTransfer, w, nil, maxWriteSize)
+	t := newTransfer(baseTransfer, w, nil, maxWriteSize)
 
 	return c.getUploadFileData(sizeToRead, t)
 }
@@ -484,7 +484,7 @@ func (c *scpCommand) handleDownload(filePath string) error {
 
 	baseTransfer := common.NewBaseTransfer(file, c.connection.BaseConnection, cancelFn, p, filePath,
 		common.TransferDownload, 0, 0, false)
-	t := newTranfer(baseTransfer, nil, r, 0)
+	t := newTransfer(baseTransfer, nil, r, 0)
 
 	err = c.sendDownloadFileData(p, stat, t)
 	// we need to call Close anyway and return close error if any and

sftpd/server.go

@@ -223,7 +223,7 @@ func (c Configuration) configureLoginBanner(serverConfig *ssh.ServerConfig, conf
 				return banner
 			}
 		} else {
-			logger.WarnToConsole("unable to read login banner file: %v", err)
+			logger.WarnToConsole("unable to read SFTPD login banner file: %v", err)
 			logger.Warn(logSender, "", "unable to read login banner file: %v", err)
 		}
 	}
@@ -269,6 +269,7 @@ func (c Configuration) AcceptInboundConnection(conn net.Conn, config *ssh.Server
 		logger.Warn(logSender, "", "failed to accept an incoming connection: %v", err)
 		if _, ok := err.(*ssh.ServerAuthError); !ok {
 			logger.ConnectionFailedLog("", utils.GetIPFromRemoteAddress(remoteAddr.String()), "no_auth_tryed", err.Error())
+			metrics.AddNoAuthTryed()
 		}
 		return
 	}
@@ -348,7 +349,7 @@ func (c Configuration) AcceptInboundConnection(conn net.Conn, config *ssh.Server
 
 func (c Configuration) handleSftpConnection(channel ssh.Channel, connection *Connection) {
 	common.Connections.Add(connection)
-	defer common.Connections.Remove(connection)
+	defer common.Connections.Remove(connection.GetID())
 
 	// Create a new handler for the currently logged in user's server.
 	handler := c.createHandler(connection)

sftpd/sftpd_test.go

@@ -1405,6 +1405,8 @@ func TestPreLoginUserCreation(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, 1, len(users))
 	user := users[0]
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
 	err = os.RemoveAll(user.GetHomeDir())
 	assert.NoError(t, err)
 	err = dataprovider.Close()
@@ -4567,7 +4569,7 @@ func TestPermChmod(t *testing.T) {
 		assert.NoError(t, err)
 		err = sftpUploadFile(testFilePath, testFileName, testFileSize, client)
 		assert.NoError(t, err)
-		err = client.Chmod(testFileName, 0666)
+		err = client.Chmod(testFileName, os.ModePerm)
 		assert.Error(t, err, "chmod without permission should not succeed")
 		err = client.Remove(testFileName)
 		assert.NoError(t, err)
@@ -7084,7 +7086,7 @@ func createTestFile(path string, size int64) error {
 	if err != nil {
 		return err
 	}
-	return ioutil.WriteFile(path, content, 0666)
+	return ioutil.WriteFile(path, content, os.ModePerm)
 }
 
 func appendToTestFile(path string, size int64) error {
@@ -7093,7 +7095,7 @@ func appendToTestFile(path string, size int64) error {
 	if err != nil {
 		return err
 	}
-	f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0666)
+	f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, os.ModePerm)
 	if err != nil {
 		return err
 	}

sftpd/ssh_cmd.go

@@ -88,7 +88,7 @@ func processSSHCommand(payload []byte, connection *Connection, channel ssh.Chann
 
 func (c *sshCommand) handle() error {
 	common.Connections.Add(c.connection)
-	defer common.Connections.Remove(c.connection)
+	defer common.Connections.Remove(c.connection.GetID())
 
 	c.connection.UpdateLastActivity()
 	if utils.IsStringInSlice(c.command, sshHashCommands) {
@@ -354,7 +354,7 @@ func (c *sshCommand) executeSystemCommand(command systemCommand) error {
 		defer stdin.Close()
 		baseTransfer := common.NewBaseTransfer(nil, c.connection.BaseConnection, nil, command.fsPath, sshDestPath,
 			common.TransferUpload, 0, 0, false)
-		transfer := newTranfer(baseTransfer, nil, nil, remainingQuotaSize)
+		transfer := newTransfer(baseTransfer, nil, nil, remainingQuotaSize)
 
 		w, e := transfer.copyFromReaderToWriter(stdin, c.connection.channel)
 		c.connection.Log(logger.LevelDebug, "command: %#v, copy from remote command to sdtin ended, written: %v, "+
@@ -367,7 +367,7 @@ func (c *sshCommand) executeSystemCommand(command systemCommand) error {
 	go func() {
 		baseTransfer := common.NewBaseTransfer(nil, c.connection.BaseConnection, nil, command.fsPath, sshDestPath,
 			common.TransferDownload, 0, 0, false)
-		transfer := newTranfer(baseTransfer, nil, nil, 0)
+		transfer := newTransfer(baseTransfer, nil, nil, 0)
 
 		w, e := transfer.copyFromReaderToWriter(c.connection.channel, stdout)
 		c.connection.Log(logger.LevelDebug, "command: %#v, copy from sdtout to remote command ended, written: %v err: %v",
@@ -381,7 +381,7 @@ func (c *sshCommand) executeSystemCommand(command systemCommand) error {
 	go func() {
 		baseTransfer := common.NewBaseTransfer(nil, c.connection.BaseConnection, nil, command.fsPath, sshDestPath,
 			common.TransferDownload, 0, 0, false)
-		transfer := newTranfer(baseTransfer, nil, nil, 0)
+		transfer := newTransfer(baseTransfer, nil, nil, 0)
 
 		w, e := transfer.copyFromReaderToWriter(c.connection.channel.Stderr(), stderr)
 		c.connection.Log(logger.LevelDebug, "command: %#v, copy from sdterr to remote command ended, written: %v err: %v",

sftpd/transfer.go

@@ -32,7 +32,7 @@ type transfer struct {
 	maxWriteSize int64
 }
 
-func newTranfer(baseTransfer *common.BaseTransfer, pipeWriter *vfs.PipeWriter, pipeReader *pipeat.PipeReaderAt,
+func newTransfer(baseTransfer *common.BaseTransfer, pipeWriter *vfs.PipeWriter, pipeReader *pipeat.PipeReaderAt,
 	maxWriteSize int64) *transfer {
 	var writer writerAtCloser
 	var reader readerAtCloser

sftpgo.json

@@ -30,6 +30,20 @@
     ],
     "keyboard_interactive_auth_hook": ""
   },
+  "ftpd": {
+    "bind_port": 0,
+    "bind_address": "",
+    "banner": "",
+    "banner_file": "",
+    "active_transfers_port_non_20": false,
+    "force_passive_ip": "",
+    "passive_port_range": {
+      "start": 50000,
+      "end": 50100
+    },
+    "certificate_file": "",
+    "certificate_key_file": ""
+  },
   "data_provider": {
     "driver": "sqlite",
     "name": "sftpgo.db",

vfs/osfs.go

@@ -71,7 +71,7 @@ func (OsFs) Create(name string, flag int) (*os.File, *PipeWriter, func(), error)
 	if flag == 0 {
 		f, err = os.Create(name)
 	} else {
-		f, err = os.OpenFile(name, flag, 0666)
+		f, err = os.OpenFile(name, flag, os.ModePerm)
 	}
 	return f, nil, nil, err
 }